From bdc9acbf2e304683d8d8f19720c9acbabbf4d17d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 12:26:15 +0800 Subject: [PATCH 001/132] enh(sync): add snapshotEnable --- include/libs/sync/sync.h | 3 ++- source/dnode/mnode/impl/src/mndSync.c | 1 + source/dnode/vnode/src/vnd/vnodeSync.c | 2 ++ source/libs/sync/inc/syncElection.h | 1 + source/libs/sync/inc/syncRaftCfg.h | 8 +++++++- source/libs/sync/src/syncElection.c | 2 ++ source/libs/sync/src/syncMain.c | 7 +++++-- source/libs/sync/src/syncRaftCfg.c | 9 +++++++-- source/libs/sync/test/syncConfigChangeTest.cpp | 1 - source/libs/sync/test/syncRaftCfgTest.cpp | 6 +++++- 10 files changed, 32 insertions(+), 8 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a587ad6ef2..1f71c242ae 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -88,7 +88,7 @@ typedef struct SReConfigCbMeta { } SReConfigCbMeta; typedef struct SSnapshot { - void *data; + void* data; SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; } SSnapshot; @@ -145,6 +145,7 @@ typedef struct SSyncLogStore { typedef struct SSyncInfo { bool isStandBy; + bool snapshotEnable; SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index ecf7cabc13..8e18196314 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -146,6 +146,7 @@ int32_t mndInitSync(SMnode *pMnode) { syncInfo.pWal = pMgmt->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); syncInfo.isStandBy = pMgmt->standby; + syncInfo.snapshotEnable = true; SSyncCfg *pCfg = &syncInfo.syncCfg; pCfg->replicaNum = pMnode->replica; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 240b478ac0..ed10a0a8a4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -26,6 +26,8 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { + .isStandBy = false, + .snapshotEnable = false, .vgId = pVnode->config.vgId, .syncCfg = pVnode->config.syncCfg, .pWal = pVnode->pWal, diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 85a82dcfb7..7a3d848d46 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -39,6 +39,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode); +int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode); int32_t syncNodeElect(SSyncNode* pSyncNode); int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 1061e8bdc4..86c5fab87c 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -34,6 +34,7 @@ typedef struct SRaftCfg { TdFilePtr pFile; char path[TSDB_FILENAME_LEN * 2]; int8_t isStandBy; + int8_t snapshotEnable; } SRaftCfg; SRaftCfg *raftCfgOpen(const char *path); @@ -50,7 +51,12 @@ char * raftCfg2Str(SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); -int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path); +typedef struct SRaftCfgMeta { + int8_t isStandBy; + int8_t snapshotEnable; +} SRaftCfgMeta; + +int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path); // for debug ---------------------- void syncCfgPrint(SSyncCfg *pCfg); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 5101344b84..6f2088e967 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -49,6 +49,8 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) { return 0; } + int32_t syncNodeElect(SSyncNode* pSyncNode) { int32_t ret = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5ad8df11a9..deec71c4a4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -411,8 +411,11 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); if (!taosCheckExistFile(pSyncNode->configPath)) { - // create raft config file - ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), pSyncInfo->isStandBy, pSyncNode->configPath); + // create a new raft config file + SRaftCfgMeta meta; + meta.isStandBy = pSyncInfo->isStandBy; + meta.snapshotEnable = pSyncInfo->snapshotEnable; + ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); assert(ret == 0); } else { diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 86a07ca4f8..cb7562f2ca 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -148,6 +148,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { cJSON *pRoot = cJSON_CreateObject(); cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg))); cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy); + cJSON_AddNumberToObject(pRoot, "snapshotEnable", pRaftCfg->snapshotEnable); cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RaftCfg", pRoot); @@ -161,7 +162,7 @@ char *raftCfg2Str(SRaftCfg *pRaftCfg) { return serialized; } -int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) { +int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { assert(pCfg != NULL); TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); @@ -169,7 +170,8 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) { SRaftCfg raftCfg; raftCfg.cfg = *pCfg; - raftCfg.isStandBy = isStandBy; + raftCfg.isStandBy = meta.isStandBy; + raftCfg.snapshotEnable = meta.snapshotEnable; char *s = raftCfg2Str(&raftCfg); char buf[CONFIG_FILE_LEN]; @@ -194,6 +196,9 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy"); pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy); + cJSON *pJsonSnapshotEnable = cJSON_GetObjectItem(pJson, "snapshotEnable"); + pRaftCfg->snapshotEnable = cJSON_GetNumberValue(pJsonSnapshotEnable); + cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index 2880d89514..143d722b26 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -93,7 +93,6 @@ SSyncFSM* createFsm() { pFsm->FpGetSnapshot = GetSnapshotCb; pFsm->FpRestoreFinishCb = RestoreFinishCb; - pFsm->FpReConfigCb = ReConfigCb; return pFsm; diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index f5b24db651..564cbdb69a 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -71,7 +71,10 @@ void test3() { if (taosCheckExistFile(s)) { printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s); } else { - raftCfgCreateFile(pCfg, 7, s); + SRaftCfgMeta meta; + meta.isStandBy = 7; + meta.snapshotEnable = 9; + raftCfgCreateFile(pCfg, meta, s); printf("%s create json file: %s \n", (char*)__FUNCTION__, s); } @@ -94,6 +97,7 @@ void test5() { pCfg->cfg.myIndex = taosGetTimestampSec(); pCfg->isStandBy += 2; + pCfg->snapshotEnable += 3; raftCfgPersist(pCfg); printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); From d6c57bd372aee9e9180cd093b77596f8e3562ff3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 13:14:48 +0800 Subject: [PATCH 002/132] enh(sync): add snapshotEnable2 --- include/libs/sync/syncTools.h | 17 +++ source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mnode.c | 111 ++++++++++++------ source/libs/sync/inc/syncAppendEntries.h | 1 + source/libs/sync/inc/syncAppendEntriesReply.h | 1 + source/libs/sync/inc/syncInt.h | 16 ++- source/libs/sync/inc/syncReplication.h | 1 + source/libs/sync/inc/syncRequestVote.h | 1 + source/libs/sync/inc/syncRequestVoteReply.h | 1 + source/libs/sync/src/syncAppendEntries.c | 2 + source/libs/sync/src/syncAppendEntriesReply.c | 2 + source/libs/sync/src/syncElection.c | 8 +- source/libs/sync/src/syncMain.c | 19 ++- source/libs/sync/src/syncReplication.c | 11 +- source/libs/sync/src/syncRequestVote.c | 2 + source/libs/sync/src/syncRequestVoteReply.c | 2 + 16 files changed, 155 insertions(+), 41 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 4b160c9e61..bacdc4fbe3 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -394,7 +394,24 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg); +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +// option ---------------------------------- +typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); +typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg); +typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); +typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); +typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); +typedef int32_t (*FpOnAppendEntriesReplyCb)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +typedef int32_t (*FpOnTimeoutCb)(SSyncNode* pSyncNode, SyncTimeout* pMsg); + +// --------------------------------------------- + +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); // --------------------------------------------- #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index cec49a4cbe..f66827af3e 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -19,6 +19,7 @@ #include "mndDef.h" #include "sdb.h" +#include "sync.h" #include "syncTools.h" #include "tcache.h" #include "tdatablock.h" diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5458bc5126..73b63d76e6 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -368,41 +368,84 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); - if (pMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); - code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); - code = syncNodeOnPingCb(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); - code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); - code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); - code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); + + // ugly! use function pointer + if (syncNodeSnapshotEnable(pSyncNode)) { + if (pMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); + code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); + code = syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); + code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + + } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteSnapshotCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteReplySnapshotCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesSnapshotCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesReplySnapshotCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + } else { + mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + } + } else { - mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + if (pMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); + code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); + code = syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); + code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); + code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + } else { + mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + } } return code; diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 5999ef8300..98df22d51b 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -93,6 +93,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index c0c1f76707..e509a50dc4 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -41,6 +41,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 4100aa0216..a47dd0d74a 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -137,14 +137,25 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // callback + FpOnPingCb FpOnPing; + FpOnPingReplyCb FpOnPingReply; + FpOnClientRequestCb FpOnClientRequest; + FpOnTimeoutCb FpOnTimeout; + FpOnRequestVoteCb FpOnRequestVote; + FpOnRequestVoteReplyCb FpOnRequestVoteReply; + FpOnAppendEntriesCb FpOnAppendEntries; + FpOnAppendEntriesReplyCb FpOnAppendEntriesReply; + + /* int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg); + int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg); int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); - int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg); + */ // tools SSyncRespMgr* pSyncRespMgr; @@ -164,6 +175,9 @@ void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); +// option +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); + // ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg); int32_t syncNodePingSelf(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 6fe18dae38..4b1f5b4638 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -52,6 +52,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); +int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode); int32_t syncNodeReplicate(SSyncNode* pSyncNode); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index fd4ccd5371..3fe8dc0237 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -50,6 +50,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index bcaf71a541..ac47a8d026 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -45,6 +45,7 @@ extern "C" { // /\ UNCHANGED <> // int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 46be7597c7..fc20d6dd87 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -427,3 +427,5 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } + +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 77d85e2915..4c582dd39c 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -94,3 +94,5 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } + +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 6f2088e967..4fe2715b45 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -15,6 +15,7 @@ #include "syncElection.h" #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncVoteMgr.h" @@ -73,7 +74,12 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { return ret; } - ret = syncNodeRequestVotePeers(pSyncNode); + if (pSyncNode->pRaftCfg->snapshotEnable) { + ret = syncNodeRequestVotePeersSnapshot(pSyncNode); + } else { + ret = syncNodeRequestVotePeers(pSyncNode); + } + assert(ret == 0); syncNodeResetElectTimer(pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index deec71c4a4..8f415019ab 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -555,12 +555,20 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { pSyncNode->FpOnPing = syncNodeOnPingCb; pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb; pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb; - pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; - pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; - pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; - pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + if (pSyncNode->pRaftCfg->snapshotEnable) { + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; + } else { + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb; + } + // tools pSyncNode->pSyncRespMgr = syncRespMgrCreate(NULL, 0); assert(pSyncNode->pSyncRespMgr != NULL); @@ -675,6 +683,9 @@ void syncNodeClose(SSyncNode* pSyncNode) { // taosMemoryFree(pSyncNode); } +// option +bool syncNodeSnapshotEnable(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotEnable; } + // ping -------------- int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { syncPingLog2((char*)"==syncNodePing==", pMsg); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index d17e64d936..cdf05b5560 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -16,6 +16,7 @@ #include "syncReplication.h" #include "syncIndexMgr.h" #include "syncMessage.h" +#include "syncRaftCfg.h" #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" @@ -113,9 +114,17 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { return 0; } + int32_t syncNodeReplicate(SSyncNode* pSyncNode) { // start replicate - int32_t ret = syncNodeAppendEntriesPeers(pSyncNode); + int32_t ret = 0; + + if (pSyncNode->pRaftCfg->snapshotEnable) { + ret = syncNodeAppendEntriesPeersSnapshot(pSyncNode); + } else { + ret = syncNodeAppendEntriesPeers(pSyncNode); + } return ret; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 619a1546a9..92510344ca 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -77,3 +77,5 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } + +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index a6348dff50..d98b62008e 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -92,3 +92,5 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } + +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { return 0; } \ No newline at end of file From 43c383dcbc14824aa39ef7d3a8cec95a2d42754a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 17:19:31 +0800 Subject: [PATCH 003/132] enh(sync): add snapshotEnable2 --- include/libs/sync/syncTools.h | 6 +++--- source/libs/sync/inc/syncInt.h | 4 ++-- source/libs/sync/inc/syncSnapshot.h | 2 ++ source/libs/sync/src/syncSnapshot.c | 4 ++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index bacdc4fbe3..4db9dc0051 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -399,7 +399,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); -// option ---------------------------------- +// ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg); @@ -409,9 +409,9 @@ typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); typedef int32_t (*FpOnAppendEntriesReplyCb)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); typedef int32_t (*FpOnTimeoutCb)(SSyncNode* pSyncNode, SyncTimeout* pMsg); -// --------------------------------------------- - +// option ---------------------------------- bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); + // --------------------------------------------- #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index a47dd0d74a..64552fb10a 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -164,8 +164,8 @@ typedef struct SSyncNode { // sem_t restoreSem; bool restoreFinish; SSnapshot* pSnapshot; - SSyncSnapshotSender* pSender; - SSyncSnapshotReceiver* pReceiver; + SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; + SSyncSnapshotReceiver* receivers[TSDB_MAX_REPLICA]; } SSyncNode; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 43d1c0c0c3..2e2975fbc9 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -40,6 +40,8 @@ typedef struct SSyncSnapshotSender { SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); +void snapshotSenderStart(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); char * snapshotSender2Str(SSyncSnapshotSender *pSender); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ccb0e6071b..760d72235c 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -19,6 +19,10 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode) { return NULL; } void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {} +void snapshotSenderStart(SSyncSnapshotSender *pSender) {} + +void snapshotSenderStop(SSyncSnapshotSender *pSender) {} + int32_t snapshotSend(SSyncSnapshotSender *pSender) { return 0; } cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { return NULL; } From 79ae770a0bf4a7580bd50deac3802faa3caef612 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 17:31:55 +0800 Subject: [PATCH 004/132] enh(sync): add snapshotEnable2 --- source/libs/sync/inc/syncInt.h | 5 +++++ source/libs/sync/src/syncMain.c | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 64552fb10a..bd007b4799 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -219,6 +219,11 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode); void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); +// snapshot -------------- +bool syncNodeIsInSnapshot(SSyncNode* pSyncNode, SyncIndex index); +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); +int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8f415019ab..dca9e64b3d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1200,6 +1200,15 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { syncRequestVoteReplyDestroy(pMsg); } +// snapshot -------------- +bool syncNodeIsInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { return true; } + +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { return 0; } + +int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { + return 0; +} + // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); From 6be4119fe717db9111b21864b2f352de2d13e347 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 18:45:09 +0800 Subject: [PATCH 005/132] add syncNodeIsIndexInSnapshot syncNodeGetLastIndexTerm syncNodeGetPreIndexTerm --- source/libs/sync/inc/syncInt.h | 2 +- source/libs/sync/src/syncMain.c | 40 +++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index bd007b4799..41463e2f7f 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -220,7 +220,7 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- -bool syncNodeIsInSnapshot(SSyncNode* pSyncNode, SyncIndex index); +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index dca9e64b3d..dfd198c489 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1201,11 +1201,47 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { } // snapshot -------------- -bool syncNodeIsInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { return true; } +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + bool b = index <= snapshot.lastApplyIndex; + return b; +} -int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { return 0; } +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { + SyncIndex logLastIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + SyncIndex snapshotLastIndex = snapshot.lastApplyIndex; + + if (logLastIndex > snapshotLastIndex) { + *pLastIndex = logLastIndex; + *pLastTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); + } else { + *pLastIndex = snapshotLastIndex; + *pLastTerm = snapshot.lastApplyTerm; + } + return 0; +} int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { + ASSERT(index >= SYNC_INDEX_BEGIN); + int ret = 0; + + SyncIndex preIndex = index - 1; + if (syncNodeIsIndexInSnapshot(pSyncNode, preIndex)) { + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + ASSERT(preIndex == snapshot.lastApplyIndex); + *pPreIndex = snapshot.lastApplyIndex; + *pPreTerm = snapshot.lastApplyTerm; + } else { + SSyncRaftEntry *pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preIndex); + ASSERT(pPreEntry != NULL); + *pPreIndex = pPreEntry->index; + *pPreTerm = pPreEntry->term; + } + return 0; } From c5526ef9154f6a507512c36b1b35295b63b69b4b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 21:11:14 +0800 Subject: [PATCH 006/132] enh(sync): add SyncSnapshotRsp SyncSnapshotSend --- include/common/tmsgdef.h | 2 + include/libs/sync/syncTools.h | 68 +++++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 + source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncMessage.c | 321 ++++++++++++++++++++ 5 files changed, 394 insertions(+), 1 deletion(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..7d22f6efdc 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -221,6 +221,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_SNAPSHOT_SEND, "vnode-sync-snapshot-send", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SYNC_SNAPSHOT_RSP, "vnode-sync-snapshot-rsp", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 4db9dc0051..5d8d30329a 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -385,6 +385,74 @@ void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg); void syncApplyMsgLog(const SyncApplyMsg* pMsg); void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg); +// --------------------------------------------- +typedef struct SyncSnapshotSend { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + SyncTerm term; + SyncIndex lastIndex; + SyncTerm lastTerm; + int32_t seq; + int32_t ack; + uint32_t dataLen; + char data[]; +} SyncSnapshotSend; + +SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId); +void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg); +void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen); +void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg); +char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len); +SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len); +void syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg); +void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg); +SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg); +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg); + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg); +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg); +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg); +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg); + +// --------------------------------------------- +typedef struct SyncSnapshotRsp { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + SyncTerm term; + SyncIndex lastIndex; + SyncTerm lastTerm; + int32_t seq; + int32_t ack; +} SyncSnapshotRsp; + +SyncSnapshotRsp* syncSnapshotRspBuild(uint32_t dataLen, int32_t vgId); +void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg); +void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen); +void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg); +char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len); +SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len); +void syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg); +void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg); +SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg); +char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg); + +// for debug ---------------------- +void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg); +void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg); +void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg); +void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg); + // on message ---------------------- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f6350ba279..babf708f33 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -230,6 +230,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index dfd198c489..947e952276 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1236,7 +1236,7 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex *pPreIndex = snapshot.lastApplyIndex; *pPreTerm = snapshot.lastApplyTerm; } else { - SSyncRaftEntry *pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preIndex); + SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preIndex); ASSERT(pPreEntry != NULL); *pPreIndex = pPreEntry->index; *pPreTerm = pPreEntry->term; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 57cbdaaf79..7839670fd1 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1690,3 +1690,324 @@ void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) { sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } + +// --------------------------------------------- +SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId) { + uint32_t bytes = sizeof(SyncSnapshotSend) + dataLen; + SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_SNAPSHOT_SEND; + pMsg->dataLen = dataLen; + return pMsg; +} + +void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); + assert(pMsg->bytes == sizeof(SyncSnapshotSend) + pMsg->dataLen); +} + +char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncSnapshotSendSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncSnapshotSendDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} + +void syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncSnapshotSendSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg) { + syncSnapshotSendDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncSnapshotSend* pMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + assert(pMsg != NULL); + return pMsg; +} + +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); + return pJson; +} + +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { + cJSON* pJson = syncSnapshotSend2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + +// --------------------------------------------- +SyncSnapshotRsp* syncSnapshotRspBuild(uint32_t dataLen, int32_t vgId) { + uint32_t bytes = sizeof(SyncSnapshotRsp); + SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_VND_SYNC_SNAPSHOT_RSP; + return pMsg; +} + +void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + assert(buf != NULL); + syncSnapshotRspSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); + assert(pMsg != NULL); + syncSnapshotRspDeserialize(buf, len, pMsg); + assert(len == pMsg->bytes); + return pMsg; +} + +void syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncSnapshotRspSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg) { + syncSnapshotRspDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncSnapshotRsp* pMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + assert(pMsg != NULL); + return pMsg; +} + +cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotRsp", pRoot); + return pJson; +} + +char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg) { + cJSON* pJson = syncSnapshotRsp2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + printf("syncSnapshotRspPrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + printf("syncSnapshotRspPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + sTrace("syncSnapshotRspLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) { + char* serialized = syncSnapshotRsp2Str(pMsg); + sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} \ No newline at end of file From b9e4543a40c33bf0baf2fce95a229ca17d3d7e55 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 30 May 2022 21:21:51 +0800 Subject: [PATCH 007/132] enh(sync): add SyncSnapshotRsp SyncSnapshotSend --- source/libs/sync/inc/syncInt.h | 4 ++-- source/libs/sync/src/syncAppendEntries.c | 6 +++++- source/libs/sync/src/syncCommit.c | 6 +++++- source/libs/sync/src/syncMain.c | 13 ++++++++----- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 41463e2f7f..fbe31ef031 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -162,8 +162,8 @@ typedef struct SSyncNode { // restore state // sem_t restoreSem; - bool restoreFinish; - SSnapshot* pSnapshot; + bool restoreFinish; + // SSnapshot* pSnapshot; SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; SSyncSnapshotReceiver* receivers[TSDB_MAX_REPLICA]; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index fc20d6dd87..967ed5a00d 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -335,8 +335,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { cbMeta.currentTerm = ths->pRaftStore->currentTerm; cbMeta.flag = 0x11; + SSnapshot snapshot; + ASSERT(ths->pFsm->FpGetSnapshot != NULL); + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + bool needExecute = true; - if (ths->pSnapshot != NULL && cbMeta.index <= ths->pSnapshot->lastApplyIndex) { + if (cbMeta.index <= snapshot.lastApplyIndex) { needExecute = false; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 4a1a40a2d7..b683ab12b4 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -113,8 +113,12 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm; cbMeta.flag = 0x1; + SSnapshot snapshot; + ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL); + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + bool needExecute = true; - if (pSyncNode->pSnapshot != NULL && cbMeta.index <= pSyncNode->pSnapshot->lastApplyIndex) { + if (cbMeta.index <= snapshot.lastApplyIndex) { needExecute = false; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 947e952276..65101e0215 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -575,11 +575,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // restore state pSyncNode->restoreFinish = false; - pSyncNode->pSnapshot = NULL; - if (pSyncNode->pFsm->FpGetSnapshot != NULL) { - pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot)); - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot); - } + + // pSyncNode->pSnapshot = NULL; + // if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + // pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot)); + // pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot); + // } // tsem_init(&(pSyncNode->restoreSem), 0, 0); // start in syncNodeStart @@ -673,9 +674,11 @@ void syncNodeClose(SSyncNode* pSyncNode) { taosMemoryFree(pSyncNode->pFsm); } + /* if (pSyncNode->pSnapshot != NULL) { taosMemoryFree(pSyncNode->pSnapshot); } + */ // tsem_destroy(&pSyncNode->restoreSem); From 6d8733a232a9a235d86963114a439bf10b71779e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 16:14:19 +0800 Subject: [PATCH 008/132] enh(sync): add SSyncSnapshotSender --- include/libs/sync/syncTools.h | 7 +- source/dnode/mnode/impl/src/mndMain.c | 10 ++ source/libs/sync/inc/syncSnapshot.h | 42 +++-- source/libs/sync/src/syncMessage.c | 4 +- source/libs/sync/src/syncSnapshot.c | 234 +++++++++++++++++++++++++- 5 files changed, 270 insertions(+), 27 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 5d8d30329a..daa93c51ad 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -397,7 +397,6 @@ typedef struct SyncSnapshotSend { SyncIndex lastIndex; SyncTerm lastTerm; int32_t seq; - int32_t ack; uint32_t dataLen; char data[]; } SyncSnapshotSend; @@ -431,11 +430,10 @@ typedef struct SyncSnapshotRsp { SyncTerm term; SyncIndex lastIndex; SyncTerm lastTerm; - int32_t seq; int32_t ack; } SyncSnapshotRsp; -SyncSnapshotRsp* syncSnapshotRspBuild(uint32_t dataLen, int32_t vgId); +SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId); void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg); void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen); void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg); @@ -467,6 +465,9 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnSnapshotSendCb(SSyncNode* ths, SyncSnapshotSend* pMsg); +int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg); + // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index ba794b5a93..8968cd438e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -412,6 +412,16 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); code = syncNodeOnAppendEntriesReplySnapshotCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); code = TAOS_SYNC_PROPOSE_OTHER_ERROR; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 2e2975fbc9..5f4db4a0b2 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -25,42 +25,56 @@ extern "C" { #include #include "cJSON.h" #include "syncInt.h" +#include "syncMessage.h" #include "taosdef.h" +#define SYNC_SNAPSHOT_SEQ_INVALID -1 +#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2 +#define SYNC_SNAPSHOT_SEQ_BEGIN 0 +#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF + typedef struct SSyncSnapshotSender { - int32_t sending; - int32_t received; - bool finish; - void * pCurrentBlock; + bool start; + int32_t seq; + int32_t ack; + void *pReader; + void *pCurrentBlock; int32_t blockLen; + SSnapshot snapshot; int64_t sendingMS; - SSnapshot *pSnapshot; SSyncNode *pSyncNode; + int32_t replicaIndex; + SyncTerm term; } SSyncSnapshotSender; -SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode); +SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; - int32_t received; - int32_t progressIndex; - void * pCurrentBlock; + int32_t ack; + void *pWriter; + void *pCurrentBlock; int32_t len; - SSnapshot *pSnapshot; SSyncNode *pSyncNode; + int32_t replicaIndex; } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); + +int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); +int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index c04c310066..7314744e3b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1806,7 +1806,6 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); - cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); char* s; @@ -1858,7 +1857,7 @@ void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { } // --------------------------------------------- -SyncSnapshotRsp* syncSnapshotRspBuild(uint32_t dataLen, int32_t vgId) { +SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncSnapshotRsp); SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); @@ -1969,7 +1968,6 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); - cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 760d72235c..04795b65f3 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,27 +14,247 @@ */ #include "syncSnapshot.h" +#include "syncRaftStore.h" +#include "syncUtil.h" -SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode) { return NULL; } +static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); -void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {} +SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { + ASSERT(pSyncNode->pFsm->FpSnapshotStartRead != NULL); + ASSERT(pSyncNode->pFsm->FpSnapshotStopRead != NULL); + ASSERT(pSyncNode->pFsm->FpSnapshotDoRead != NULL); -void snapshotSenderStart(SSyncSnapshotSender *pSender) {} + SSyncSnapshotSender *pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + ASSERT(pSender != NULL); + memset(pSender, 0, sizeof(*pSender)); -void snapshotSenderStop(SSyncSnapshotSender *pSender) {} + pSender->start = false; + pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->pReader = NULL; + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; + pSender->sendingMS = 5000; + pSender->pSyncNode = pSyncNode; + pSender->replicaIndex = replicaIndex; + pSender->term = pSyncNode->pRaftStore->currentTerm; -int32_t snapshotSend(SSyncSnapshotSender *pSender) { return 0; } + return pSender; +} -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { return NULL; } +void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { + if (pSender != NULL) { + taosMemoryFree(pSender); + } +} -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { return NULL; } +static void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); + ASSERT(ret == 0); + + pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + syncSnapshotSendDestroy(pMsg); +} + +void snapshotSenderStart(SSyncSnapshotSender *pSender) { + if (!(pSender->start)) { + snapshotSenderDoStart(pSender); + pSender->start = true; + } else { + ASSERT(pSender->pSyncNode->pRaftStore->currentTerm >= pSender->term); + + // leader change + if (pSender->pSyncNode->pRaftStore->currentTerm > pSender->term) { + // force peer rollback + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = SYNC_SNAPSHOT_SEQ_FORCE_CLOSE; + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + syncSnapshotSendDestroy(pMsg); + + // close reader + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + + // start again + snapshotSenderDoStart(pSender); + } else { + // do nothing + } + } +} + +void snapshotSenderStop(SSyncSnapshotSender *pSender) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + pSender->blockLen = 0; + } + + ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, + &(pSender->pCurrentBlock), &(pSender->blockLen)); + ASSERT(ret == 0); +} + +// send msg from seq, seq is already updated +int32_t snapshotSend(SSyncSnapshotSender *pSender) { + // free memory last time (seq - 1) + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + pSender->blockLen = 0; + } + + // read data + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, + &(pSender->pCurrentBlock), &(pSender->blockLen)); + ASSERT(ret == 0); + + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + syncSnapshotSendDestroy(pMsg); + + return 0; +} + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilprintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pRoot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pRoot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode) { return NULL; } void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {} +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) {} + +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {} + int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver) { return 0; } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { return NULL; } char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { return NULL; } + +int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { + SSyncSnapshotReceiver *pReceiver = NULL; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { + pReceiver = (pSyncNode->receivers)[i]; + } + } + ASSERT(pReceiver != NULL); + + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pMsg->seq; + + if (pMsg->seq == 0) { + // begin + snapshotReceiverStart(pReceiver); + + } else if (pMsg->seq == -1) { + // end + snapshotReceiverStop(pReceiver); + // apply msg finish + + } else { + // transfering + // apply msg + } + + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + return 0; +} + +int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg) { return 0; } \ No newline at end of file From 21191ae2a82e45d78048b0102ee36200a1c88631 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 16:48:47 +0800 Subject: [PATCH 009/132] enh(sync): add SSyncSnapshotReceiver --- source/libs/sync/inc/syncSnapshot.h | 16 +++-- source/libs/sync/src/syncSnapshot.c | 101 +++++++++++++++++++++++++--- 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 5f4db4a0b2..99c92539f3 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -56,20 +56,22 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { - bool start; - int32_t ack; - void *pWriter; - void *pCurrentBlock; - int32_t len; + bool start; + + int32_t ack; + void *pWriter; + void *pCurrentBlock; + int32_t blockLen; + SyncTerm term; + SSyncNode *pSyncNode; int32_t replicaIndex; } SSyncSnapshotReceiver; -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode); +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 04795b65f3..6b0fea022a 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -115,10 +115,6 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender) { taosMemoryFree(pSender->pCurrentBlock); pSender->blockLen = 0; } - - ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, - &(pSender->pCurrentBlock), &(pSender->blockLen)); - ASSERT(ret == 0); } // send msg from seq, seq is already updated @@ -206,19 +202,102 @@ char *snapshotSender2Str(SSyncSnapshotSender *pSender) { } // ------------------------------------- -SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode) { return NULL; } +SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { + ASSERT(pSyncNode->pFsm->FpSnapshotStartWrite != NULL); + ASSERT(pSyncNode->pFsm->FpSnapshotStopWrite != NULL); + ASSERT(pSyncNode->pFsm->FpSnapshotDoWrite != NULL); -void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {} + SSyncSnapshotReceiver *pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); + ASSERT(pReceiver != NULL); + memset(pReceiver, 0, sizeof(*pReceiver)); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) {} + pReceiver->start = false; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + pReceiver->pWriter = NULL; + pReceiver->pCurrentBlock = NULL; + pReceiver->blockLen = 0; + pReceiver->pSyncNode = pSyncNode; + pReceiver->replicaIndex = replicaIndex; + pReceiver->term = pSyncNode->pRaftStore->currentTerm; -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {} + return pReceiver; +} -int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver) { return 0; } +void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { + if (pReceiver != NULL) { + taosMemoryFree(pReceiver); + } +} -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { return NULL; } +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { + if (!(pReceiver->start)) { + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); + ASSERT(ret == 0); + if (pReceiver->pCurrentBlock != NULL) { + taosMemoryFree(pReceiver->pCurrentBlock); + pReceiver->pCurrentBlock = NULL; + pReceiver->blockLen = 0; + } + pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + } else { + ASSERT(0); + } +} -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { return NULL; } +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); + ASSERT(ret == 0); + + if (pReceiver->pCurrentBlock != NULL) { + taosMemoryFree(pReceiver->pCurrentBlock); + pReceiver->blockLen = 0; + } +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pReceiver->blockLen); + + if (pReceiver->pCurrentBlock != NULL) { + char *s; + s = syncUtilprintBin((char *)(pReceiver->pCurrentBlock), pReceiver->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilprintBin2((char *)(pReceiver->pCurrentBlock), pReceiver->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = NULL; From ee9cdb2967e7fa4b96cbbc64997c529af6690c8d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 17:23:36 +0800 Subject: [PATCH 010/132] enh(sync): syncNodeOnSnapshotSendCb, syncNodeOnSnapshotRspCb --- source/dnode/mnode/impl/src/mndMain.c | 21 ++++---- source/libs/sync/src/syncSnapshot.c | 77 +++++++++++++++++++-------- 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8968cd438e..fbfd27c1b1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -369,15 +369,14 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); return TAOS_SYNC_PROPOSE_OTHER_ERROR; } - + char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); - - // ugly! use function pointer + // ToDo: ugly! use function pointer if (syncNodeSnapshotEnable(pSyncNode)) { if (pMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); @@ -413,14 +412,14 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { code = syncNodeOnAppendEntriesReplySnapshotCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); - code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); - code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 6b0fea022a..ab09c3eee9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -299,6 +299,7 @@ char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { return serialized; } +// receiver do something int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSyncSnapshotReceiver *pReceiver = NULL; for (int i = 0; i < pSyncNode->replicaNum; ++i) { @@ -308,32 +309,64 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } ASSERT(pReceiver != NULL); - SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pMsg->seq; + // state, term, seq/ack + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pMsg->seq; - if (pMsg->seq == 0) { - // begin - snapshotReceiverStart(pReceiver); + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { + // begin + snapshotReceiverStart(pReceiver); - } else if (pMsg->seq == -1) { - // end - snapshotReceiverStop(pReceiver); - // apply msg finish + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { + // end + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + snapshotReceiverStop(pReceiver); - } else { - // transfering - // apply msg + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); + snapshotReceiverStop(pReceiver); + + } else { + // transfering + if (pMsg->seq == pReceiver->ack + 1) { + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + } + } + + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); } - - SRpcMsg rpcMsg; - syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); - syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); return 0; } -int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg) { return 0; } \ No newline at end of file +// sender do something +int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + SSyncSnapshotSender *pSender = NULL; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { + pSender = (pSyncNode->senders)[i]; + } + } + ASSERT(pSender != NULL); + + // state, term, seq/ack + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + if (pMsg->ack == pSender->seq) { + pSender->ack = pMsg->ack; + snapshotSend(pSender); + (pSender->seq)++; + } + } + } + + return 0; +} \ No newline at end of file From ba212ce3962235c342fb5d0b985307c637906d67 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 17:37:57 +0800 Subject: [PATCH 011/132] enh(sync): syncNodeRequestVotePeersSnapshot --- include/libs/sync/syncTools.h | 1 + source/libs/sync/src/syncElection.c | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index daa93c51ad..e619f50c90 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -460,6 +460,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg); int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg); + int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg); int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 4fe2715b45..4a470af9c1 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -50,7 +50,25 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { return ret; } -int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) { return 0; } +int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = pSyncNode->peersId[i]; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + + ret = syncNodeGetLastIndexTerm(pSyncNode, &(pMsg->lastLogIndex), &(pMsg->lastLogTerm)); + ASSERT(ret == 0); + + ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg); + ASSERT(ret == 0); + syncRequestVoteDestroy(pMsg); + } + return ret; +} int32_t syncNodeElect(SSyncNode* pSyncNode) { int32_t ret = 0; From b9472dc386931b76ef648e1af4f4241d8d95fbaa Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 17:53:38 +0800 Subject: [PATCH 012/132] enh(sync): syncNodeAppendEntriesPeersSnapshot --- source/libs/sync/src/syncReplication.c | 93 +++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index cdf05b5560..864acb2993 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -20,6 +20,7 @@ #include "syncRaftEntry.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" // TLA+ Spec @@ -114,7 +115,97 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { return ret; } -int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { return 0; } +int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { + assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + + syncIndexMgrLog2("==syncNodeAppendEntriesPeersSnapshot== pNextIndex", pSyncNode->pNextIndex); + syncIndexMgrLog2("==syncNodeAppendEntriesPeersSnapshot== pMatchIndex", pSyncNode->pMatchIndex); + logStoreSimpleLog2("==syncNodeAppendEntriesPeersSnapshot==", pSyncNode->pLogStore); + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* pDestId = &(pSyncNode->peersId[i]); + + // set prevLogIndex + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + + SyncIndex preLogIndex; + SyncTerm preLogTerm; + + ret = syncNodeGetPreIndexTerm(pSyncNode, nextIndex, &preLogIndex, &preLogTerm); + ASSERT(ret == 0); + + // batch optimized + // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); + + if (syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { + // to claim leader + SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); + assert(pMsg != NULL); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + + syncAppendEntriesLog2("==syncNodeAppendEntriesPeersSnapshot==", pMsg); + + // send AppendEntries + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); + syncAppendEntriesDestroy(pMsg); + + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncUtilSameId(&((pSyncNode->replicasId)[i]), pDestId)) { + pSender = (pSyncNode->senders)[i]; + break; + } + } + ASSERT(pSender != NULL); + + snapshotSenderStart(pSender); + + } else { + SyncAppendEntries* pMsg = NULL; + SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, nextIndex); + if (pEntry != NULL) { + pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); + assert(pMsg != NULL); + + // add pEntry into msg + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(len == pEntry->bytes); + memcpy(pMsg->data, serialized, len); + + taosMemoryFree(serialized); + syncEntryDestory(pEntry); + + } else { + // maybe overflow, send empty record + pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); + assert(pMsg != NULL); + } + + assert(pMsg != NULL); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + + syncAppendEntriesLog2("==syncNodeAppendEntriesPeersSnapshot==", pMsg); + + // send AppendEntries + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); + syncAppendEntriesDestroy(pMsg); + } + } + + return ret; +} int32_t syncNodeReplicate(SSyncNode* pSyncNode) { // start replicate From a20cac89cc2fc9e8aa47cdbbe0a06b29d1f327ae Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 18:04:49 +0800 Subject: [PATCH 013/132] enh(sync): syncNodeOnRequestVoteSnapshotCb --- source/libs/sync/src/syncRequestVote.c | 40 +++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 5708135580..460362e728 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -78,4 +78,42 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } -int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { return 0; } \ No newline at end of file +int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteSnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteLog2(logBuf, pMsg); + + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + SyncIndex lastIndex; + SyncTerm lastTerm; + ret = syncNodeGetLastIndexTerm(ths, &lastIndex, &lastTerm); + ASSERT(ret == 0); + + bool logOK = (pMsg->lastLogTerm > lastTerm) || ((pMsg->lastLogTerm == lastTerm) && (pMsg->lastLogIndex >= lastIndex)); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + } + + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} \ No newline at end of file From fd823c867091125d12bea738912b33d3e01fa0f5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 18:12:17 +0800 Subject: [PATCH 014/132] enh(sync): syncNodeOnRequestVoteReplySnapshotCb --- source/libs/sync/src/syncRequestVoteReply.c | 57 ++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 91262ebe99..cff0167f1a 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -93,4 +93,59 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } -int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { return 0; } \ No newline at end of file +int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplySnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + // assert(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplySnapshotCb error term, receive:%lu current:%lu", + pMsg->term, ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + // add vote + voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader + if (voteGrantedMajority(ths->pVotesGranted)) { + if (!ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + + // prevent to leader again! + ths->pVotesGranted->toLeader = true; + } + } + } else { + ; + // do nothing + // UNCHANGED <> + } + } + + return ret; +} \ No newline at end of file From 7917716b4f3b090366b5880b7b169afd9d132cfd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 19:06:55 +0800 Subject: [PATCH 015/132] enh(sync): syncNodeOnAppendEntriesReplySnapshotCb --- source/libs/sync/src/syncAppendEntriesReply.c | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 5cda56cb9a..8ba169d093 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -95,4 +95,62 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } -int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { return 0; } \ No newline at end of file +int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplySnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesReplyLog2(logBuf, pMsg); + + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== before pMatchIndex", ths->pMatchIndex); + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnAppendEntriesReplySnapshotCb error term, receive:%lu current:%lu", + pMsg->term, ths->pRaftStore->currentTerm); + syncNodeLog2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + assert(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + + // notice! int64, uint64 + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== after pMatchIndex", ths->pMatchIndex); + + return ret; +} \ No newline at end of file From 057fda21cee08ede84a3ebb2c78fe7c3c9968d1b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 19:46:49 +0800 Subject: [PATCH 016/132] enh(sync): syncNodeOnAppendEntriesSnapshotCb --- source/libs/sync/src/syncAppendEntries.c | 350 ++++++++++++++++++++++- 1 file changed, 349 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 3f8728bffc..5f5e464984 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -432,4 +432,352 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } -int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return 0; } \ No newline at end of file +int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesSnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + syncAppendEntriesLog2(logBuf, pMsg); + + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + assert(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + assert(pMsg->dataLen >= 0); + + SyncIndex localPreLogIndex; + SyncTerm localPreLogTerm; + ret = syncNodeGetPreIndexTerm(ths, pMsg->prevLogIndex + 1, &localPreLogIndex, &localPreLogTerm); + ASSERT(ret == 0); + + SyncIndex localLastIndex; + SyncTerm localLastTerm; + ret = syncNodeGetLastIndexTerm(ths, &localLastIndex, &localLastTerm); + ASSERT(ret == 0); + + bool logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && (pMsg->prevLogIndex <= localLastIndex) && + (pMsg->prevLogTerm == localPreLogTerm)); + + // reject request + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + sTrace( + "syncNodeOnAppendEntriesSnapshotCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, " + "logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + sTrace( + "syncNodeOnAppendEntriesSnapshotCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + + syncNodeBecomeFollower(ths); + + // ret or reply? + return ret; + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + // preIndex = -1, or has preIndex entry in local log + assert(pMsg->prevLogIndex <= localLastIndex); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < localLastIndex; + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + sTrace( + "syncNodeOnAppendEntriesSnapshotCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, " + "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // log not match, conflict + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("syncNodeOnAppendEntriesSnapshotCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, + delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); + + // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); + + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pAppendEntry->index; + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + assert(0); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index from leader + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + // execute fsm + if (ths->pFsm != NULL) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + if (i != SYNC_INDEX_INVALID) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, i); + assert(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = 0x11; + + SSnapshot snapshot; + ASSERT(ths->pFsm->FpGetSnapshot != NULL); + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + + bool needExecute = true; + if (cbMeta.index <= snapshot.lastApplyIndex) { + needExecute = false; + } + + if (needExecute) { + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + + // config change + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; + + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); + + // update new config myIndex + bool hit = false; + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + hit = true; + break; + } + } + + SReConfigCbMeta cbMeta = {0}; + bool isDrop; + + // I am in newConfig + if (hit) { + syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); + + // change isStandBy to normal + if (!isDrop) { + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths); + } else { + syncNodeBecomeFollower(ths); + } + } + + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); + taosMemoryFree(sOld); + taosMemoryFree(sNew); + } + + // always call FpReConfigCb + if (ths->pFsm->FpReConfigCb != NULL) { + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.index = pEntry->index; + cbMeta.term = pEntry->term; + cbMeta.oldCfg = oldSyncCfg; + cbMeta.flag = 0x11; + cbMeta.isDrop = isDrop; + ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); + } + } + + // restore finish + if (pEntry->index == ths->pLogStore->getLastIndex(ths->pLogStore)) { + if (ths->restoreFinish == false) { + if (ths->pFsm->FpRestoreFinishCb != NULL) { + ths->pFsm->FpRestoreFinishCb(ths->pFsm); + } + ths->restoreFinish = true; + sInfo("==syncNodeOnAppendEntriesSnapshotCb== restoreFinish set true %p vgId:%d", ths, ths->vgId); + + /* + tsem_post(&ths->restoreSem); + sInfo("==syncNodeOnAppendEntriesSnapshotCb== RestoreFinish tsem_post %p", ths); + */ + } + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } + } + } + } + } + + return ret; +} \ No newline at end of file From 995317bf9cca443d2740c4ee413c213f6e86da8c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 20:38:59 +0800 Subject: [PATCH 017/132] enh(sync): add test syncSnapshotSendTest.cpp --- source/libs/sync/test/CMakeLists.txt | 14 +++ .../libs/sync/test/syncSnapshotSendTest.cpp | 100 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 source/libs/sync/test/syncSnapshotSendTest.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index cfbdf0e961..20188d51f3 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -38,6 +38,7 @@ add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") add_executable(syncConfigChangeTest "") +add_executable(syncSnapshotSendTest "") target_sources(syncTest @@ -200,6 +201,10 @@ target_sources(syncConfigChangeTest PRIVATE "syncConfigChangeTest.cpp" ) +target_sources(syncSnapshotSendTest + PRIVATE + "syncSnapshotSendTest.cpp" +) target_include_directories(syncTest @@ -402,6 +407,11 @@ target_include_directories(syncConfigChangeTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncSnapshotSendTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -564,6 +574,10 @@ target_link_libraries(syncConfigChangeTest sync gtest_main ) +target_link_libraries(syncSnapshotSendTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp new file mode 100644 index 0000000000..8985a3f1a2 --- /dev/null +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -0,0 +1,100 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncSnapshotSend *createMsg() { + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(20, 1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 11; + pMsg->lastIndex = 22; + pMsg->lastTerm = 33; + pMsg->seq = 44; + strcpy(pMsg->data, "hello world"); + return pMsg; +} + +void test1() { + SyncSnapshotSend *pMsg = createMsg(); + syncSnapshotSendLog2((char *)"test1:", pMsg); + syncSnapshotSendDestroy(pMsg); +} + +void test2() { + SyncSnapshotSend *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncSnapshotSendSerialize(pMsg, serialized, len); + SyncSnapshotSend *pMsg2 = syncSnapshotSendBuild(pMsg->dataLen, 1000); + syncSnapshotSendDeserialize(serialized, len, pMsg2); + syncSnapshotSendLog2((char *)"test2: syncSnapshotSendSerialize -> syncSnapshotSendDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test3() { + SyncSnapshotSend *pMsg = createMsg(); + uint32_t len; + char * serialized = syncSnapshotSendSerialize2(pMsg, &len); + SyncSnapshotSend *pMsg2 = syncSnapshotSendDeserialize2(serialized, len); + syncSnapshotSendLog2((char *)"test3: syncSnapshotSendSerialize2 -> syncSnapshotSendDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test4() { + SyncSnapshotSend *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotSend *pMsg2 = (SyncSnapshotSend *)taosMemoryMalloc(rpcMsg.contLen); + syncSnapshotSendFromRpcMsg(&rpcMsg, pMsg2); + syncSnapshotSendLog2((char *)"test4: syncSnapshotSend2RpcMsg -> syncSnapshotSendFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +void test5() { + SyncSnapshotSend *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotSend *pMsg2 = syncSnapshotSendFromRpcMsg2(&rpcMsg); + syncSnapshotSendLog2((char *)"test5: syncSnapshotSend2RpcMsg -> syncSnapshotSendFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotSendDestroy(pMsg); + syncSnapshotSendDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} From 7922997806c83c8dc53fcee356fe2ea46e8b5fef Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 20:54:39 +0800 Subject: [PATCH 018/132] enh(sync): add test syncSnapshotRspTest.cpp --- source/libs/sync/test/CMakeLists.txt | 14 +++ source/libs/sync/test/syncSnapshotRspTest.cpp | 99 +++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 source/libs/sync/test/syncSnapshotRspTest.cpp diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 20188d51f3..b348051eb6 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -39,6 +39,7 @@ add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") add_executable(syncConfigChangeTest "") add_executable(syncSnapshotSendTest "") +add_executable(syncSnapshotRspTest "") target_sources(syncTest @@ -205,6 +206,10 @@ target_sources(syncSnapshotSendTest PRIVATE "syncSnapshotSendTest.cpp" ) +target_sources(syncSnapshotRspTest + PRIVATE + "syncSnapshotRspTest.cpp" +) target_include_directories(syncTest @@ -412,6 +417,11 @@ target_include_directories(syncSnapshotSendTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncSnapshotRspTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -578,6 +588,10 @@ target_link_libraries(syncSnapshotSendTest sync gtest_main ) +target_link_libraries(syncSnapshotRspTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp new file mode 100644 index 0000000000..9e455c6efd --- /dev/null +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -0,0 +1,99 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncSnapshotRsp *createMsg() { + SyncSnapshotRsp *pMsg = syncSnapshotRspBuild(1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 11; + pMsg->lastIndex = 22; + pMsg->lastTerm = 33; + pMsg->ack = 44; + return pMsg; +} + +void test1() { + SyncSnapshotRsp *pMsg = createMsg(); + syncSnapshotRspLog2((char *)"test1:", pMsg); + syncSnapshotRspDestroy(pMsg); +} + +void test2() { + SyncSnapshotRsp *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncSnapshotRspSerialize(pMsg, serialized, len); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspBuild(1000); + syncSnapshotRspDeserialize(serialized, len, pMsg2); + syncSnapshotRspLog2((char *)"test2: syncSnapshotRspSerialize -> syncSnapshotRspDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test3() { + SyncSnapshotRsp *pMsg = createMsg(); + uint32_t len; + char * serialized = syncSnapshotRspSerialize2(pMsg, &len); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspDeserialize2(serialized, len); + syncSnapshotRspLog2((char *)"test3: syncSnapshotRspSerialize2 -> syncSnapshotRspDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test4() { + SyncSnapshotRsp *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotRsp *pMsg2 = (SyncSnapshotRsp *)taosMemoryMalloc(rpcMsg.contLen); + syncSnapshotRspFromRpcMsg(&rpcMsg, pMsg2); + syncSnapshotRspLog2((char *)"test4: syncSnapshotRsp2RpcMsg -> syncSnapshotRspFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +void test5() { + SyncSnapshotRsp *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); + SyncSnapshotRsp *pMsg2 = syncSnapshotRspFromRpcMsg2(&rpcMsg); + syncSnapshotRspLog2((char *)"test5: syncSnapshotRsp2RpcMsg -> syncSnapshotRspFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncSnapshotRspDestroy(pMsg); + syncSnapshotRspDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} From 9988e85f852128bf76e7d7c4221012a2c0d9189a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 31 May 2022 21:31:07 +0800 Subject: [PATCH 019/132] add test sender, receiver --- source/libs/sync/inc/syncSnapshot.h | 16 ++--- source/libs/sync/src/syncSnapshot.c | 4 +- source/libs/sync/test/CMakeLists.txt | 28 ++++++++ .../sync/test/syncSnapshotReceiverTest.cpp | 65 +++++++++++++++++ source/libs/sync/test/syncSnapshotRspTest.cpp | 12 ++-- .../libs/sync/test/syncSnapshotSendTest.cpp | 12 ++-- .../libs/sync/test/syncSnapshotSenderTest.cpp | 70 +++++++++++++++++++ 7 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 source/libs/sync/test/syncSnapshotReceiverTest.cpp create mode 100644 source/libs/sync/test/syncSnapshotSenderTest.cpp diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 99c92539f3..e1a252eed4 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -37,8 +37,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -52,15 +52,15 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; - void *pCurrentBlock; + void * pWriter; + void * pCurrentBlock; int32_t blockLen; SyncTerm term; @@ -72,8 +72,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ab09c3eee9..5fc49d9ffa 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -175,9 +175,9 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON *pSnapshot = cJSON_CreateObject(); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pRoot, "lastApplyIndex", u64buf); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pRoot, "lastApplyTerm", u64buf); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index b348051eb6..d1e747f29d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -40,6 +40,8 @@ add_executable(syncApplyMsgTest "") add_executable(syncConfigChangeTest "") add_executable(syncSnapshotSendTest "") add_executable(syncSnapshotRspTest "") +add_executable(syncSnapshotSenderTest "") +add_executable(syncSnapshotReceiverTest "") target_sources(syncTest @@ -210,6 +212,14 @@ target_sources(syncSnapshotRspTest PRIVATE "syncSnapshotRspTest.cpp" ) +target_sources(syncSnapshotSenderTest + PRIVATE + "syncSnapshotSenderTest.cpp" +) +target_sources(syncSnapshotReceiverTest + PRIVATE + "syncSnapshotReceiverTest.cpp" +) target_include_directories(syncTest @@ -422,6 +432,16 @@ target_include_directories(syncSnapshotRspTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncSnapshotSenderTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncSnapshotReceiverTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -592,6 +612,14 @@ target_link_libraries(syncSnapshotRspTest sync gtest_main ) +target_link_libraries(syncSnapshotSenderTest + sync + gtest_main +) +target_link_libraries(syncSnapshotReceiverTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp new file mode 100644 index 0000000000..e912ecc597 --- /dev/null +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -0,0 +1,65 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftStore.h" +#include "syncSnapshot.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} + +void RestoreFinishCb(struct SSyncFSM* pFsm) {} +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {} + +int32_t GetSnapshot(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { return 0; } +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } + +SSyncSnapshotReceiver* createReceiver() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); + pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + pSyncNode->pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pSyncNode->pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pSyncNode->pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + + SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, 2); + pReceiver->start = true; + pReceiver->ack = 20; + pReceiver->pWriter = (void*)0x11; + pReceiver->blockLen = 20; + pReceiver->pCurrentBlock = taosMemoryMalloc(pReceiver->blockLen); + snprintf((char*)(pReceiver->pCurrentBlock), pReceiver->blockLen, "%s", "hello"); + pReceiver->term = 66; + + return pReceiver; +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + SSyncSnapshotReceiver* pReceiver = createReceiver(); + sTrace("%s", snapshotReceiver2Str(pReceiver)); + + return 0; +} diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 9e455c6efd..3c1ce11456 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -35,8 +35,8 @@ void test1() { void test2() { SyncSnapshotRsp *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); syncSnapshotRspSerialize(pMsg, serialized, len); SyncSnapshotRsp *pMsg2 = syncSnapshotRspBuild(1000); syncSnapshotRspDeserialize(serialized, len, pMsg2); @@ -49,8 +49,8 @@ void test2() { void test3() { SyncSnapshotRsp *pMsg = createMsg(); - uint32_t len; - char * serialized = syncSnapshotRspSerialize2(pMsg, &len); + uint32_t len; + char * serialized = syncSnapshotRspSerialize2(pMsg, &len); SyncSnapshotRsp *pMsg2 = syncSnapshotRspDeserialize2(serialized, len); syncSnapshotRspLog2((char *)"test3: syncSnapshotRspSerialize2 -> syncSnapshotRspDeserialize2 ", pMsg2); @@ -61,7 +61,7 @@ void test3() { void test4() { SyncSnapshotRsp *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); SyncSnapshotRsp *pMsg2 = (SyncSnapshotRsp *)taosMemoryMalloc(rpcMsg.contLen); syncSnapshotRspFromRpcMsg(&rpcMsg, pMsg2); @@ -74,7 +74,7 @@ void test4() { void test5() { SyncSnapshotRsp *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pMsg, &rpcMsg); SyncSnapshotRsp *pMsg2 = syncSnapshotRspFromRpcMsg2(&rpcMsg); syncSnapshotRspLog2((char *)"test5: syncSnapshotRsp2RpcMsg -> syncSnapshotRspFromRpcMsg2 ", pMsg2); diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 8985a3f1a2..ca4ac2abeb 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -36,8 +36,8 @@ void test1() { void test2() { SyncSnapshotSend *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); syncSnapshotSendSerialize(pMsg, serialized, len); SyncSnapshotSend *pMsg2 = syncSnapshotSendBuild(pMsg->dataLen, 1000); syncSnapshotSendDeserialize(serialized, len, pMsg2); @@ -50,8 +50,8 @@ void test2() { void test3() { SyncSnapshotSend *pMsg = createMsg(); - uint32_t len; - char * serialized = syncSnapshotSendSerialize2(pMsg, &len); + uint32_t len; + char * serialized = syncSnapshotSendSerialize2(pMsg, &len); SyncSnapshotSend *pMsg2 = syncSnapshotSendDeserialize2(serialized, len); syncSnapshotSendLog2((char *)"test3: syncSnapshotSendSerialize2 -> syncSnapshotSendDeserialize2 ", pMsg2); @@ -62,7 +62,7 @@ void test3() { void test4() { SyncSnapshotSend *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); SyncSnapshotSend *pMsg2 = (SyncSnapshotSend *)taosMemoryMalloc(rpcMsg.contLen); syncSnapshotSendFromRpcMsg(&rpcMsg, pMsg2); @@ -75,7 +75,7 @@ void test4() { void test5() { SyncSnapshotSend *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); SyncSnapshotSend *pMsg2 = syncSnapshotSendFromRpcMsg2(&rpcMsg); syncSnapshotSendLog2((char *)"test5: syncSnapshotSend2RpcMsg -> syncSnapshotSendFromRpcMsg2 ", pMsg2); diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp new file mode 100644 index 0000000000..76d021004f --- /dev/null +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -0,0 +1,70 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftStore.h" +#include "syncSnapshot.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {} + +void RestoreFinishCb(struct SSyncFSM* pFsm) {} +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {} + +int32_t GetSnapshot(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { return 0; } +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } + +SSyncSnapshotSender* createSender() { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); + pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead; + pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead; + pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead; + + SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2); + pSender->start = true; + pSender->seq = 10; + pSender->ack = 20; + pSender->pReader = (void*)0x11; + pSender->blockLen = 20; + pSender->pCurrentBlock = taosMemoryMalloc(pSender->blockLen); + snprintf((char*)(pSender->pCurrentBlock), pSender->blockLen, "%s", "hello"); + + pSender->snapshot.lastApplyIndex = 99; + pSender->snapshot.lastApplyTerm = 88; + pSender->sendingMS = 77; + pSender->term = 66; + + return pSender; +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + SSyncSnapshotSender* pSender = createSender(); + sTrace("%s", snapshotSender2Str(pSender)); + + return 0; +} From 71c57d7769d54ffc65e7d694c494dfeebaec9f39 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 11:28:33 +0800 Subject: [PATCH 020/132] refactor: sender code --- include/libs/sync/syncTools.h | 4 +- source/libs/sync/inc/syncSnapshot.h | 18 +++--- source/libs/sync/src/syncMain.c | 46 +++++++++++++++ source/libs/sync/src/syncSnapshot.c | 87 +++++++++++++++++++++-------- 4 files changed, 121 insertions(+), 34 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index e619f50c90..f8d5768b5d 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -394,8 +394,8 @@ typedef struct SyncSnapshotSend { SRaftId destId; SyncTerm term; - SyncIndex lastIndex; - SyncTerm lastTerm; + SyncIndex lastIndex; // lastIndex of snapshot + SyncTerm lastTerm; // lastTerm of snapshot int32_t seq; uint32_t dataLen; char data[]; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index e1a252eed4..281e30f44e 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -33,12 +33,14 @@ extern "C" { #define SYNC_SNAPSHOT_SEQ_BEGIN 0 #define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF +#define SYNC_SNAPSHOT_RETRY_MS 5000 + typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -52,15 +54,15 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void * pWriter; - void * pCurrentBlock; + void *pWriter; + void *pCurrentBlock; int32_t blockLen; SyncTerm term; @@ -72,8 +74,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b26fda446e..9ccd98f557 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -29,6 +29,7 @@ #include "syncRequestVote.h" #include "syncRequestVoteReply.h" #include "syncRespMgr.h" +#include "syncSnapshot.h" #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -583,6 +584,20 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // } // tsem_init(&(pSyncNode->restoreSem), 0, 0); + // snapshot senders + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); + // ASSERT(pSender != NULL); + (pSyncNode->senders)[i] = pSender; + } + + // snapshot receivers + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, i); + // ASSERT(pReceiver != NULL); + (pSyncNode->receivers)[i] = pReceiver; + } + // start in syncNodeStart // start raft // syncNodeBecomeFollower(pSyncNode); @@ -674,6 +689,20 @@ void syncNodeClose(SSyncNode* pSyncNode) { taosMemoryFree(pSyncNode->pFsm); } + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + if ((pSyncNode->senders)[i] != NULL) { + snapshotSenderDestroy((pSyncNode->senders)[i]); + (pSyncNode->senders)[i] = NULL; + } + } + + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + if ((pSyncNode->receivers)[i] != NULL) { + snapshotReceiverDestroy((pSyncNode->receivers)[i]); + (pSyncNode->receivers)[i] = NULL; + } + } + /* if (pSyncNode->pSnapshot != NULL) { taosMemoryFree(pSyncNode->pSnapshot); @@ -969,6 +998,23 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + // restoreFinish + cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); + + // snapshot senders + cJSON* pSenders = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "senders", pSenders); + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); + } + + // snapshot receivers + cJSON* pReceivers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "receivers", pReceivers); + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pReceivers, snapshotReceiver2Json((pSyncNode->receivers)[i])); + } } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 5fc49d9ffa..b9cd5422c9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -20,24 +20,28 @@ static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { - ASSERT(pSyncNode->pFsm->FpSnapshotStartRead != NULL); - ASSERT(pSyncNode->pFsm->FpSnapshotStopRead != NULL); - ASSERT(pSyncNode->pFsm->FpSnapshotDoRead != NULL); + bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && + (pSyncNode->pFsm->FpSnapshotDoRead != NULL); - SSyncSnapshotSender *pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); - ASSERT(pSender != NULL); - memset(pSender, 0, sizeof(*pSender)); + SSyncSnapshotSender *pSender = NULL; + if (condition) { + pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + ASSERT(pSender != NULL); + memset(pSender, 0, sizeof(*pSender)); - pSender->start = false; - pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; - pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; - pSender->pReader = NULL; - pSender->pCurrentBlock = NULL; - pSender->blockLen = 0; - pSender->sendingMS = 5000; - pSender->pSyncNode = pSyncNode; - pSender->replicaIndex = replicaIndex; - pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->start = false; + pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->pReader = NULL; + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; + pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; + pSender->pSyncNode = pSyncNode; + pSender->replicaIndex = replicaIndex; + pSender->term = pSyncNode->pRaftStore->currentTerm; + } else { + sInfo("snapshotSenderCreate cannot create sender"); + } return pSender; } @@ -48,38 +52,47 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { } } +// begin send snapshot (current term, seq begin) static void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + // open snapshot reader + ASSERT(pSender->pReader == NULL); int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); ASSERT(ret == 0); + // get current snapshot info pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; - pMsg->seq = pSender->seq; + pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN + // send SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); syncSnapshotSendDestroy(pMsg); } +// when entry in snapshot, start sender void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (!(pSender->start)) { + // start snapshotSenderDoStart(pSender); pSender->start = true; } else { + // already start ASSERT(pSender->pSyncNode->pRaftStore->currentTerm >= pSender->term); - // leader change + // if current term is higher, need start again if (pSender->pSyncNode->pRaftStore->currentTerm > pSender->term) { // force peer rollback SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); @@ -98,30 +111,40 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { // close reader int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); ASSERT(ret == 0); + pSender->pReader = NULL; // start again snapshotSenderDoStart(pSender); } else { - // do nothing + // current term, do nothing + ASSERT(pSender->pSyncNode->pRaftStore->currentTerm == pSender->term); } } } void snapshotSenderStop(SSyncSnapshotSender *pSender) { - int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); - ASSERT(ret == 0); + if (pSender->pReader != NULL) { + int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); + ASSERT(ret == 0); + pSender->pReader = NULL; + } if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } + + pSender->start = false; } -// send msg from seq, seq is already updated +// when sender receiver ack, call this function to send msg from seq +// seq = ack + 1, already updated int32_t snapshotSend(SSyncSnapshotSender *pSender) { // free memory last time (seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); + pSender->pCurrentBlock = NULL; pSender->blockLen = 0; } @@ -129,6 +152,12 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader, &(pSender->pCurrentBlock), &(pSender->blockLen)); ASSERT(ret == 0); + if (pSender->blockLen > 0) { + // has read data + } else { + // read finish + pSender->seq = SYNC_SNAPSHOT_SEQ_END; + } SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; @@ -347,8 +376,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; } -// sender do something +// sender receives ack, set seq = ack + 1, send msg from seq +// if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + // get sender SSyncSnapshotSender *pSender = NULL; for (int i = 0; i < pSyncNode->replicaNum; ++i) { if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { @@ -360,10 +391,18 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + // receiver ack is finish, close sender + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { + snapshotSenderStop(pSender); + return 0; + } + + // send next msg if (pMsg->ack == pSender->seq) { + // update sender ack pSender->ack = pMsg->ack; - snapshotSend(pSender); (pSender->seq)++; + snapshotSend(pSender); } } } From 060f8224f4ef18088ff079bc7617f31edb9f3362 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 13:32:52 +0800 Subject: [PATCH 021/132] refactor: snapshotReSend --- source/libs/sync/inc/syncSnapshot.h | 3 +- source/libs/sync/src/syncSnapshot.c | 159 ++++++++++++------ .../sync/test/syncSnapshotReceiverTest.cpp | 3 - 3 files changed, 104 insertions(+), 61 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 281e30f44e..cf69b385ec 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -54,6 +54,7 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); +int32_t snapshotReSend(SSyncSnapshotSender *pSender); cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); char *snapshotSender2Str(SSyncSnapshotSender *pSender); @@ -62,8 +63,6 @@ typedef struct SSyncSnapshotReceiver { int32_t ack; void *pWriter; - void *pCurrentBlock; - int32_t blockLen; SyncTerm term; SSyncNode *pSyncNode; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index b9cd5422c9..7825a7cd95 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -18,6 +18,7 @@ #include "syncUtil.h" static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && @@ -176,6 +177,26 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { return 0; } +// send snapshot data from cache +int32_t snapshotReSend(SSyncSnapshotSender *pSender) { + if (pSender->pCurrentBlock != NULL) { + SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->seq = pSender->seq; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); + syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + syncSnapshotSendDestroy(pMsg); + } + return 0; +} + cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char u64buf[128]; cJSON *pRoot = cJSON_CreateObject(); @@ -232,22 +253,24 @@ char *snapshotSender2Str(SSyncSnapshotSender *pSender) { // ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { - ASSERT(pSyncNode->pFsm->FpSnapshotStartWrite != NULL); - ASSERT(pSyncNode->pFsm->FpSnapshotStopWrite != NULL); - ASSERT(pSyncNode->pFsm->FpSnapshotDoWrite != NULL); + bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && + (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); - SSyncSnapshotReceiver *pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); - ASSERT(pReceiver != NULL); - memset(pReceiver, 0, sizeof(*pReceiver)); + SSyncSnapshotReceiver *pReceiver; + if (condition) { + pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); + ASSERT(pReceiver != NULL); + memset(pReceiver, 0, sizeof(*pReceiver)); - pReceiver->start = false; - pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; - pReceiver->pWriter = NULL; - pReceiver->pCurrentBlock = NULL; - pReceiver->blockLen = 0; - pReceiver->pSyncNode = pSyncNode; - pReceiver->replicaIndex = replicaIndex; - pReceiver->term = pSyncNode->pRaftStore->currentTerm; + pReceiver->start = false; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + pReceiver->pWriter = NULL; + pReceiver->pSyncNode = pSyncNode; + pReceiver->replicaIndex = replicaIndex; + pReceiver->term = pSyncNode->pRaftStore->currentTerm; + } else { + sInfo("snapshotReceiverCreate cannot create receiver"); + } return pReceiver; } @@ -258,30 +281,48 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { } } +// begin receive snapshot msg (current term, seq begin) +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver) { + pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; + + ASSERT(pReceiver->pWriter == NULL); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); + ASSERT(ret == 0); +} + +// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver +// if already start, force close, start again void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { if (!(pReceiver->start)) { - pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; - int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); - ASSERT(ret == 0); - if (pReceiver->pCurrentBlock != NULL) { - taosMemoryFree(pReceiver->pCurrentBlock); - pReceiver->pCurrentBlock = NULL; - pReceiver->blockLen = 0; - } - pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + // start + snapshotReceiverDoStart(pReceiver); + } else { + // already start + + // force close, abandon incomplete data + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; + + // start again + snapshotReceiverDoStart(pReceiver); + ASSERT(0); } } void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { - int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); - ASSERT(ret == 0); - - if (pReceiver->pCurrentBlock != NULL) { - taosMemoryFree(pReceiver->pCurrentBlock); - pReceiver->blockLen = 0; + if (pReceiver->pWriter != NULL) { + int32_t ret = + pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + ASSERT(ret == 0); + pReceiver->pWriter = NULL; } + + pReceiver->start = false; } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { @@ -295,20 +336,6 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); cJSON_AddStringToObject(pRoot, "pWriter", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pCurrentBlock); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); - cJSON_AddNumberToObject(pRoot, "blockLen", pReceiver->blockLen); - - if (pReceiver->pCurrentBlock != NULL) { - char *s; - s = syncUtilprintBin((char *)(pReceiver->pCurrentBlock), pReceiver->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); - taosMemoryFree(s); - s = syncUtilprintBin2((char *)(pReceiver->pCurrentBlock), pReceiver->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); - taosMemoryFree(s); - } - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex); @@ -330,6 +357,7 @@ char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { // receiver do something int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { + // get receiver SSyncSnapshotReceiver *pReceiver = NULL; for (int i = 0; i < pSyncNode->replicaNum; ++i) { if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { @@ -338,40 +366,55 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } ASSERT(pReceiver != NULL); + bool needRsp = false; + // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { - SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pMsg->seq; - if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { // begin snapshotReceiverStart(pReceiver); + pReceiver->ack = pMsg->seq; + needRsp = true; } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { - // end + // end, finish FSM pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); snapshotReceiverStop(pReceiver); + pReceiver->ack = pMsg->seq; + needRsp = true; } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); snapshotReceiverStop(pReceiver); + needRsp = false; - } else { + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering if (pMsg->seq == pReceiver->ack + 1) { pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + pReceiver->ack = pMsg->seq; } + needRsp = true; + + } else { + ASSERT(0); } - SRpcMsg rpcMsg; - syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); - syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + if (needRsp) { + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pReceiver->ack; + + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + syncSnapshotRspDestroy(pRspMsg); + } } return 0; } @@ -403,6 +446,10 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { pSender->ack = pMsg->ack; (pSender->seq)++; snapshotSend(pSender); + } else if (pMsg->ack == pSender->seq - 1) { + snapshotReSend(pSender); + } else { + ASSERT(0); } } } diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index e912ecc597..a9e87ba12d 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -45,9 +45,6 @@ SSyncSnapshotReceiver* createReceiver() { pReceiver->start = true; pReceiver->ack = 20; pReceiver->pWriter = (void*)0x11; - pReceiver->blockLen = 20; - pReceiver->pCurrentBlock = taosMemoryMalloc(pReceiver->blockLen); - snprintf((char*)(pReceiver->pCurrentBlock), pReceiver->blockLen, "%s", "hello"); pReceiver->term = 66; return pReceiver; From 5f0d23426dac8b99d74aeda91997811c840b8fcd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 13:35:23 +0800 Subject: [PATCH 022/132] refactor: snapshotReSend --- source/libs/sync/src/syncSnapshot.c | 82 +++++++++++++++-------------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 7825a7cd95..38d3dc4995 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -370,50 +370,52 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { - if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - // begin - snapshotReceiverStart(pReceiver); - pReceiver->ack = pMsg->seq; - needRsp = true; - - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { - // end, finish FSM - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); - snapshotReceiverStop(pReceiver); - pReceiver->ack = pMsg->seq; - needRsp = true; - - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); - snapshotReceiverStop(pReceiver); - needRsp = false; - - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { - // transfering - if (pMsg->seq == pReceiver->ack + 1) { - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { + // begin + snapshotReceiverStart(pReceiver); pReceiver->ack = pMsg->seq; + needRsp = true; + + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { + // end, finish FSM + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + snapshotReceiverStop(pReceiver); + pReceiver->ack = pMsg->seq; + needRsp = true; + + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); + snapshotReceiverStop(pReceiver); + needRsp = false; + + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { + // transfering + if (pMsg->seq == pReceiver->ack + 1) { + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + pReceiver->ack = pMsg->seq; + } + needRsp = true; + + } else { + ASSERT(0); } - needRsp = true; - } else { - ASSERT(0); - } + if (needRsp) { + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pReceiver->ack; - if (needRsp) { - SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pReceiver->ack; - - SRpcMsg rpcMsg; - syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); - syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); - syncSnapshotRspDestroy(pRspMsg); + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + syncSnapshotRspDestroy(pRspMsg); + } } } return 0; From 56253aedc9994ff729ce95303cbf22f3b23c53a6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 13:40:43 +0800 Subject: [PATCH 023/132] refactor: snapshotReSend --- source/libs/sync/src/syncSnapshot.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 38d3dc4995..9b099767c7 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -116,6 +116,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { // start again snapshotSenderDoStart(pSender); + pSender->start = true; } else { // current term, do nothing ASSERT(pSender->pSyncNode->pRaftStore->currentTerm == pSender->term); @@ -297,6 +298,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { if (!(pReceiver->start)) { // start snapshotReceiverDoStart(pReceiver); + pReceiver->start = true; } else { // already start @@ -309,6 +311,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { // start again snapshotReceiverDoStart(pReceiver); + pReceiver->start = true; ASSERT(0); } From 2e840969c86d2b9920fc2932ea1532ff9dbcad0e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 13:53:21 +0800 Subject: [PATCH 024/132] refactor: snapshotReSend --- source/libs/sync/src/syncMain.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9ccd98f557..91825221e4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1253,10 +1253,11 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - bool b = index <= snapshot.lastApplyIndex; + bool b = (index <= snapshot.lastApplyIndex); return b; } +// get last index and term along with snapshot int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { SyncIndex logLastIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); SSnapshot snapshot; @@ -1266,13 +1267,17 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy if (logLastIndex > snapshotLastIndex) { *pLastIndex = logLastIndex; *pLastTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); - } else { + } else if (logLastIndex == snapshotLastIndex) { *pLastIndex = snapshotLastIndex; *pLastTerm = snapshot.lastApplyTerm; + } else { + ASSERT(0); } + return 0; } +// get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { ASSERT(index >= SYNC_INDEX_BEGIN); int ret = 0; @@ -1282,6 +1287,7 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); ASSERT(preIndex == snapshot.lastApplyIndex); + *pPreIndex = snapshot.lastApplyIndex; *pPreTerm = snapshot.lastApplyTerm; } else { From 295caad7c10260c8fc1999a8df142fa7bd892449 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 14:12:11 +0800 Subject: [PATCH 025/132] refactor: add snapshot send log --- source/libs/sync/src/syncSnapshot.c | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 9b099767c7..6a24071824 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -80,6 +80,11 @@ static void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot send begin seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + taosMemoryFree(msgStr); + syncSnapshotSendDestroy(pMsg); } @@ -107,6 +112,11 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot send force close seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + taosMemoryFree(msgStr); + syncSnapshotSendDestroy(pMsg); // close reader @@ -122,6 +132,10 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { ASSERT(pSender->pSyncNode->pRaftStore->currentTerm == pSender->term); } } + + char *s = snapshotSender2Str(pSender); + sInfo("snapshotSenderStart %s", s); + taosMemoryFree(s); } void snapshotSenderStop(SSyncSnapshotSender *pSender) { @@ -138,6 +152,10 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender) { } pSender->start = false; + + char *s = snapshotSender2Str(pSender); + sInfo("snapshotSenderStop %s", s); + taosMemoryFree(s); } // when sender receiver ack, call this function to send msg from seq @@ -173,6 +191,15 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { + sTrace("snapshot send finish seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + } else { + sTrace("snapshot send sending seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + } + taosMemoryFree(msgStr); + syncSnapshotSendDestroy(pMsg); return 0; @@ -193,6 +220,11 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); + + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot send resend seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + taosMemoryFree(msgStr); + syncSnapshotSendDestroy(pMsg); } return 0; @@ -417,6 +449,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + syncSnapshotRspDestroy(pRspMsg); } } From c706c6d0ced03a9f3f92fee55a3b6cd5c2454b9b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 14:35:20 +0800 Subject: [PATCH 026/132] add snapshot recv log --- source/libs/sync/src/syncSnapshot.c | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 6a24071824..f5743ff6a9 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -82,7 +82,7 @@ static void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot send begin seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("snapshot send begin seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -114,7 +114,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot send force close seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("snapshot send force close seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -194,9 +194,9 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { char *msgStr = syncSnapshotSend2Str(pMsg); if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sTrace("snapshot send finish seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("snapshot send finish seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); } else { - sTrace("snapshot send sending seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("snapshot send sending seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); } taosMemoryFree(msgStr); @@ -222,7 +222,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot send resend seq:%d ack:%d msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("snapshot send resend seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -347,6 +347,10 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { ASSERT(0); } + + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStart %s", s); + taosMemoryFree(s); } void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { @@ -358,6 +362,10 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; + + char *s = snapshotReceiver2Str(pReceiver); + sInfo("snapshotReceiverStop %s", s); + taosMemoryFree(s); } cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { @@ -412,6 +420,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->ack = pMsg->seq; needRsp = true; + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot recv begin ack:%d recv msg:%s", pReceiver->ack, msgStr); + taosMemoryFree(msgStr); + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // end, finish FSM pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); @@ -420,11 +432,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->ack = pMsg->seq; needRsp = true; + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot recv end ack:%d recv msg:%s", pReceiver->ack, msgStr); + taosMemoryFree(msgStr); + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); snapshotReceiverStop(pReceiver); needRsp = false; + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot recv force close ack:%d recv msg:%s", pReceiver->ack, msgStr); + taosMemoryFree(msgStr); + } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering if (pMsg->seq == pReceiver->ack + 1) { @@ -433,6 +453,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } needRsp = true; + char *msgStr = syncSnapshotSend2Str(pMsg); + sTrace("snapshot recv receiving ack:%d recv msg:%s", pReceiver->ack, msgStr); + taosMemoryFree(msgStr); + } else { ASSERT(0); } From 4453ebab3abb02f7413399ca6101365488fb06ea Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 14:40:20 +0800 Subject: [PATCH 027/132] add snapshot recv log --- source/libs/sync/src/syncSnapshot.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index f5743ff6a9..abeb1722fd 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -477,7 +477,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { syncSnapshotRspDestroy(pRspMsg); } } + } else { + syncNodeLog2("syncNodeOnSnapshotSendCb not follower", pSyncNode); } + return 0; } @@ -514,6 +517,8 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { ASSERT(0); } } + } else { + syncNodeLog2("syncNodeOnSnapshotRspCb not leader", pSyncNode); } return 0; From 13b336d56929f32250d425a21d52a29f494255e5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 15:03:01 +0800 Subject: [PATCH 028/132] add snapshot test --- include/libs/sync/syncTools.h | 2 + source/libs/sync/inc/syncIO.h | 3 + source/libs/sync/inc/syncInt.h | 13 +- source/libs/sync/inc/syncSnapshot.h | 14 +- source/libs/sync/src/syncMain.c | 3 + source/libs/sync/src/syncSnapshot.c | 4 +- source/libs/sync/test/CMakeLists.txt | 14 + .../test/syncConfigChangeSnapshotTest.cpp | 283 ++++++++++++++++++ 8 files changed, 316 insertions(+), 20 deletions(-) create mode 100644 source/libs/sync/test/syncConfigChangeSnapshotTest.cpp diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index f8d5768b5d..21ce75c28e 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -478,6 +478,8 @@ typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); typedef int32_t (*FpOnAppendEntriesReplyCb)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); typedef int32_t (*FpOnTimeoutCb)(SSyncNode* pSyncNode, SyncTimeout* pMsg); +typedef int32_t (*FpOnSnapshotSendCb)(SSyncNode* ths, SyncSnapshotSend* pMsg); +typedef int32_t (*FpOnSnapshotRspCb)(SSyncNode* ths, SyncSnapshotRsp* pMsg); // option ---------------------------------- bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index b69c087b5f..88d3065375 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -57,6 +57,9 @@ typedef struct SSyncIO { int32_t (*FpOnSyncAppendEntriesReply)(SSyncNode *pSyncNode, SyncAppendEntriesReply *pMsg); int32_t (*FpOnSyncTimeout)(SSyncNode *pSyncNode, SyncTimeout *pMsg); + int32_t (*FpOnSyncSnapshotSend)(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg); + int32_t (*FpOnSyncSnapshotRsp)(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg); + int8_t isStart; } SSyncIO; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index fbe31ef031..22aa7b1224 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -145,17 +145,8 @@ typedef struct SSyncNode { FpOnRequestVoteReplyCb FpOnRequestVoteReply; FpOnAppendEntriesCb FpOnAppendEntries; FpOnAppendEntriesReplyCb FpOnAppendEntriesReply; - - /* - int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg); - int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg); - int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg); - int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg); - int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg); - int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg); - int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg); - int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg); - */ + FpOnSnapshotSendCb FpOnSnapshotSend; + FpOnSnapshotRspCb FpOnSnapshotRsp; // tools SSyncRespMgr* pSyncRespMgr; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index cf69b385ec..ccdf099482 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -55,14 +55,14 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; + void * pWriter; SyncTerm term; SSyncNode *pSyncNode; @@ -73,8 +73,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 91825221e4..a41c1a57c3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -558,6 +558,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + pSyncNode->FpOnSnapshotSend = syncNodeOnSnapshotSendCb; + pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb; + if (pSyncNode->pRaftCfg->snapshotEnable) { pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index abeb1722fd..7521416379 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -279,7 +279,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -393,7 +393,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index d1e747f29d..530d39a55f 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -38,6 +38,7 @@ add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") add_executable(syncConfigChangeTest "") +add_executable(syncConfigChangeSnapshotTest "") add_executable(syncSnapshotSendTest "") add_executable(syncSnapshotRspTest "") add_executable(syncSnapshotSenderTest "") @@ -204,6 +205,10 @@ target_sources(syncConfigChangeTest PRIVATE "syncConfigChangeTest.cpp" ) +target_sources(syncConfigChangeSnapshotTest + PRIVATE + "syncConfigChangeSnapshotTest.cpp" +) target_sources(syncSnapshotSendTest PRIVATE "syncSnapshotSendTest.cpp" @@ -422,6 +427,11 @@ target_include_directories(syncConfigChangeTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncConfigChangeSnapshotTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncSnapshotSendTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -604,6 +614,10 @@ target_link_libraries(syncConfigChangeTest sync gtest_main ) +target_link_libraries(syncConfigChangeSnapshotTest + sync + gtest_main +) target_link_libraries(syncSnapshotSendTest sync gtest_main diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp new file mode 100644 index 0000000000..02fc138e4c --- /dev/null +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -0,0 +1,283 @@ +#include +#include +#include "os.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncReplicateTest"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); + + sprintf(tsTempDir, "%s", "."); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = 100; + return 0; +} + +void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } + +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { + sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu", + cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + memset(pFsm, 0, sizeof(*pFsm)); + + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + + pFsm->FpGetSnapshot = GetSnapshotCb; + pFsm->FpRestoreFinishCb = RestoreFinishCb; + + pFsm->FpReConfigCb = ReConfigCb; + + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.msgcb = &gSyncIO->msgcb; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + syncInfo.isStandBy = isStandBy; + syncInfo.snapshotEnable = true; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + + gSyncIO->FpOnSyncSnapshotSend = pSyncNode->FpOnSnapshotSend; + gSyncIO->FpOnSyncSnapshotRsp = pSyncNode->FpOnSnapshotRsp; + + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe); +} + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_INFO; + if (argc != 7) { + usage(argv[0]); + exit(-1); + } + + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + int32_t lastApplyIndex = atoi(argv[3]); + int32_t writeRecordNum = atoi(argv[4]); + bool isStandBy = atoi(argv[5]); + bool isConfigChange = atoi(argv[6]); + gSnapshotLastApplyIndex = lastApplyIndex; + + if (!isStandBy) { + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + } + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); + assert(rid > 0); + + syncStart(rid); + + /* + if (isStandBy) { + syncStartStandBy(rid); + } else { + syncStart(rid); + } + */ + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + if (isConfigChange) { + configChange(rid, 2, myIndex); + } + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* s = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", s, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", s, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", s); + } + + taosMsleep(1000); + taosMemoryFree(s); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + return 0; +} From 320f21820ba6b0191d1a1c4331ae1e0918f81954 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 15:25:06 +0800 Subject: [PATCH 029/132] add snapshot test --- .../test/syncConfigChangeSnapshotTest.cpp | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 02fc138e4c..c7fcd9f0d4 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -75,6 +75,66 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { + *ppReader = (void*)0xABCD; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartRead== pFsm:%p, *ppReader:%p", pFsm, *ppReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopRead== pFsm:%p, pReader:%p", pFsm, pReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { + static int readIter = 0; + + if (readIter == 5) { + *len = 20; + *ppBuf = taosMemoryMalloc(*len); + snprintf((char*)*ppBuf, *len, "data iter:%d", readIter); + } else { + *len = 0; + *ppBuf = NULL; + } + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d *ppBuf:%s", pFsm, + pReader, *len, (char*)(*ppBuf)); + sTrace("%s", logBuf); + + readIter++; + return 0; +} + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { + *ppWriter = (void*)0xCDEF; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, + isApply); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoWrite== pFsm:%p, pWriter:%p, len:%d pBuf:%s", pFsm, + pWriter, len, (char*)pBuf); + sTrace("%s", logBuf); + return 0; +} + void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { @@ -92,6 +152,12 @@ SSyncFSM* createFsm() { pFsm->FpGetSnapshot = GetSnapshotCb; pFsm->FpRestoreFinishCb = RestoreFinishCb; + pFsm->FpSnapshotStartRead = SnapshotStartRead; + pFsm->FpSnapshotStopRead = SnapshotStopRead; + pFsm->FpSnapshotDoRead = SnapshotDoRead; + pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pFsm->FpSnapshotDoWrite = SnapshotDoWrite; pFsm->FpReConfigCb = ReConfigCb; From 403f6085d57b4544f36ad57e37af7ecfbdfd5811 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 15:52:09 +0800 Subject: [PATCH 030/132] refactor: review syncNodeRequestVotePeersSnapshot --- source/libs/sync/src/syncMain.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a41c1a57c3..f4696cd019 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1273,16 +1273,20 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy } else if (logLastIndex == snapshotLastIndex) { *pLastIndex = snapshotLastIndex; *pLastTerm = snapshot.lastApplyTerm; + } else if (logLastIndex < snapshotLastIndex) { + // maybe wal is deleted + *pLastIndex = snapshotLastIndex; + *pLastTerm = snapshot.lastApplyTerm; } else { ASSERT(0); } - return 0; } // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { ASSERT(index >= SYNC_INDEX_BEGIN); + ASSERT(!syncNodeIsIndexInSnapshot(pSyncNode, index)); int ret = 0; SyncIndex preIndex = index - 1; From 78db753a6441b8cad19f39609d3337f14d8dffcf Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 16:30:04 +0800 Subject: [PATCH 031/132] refactor: review syncNodeAppendEntriesPeersSnapshot --- source/libs/sync/src/syncMain.c | 20 ++++++++++++++++---- source/libs/sync/src/syncReplication.c | 8 ++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f4696cd019..219aecd22d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1169,7 +1169,15 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) { // maybe overwrite myself, no harm // just do it! - pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + + // pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + + // maybe wal is deleted + SyncIndex lastIndex; + SyncTerm lastTerm; + int32_t code = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); + ASSERT(code == 0); + pSyncNode->pNextIndex->index[i] = lastIndex + 1; } for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { @@ -1299,9 +1307,13 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex *pPreTerm = snapshot.lastApplyTerm; } else { SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preIndex); - ASSERT(pPreEntry != NULL); - *pPreIndex = pPreEntry->index; - *pPreTerm = pPreEntry->term; + if (pPreEntry != NULL) { + *pPreIndex = pPreEntry->index; + *pPreTerm = pPreEntry->term; + } else { + *pPreIndex = SYNC_INDEX_INVALID; + *pPreTerm = 0; + } } return 0; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 864acb2993..7f2968e2d7 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -126,7 +126,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); - // set prevLogIndex SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); SyncIndex preLogIndex; @@ -139,6 +138,12 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); if (syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { + // will send this msg until snapshot receive finish! + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + sInfo("nextIndex:%ld in snapshot: , begin snapshot", nextIndex, + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + // to claim leader SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); assert(pMsg != NULL); @@ -163,7 +168,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { } } ASSERT(pSender != NULL); - snapshotSenderStart(pSender); } else { From 80c4e77b0bc3986910417575747bf3a8c7c55bce Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 17:07:14 +0800 Subject: [PATCH 032/132] refactor: modify syncNodeAppendEntriesPeersSnapshot --- source/libs/sync/src/syncReplication.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7f2968e2d7..bb7036788e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -144,6 +144,14 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { sInfo("nextIndex:%ld in snapshot: , begin snapshot", nextIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + SyncIndex lastIndex; + SyncTerm lastTerm; + ret = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); + ASSERT(ret == 0); + + ret = syncNodeGetPreIndexTerm(pSyncNode, lastIndex + 1, &preLogIndex, &preLogTerm); + ASSERT(ret == 0); + // to claim leader SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); assert(pMsg != NULL); From df4787eff1fc04406e74e3b1d34cd710af8ecc2c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 17:08:03 +0800 Subject: [PATCH 033/132] refactor: modify syncNodeAppendEntriesPeersSnapshot --- source/libs/sync/src/syncReplication.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index bb7036788e..0746f0ed6e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -144,6 +144,8 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { sInfo("nextIndex:%ld in snapshot: , begin snapshot", nextIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + // do not use next index + // always send from new last index + 1 SyncIndex lastIndex; SyncTerm lastTerm; ret = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); From ae06cc36961349b992a524982db9ff3bf4249aef Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 19:11:31 +0800 Subject: [PATCH 034/132] refactor: after review raft 6 cb --- source/libs/sync/src/syncAppendEntries.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 5f5e464984..db38a46e31 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -461,9 +461,23 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ret = syncNodeGetLastIndexTerm(ths, &localLastIndex, &localLastTerm); ASSERT(ret == 0); - bool logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || - ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && (pMsg->prevLogIndex <= localLastIndex) && - (pMsg->prevLogTerm == localPreLogTerm)); + bool logOK; + if (syncNodeIsIndexInSnapshot(ths, pMsg->prevLogIndex)) { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + + // maybe this assert is error, because replica take takesnapshot separately + // leader will reset next index to newest + ASSERT(pMsg->prevLogIndex == snapshot.lastApplyIndex); + + logOK = (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->prevLogTerm == snapshot.lastApplyTerm); + + } else { + logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && + (pMsg->prevLogTerm == localPreLogTerm)); + } // reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || From 757a1248dc0e6e3b8d1ca6195d85d4aba8a42fb1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 21:23:39 +0800 Subject: [PATCH 035/132] fix: send snapshot --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncIO.c | 17 ++++++++++ source/libs/sync/src/syncMain.c | 10 +++++- source/libs/sync/src/syncMessage.c | 10 ++++++ source/libs/sync/src/syncReplication.c | 8 ++--- source/libs/sync/src/syncSnapshot.c | 8 +++-- .../test/syncConfigChangeSnapshotTest.cpp | 31 +++++++++++++------ 7 files changed, 68 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 22aa7b1224..3f5c081f4a 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -157,6 +157,7 @@ typedef struct SSyncNode { // SSnapshot* pSnapshot; SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; SSyncSnapshotReceiver* receivers[TSDB_MAX_REPLICA]; + SSyncSnapshotReceiver* pNewNodeReceiver; } SSyncNode; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index e30a39e634..61d801dcbf 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -319,6 +319,23 @@ static void *syncIOConsumerFunc(void *param) { io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_SEND) { + if (io->FpOnSyncSnapshotSend != NULL) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncSnapshotSend(io->pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_RSP) { + if (io->FpOnSyncSnapshotRsp != NULL) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + io->FpOnSyncSnapshotRsp(io->pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } + } else { sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 219aecd22d..7878bfe7ee 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -601,6 +601,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { (pSyncNode->receivers)[i] = pReceiver; } + pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, 100); + // start in syncNodeStart // start raft // syncNodeBecomeFollower(pSyncNode); @@ -611,6 +613,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { + raftStoreNextTerm(pSyncNode->pRaftStore); syncNodeBecomeLeader(pSyncNode); syncNodeLog2("==state change become leader immediately==", pSyncNode); @@ -706,6 +709,11 @@ void syncNodeClose(SSyncNode* pSyncNode) { } } + if (pSyncNode->pNewNodeReceiver != NULL) { + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); + pSyncNode->pNewNodeReceiver = NULL; + } + /* if (pSyncNode->pSnapshot != NULL) { taosMemoryFree(pSyncNode->pSnapshot); @@ -1294,7 +1302,7 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { ASSERT(index >= SYNC_INDEX_BEGIN); - ASSERT(!syncNodeIsIndexInSnapshot(pSyncNode, index)); + // ASSERT(!syncNodeIsIndexInSnapshot(pSyncNode, index)); int ret = 0; SyncIndex preIndex = index - 1; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 7314744e3b..e8fa146271 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -65,6 +65,16 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotSend2Json(pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotRsp2Json(pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) { pRoot = cJSON_CreateObject(); char* s; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 0746f0ed6e..b645fa9ac8 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -61,6 +61,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { // set prevLogIndex SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + SyncIndex preLogIndex = nextIndex - 1; // set preLogTerm @@ -127,13 +128,9 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { SRaftId* pDestId = &(pSyncNode->peersId[i]); SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); - SyncIndex preLogIndex; SyncTerm preLogTerm; - ret = syncNodeGetPreIndexTerm(pSyncNode, nextIndex, &preLogIndex, &preLogTerm); - ASSERT(ret == 0); - // batch optimized // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); @@ -181,6 +178,9 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { snapshotSenderStart(pSender); } else { + ret = syncNodeGetPreIndexTerm(pSyncNode, nextIndex, &preLogIndex, &preLogTerm); + ASSERT(ret == 0); + SyncAppendEntries* pMsg = NULL; SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, nextIndex); if (pEntry != NULL) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 7521416379..cfd5e2f5da 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -407,7 +407,11 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver = (pSyncNode->receivers)[i]; } } - ASSERT(pReceiver != NULL); + + // add new replica + if (pReceiver == NULL) { + pReceiver = pSyncNode->pNewNodeReceiver; + } bool needRsp = false; @@ -430,7 +434,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); snapshotReceiverStop(pReceiver); pReceiver->ack = pMsg->seq; - needRsp = true; + needRsp = false; char *msgStr = syncSnapshotSend2Str(pMsg); sTrace("snapshot recv end ack:%d recv msg:%s", pReceiver->ack, msgStr); diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index c7fcd9f0d4..8f36f19238 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -20,6 +20,7 @@ uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; const char* gDir = "./syncReplicateTest"; int32_t gVgId = 1234; SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; void init() { int code = walInit(); @@ -44,8 +45,9 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { if (cbMeta.index > beginIndex) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); @@ -71,7 +73,7 @@ void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { pSnapshot->data = NULL; pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; - pSnapshot->lastApplyTerm = 100; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; return 0; } @@ -94,17 +96,18 @@ int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32 static int readIter = 0; if (readIter == 5) { + *len = 0; + *ppBuf = NULL; + } else if (readIter < 5) { *len = 20; *ppBuf = taosMemoryMalloc(*len); snprintf((char*)*ppBuf, *len, "data iter:%d", readIter); - } else { - *len = 0; - *ppBuf = NULL; } char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d *ppBuf:%s", pFsm, - pReader, *len, (char*)(*ppBuf)); + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d, *ppBuf:%s, readIter:%d", pFsm, pReader, *len, + (char*)(*ppBuf), readIter); sTrace("%s", logBuf); readIter++; @@ -249,7 +252,7 @@ void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) { } void usage(char* exe) { - printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe); + printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange lastApplyTerm \n", exe); } SRpcMsg* createRpcMsg(int i, int count, int myIndex) { @@ -265,7 +268,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_INFO; - if (argc != 7) { + if (argc != 8) { usage(argv[0]); exit(-1); } @@ -276,7 +279,15 @@ int main(int argc, char** argv) { int32_t writeRecordNum = atoi(argv[4]); bool isStandBy = atoi(argv[5]); bool isConfigChange = atoi(argv[6]); + int32_t lastApplyTerm = atoi(argv[7]); + + sTrace( + "args: replicaNum:%d, myIndex:%d, lastApplyIndex:%d, writeRecordNum:%d, isStandBy:%d, isConfigChange:%d, " + "lastApplyTerm:%d", + replicaNum, myIndex, lastApplyIndex, writeRecordNum, isStandBy, isConfigChange, lastApplyTerm); + gSnapshotLastApplyIndex = lastApplyIndex; + gSnapshotLastApplyTerm = lastApplyTerm; if (!isStandBy) { assert(replicaNum >= 1 && replicaNum <= 5); From c3bf3cc97c39def76b075bee0e05b36166561125 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 1 Jun 2022 21:57:30 +0800 Subject: [PATCH 036/132] fix: send snapshot --- source/libs/sync/src/syncMain.c | 13 ++++++++----- source/libs/sync/src/syncSnapshot.c | 5 +++-- .../libs/sync/test/syncConfigChangeSnapshotTest.cpp | 6 ++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7878bfe7ee..ab696384a0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -562,15 +562,18 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb; if (pSyncNode->pRaftCfg->snapshotEnable) { - pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; - pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; - pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; - pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; - } else { + sInfo("sync node use snapshot"); pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb; pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb; pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb; pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb; + + } else { + sInfo("sync node do not use snapshot"); + pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb; + pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb; + pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb; + pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; } // tools diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index cfd5e2f5da..e06f1a9ec4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -279,7 +279,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -393,7 +393,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -432,6 +432,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // end, finish FSM pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + pReceiver->pWriter = NULL; snapshotReceiverStop(pReceiver); pReceiver->ack = pMsg->seq; needRsp = false; diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 8f36f19238..ba7199f9b0 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -127,6 +127,12 @@ int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, isApply); sTrace("%s", logBuf); + + if (isApply) { + gSnapshotLastApplyIndex = 7; + gSnapshotLastApplyTerm = 1; + } + return 0; } From 48a331fd2c88c07a53f129cf07e734c007698dbf Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 2 Jun 2022 00:00:27 +0800 Subject: [PATCH 037/132] refactor: print msg type --- source/libs/sync/src/syncIO.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 61d801dcbf..5915380c1a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -75,7 +75,7 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { syncUtilMsgNtoH(pMsg->pCont); char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port); + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->msgType); syncRpcMsgLog2(logBuf, pMsg); syncUtilMsgHtoN(pMsg->pCont); @@ -253,7 +253,9 @@ static void *syncIOConsumerFunc(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "==syncIOConsumerFunc== msgType:%d", pRpcMsg->msgType); + syncRpcMsgLog2(logBuf, pRpcMsg); // use switch case instead of if else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { From d04c12713ac5577f9bf8f26da3d2f3bef968d1b2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 2 Jun 2022 11:36:26 +0800 Subject: [PATCH 038/132] fix: send snapshot --- source/libs/sync/inc/syncSnapshot.h | 15 ++++++----- source/libs/sync/src/syncAppendEntries.c | 6 +++++ source/libs/sync/src/syncIO.c | 3 ++- source/libs/sync/src/syncReplication.c | 27 +++++++++++++------ source/libs/sync/src/syncSnapshot.c | 5 +++- .../test/syncConfigChangeSnapshotTest.cpp | 2 +- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index ccdf099482..f34c5fffb0 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,14 +39,15 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; SSyncNode *pSyncNode; int32_t replicaIndex; SyncTerm term; + bool finish; } SSyncSnapshotSender; SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); @@ -55,14 +56,14 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void * pWriter; + void *pWriter; SyncTerm term; SSyncNode *pSyncNode; @@ -73,8 +74,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index db38a46e31..4ff3b95890 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -471,12 +471,18 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ASSERT(pMsg->prevLogIndex == snapshot.lastApplyIndex); logOK = (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->prevLogTerm == snapshot.lastApplyTerm); + sTrace( + "1 - logOK:%d, pMsg->prevLogIndex:%ld, snapshot.lastApplyIndex:%ld, pMsg->prevLogTerm:%lu, " + "snapshot.lastApplyTerm:%lu", + logOK, pMsg->prevLogIndex, snapshot.lastApplyIndex, pMsg->prevLogTerm, snapshot.lastApplyTerm); } else { logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); + sTrace("2 - logOK:%d, pMsg->prevLogIndex:%ld, getLastIndex:%ld, pMsg->prevLogTerm:%lu, localPreLogTerm:%lu", logOK, + pMsg->prevLogIndex, ths->pLogStore->getLastIndex(ths->pLogStore), pMsg->prevLogTerm, localPreLogTerm); } // reject request diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 5915380c1a..3c69432837 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -75,7 +75,8 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { syncUtilMsgNtoH(pMsg->pCont); char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->msgType); + snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port, + pMsg->msgType); syncRpcMsgLog2(logBuf, pMsg); syncUtilMsgHtoN(pMsg->pCont); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index b645fa9ac8..83fbb0aa97 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -134,7 +134,22 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { // batch optimized // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); - if (syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { + // sending snapshot finish? + bool snapshotSendingFinish = false; + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncUtilSameId(pDestId, &((pSyncNode->replicasId)[i]))) { + pSender = (pSyncNode->senders)[i]; + } + } + ASSERT(pSender != NULL); + snapshotSendingFinish = (pSender->finish) && (pSender->term == pSyncNode->pRaftStore->currentTerm); + if (snapshotSendingFinish) { + sInfo("snapshotSendingFinish! term:%lu", pSender->term); + } + + if ((syncNodeIsIndexInSnapshot(pSyncNode, nextIndex - 1) && !snapshotSendingFinish) || + syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { // will send this msg until snapshot receive finish! SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -142,14 +157,10 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { snapshot.lastApplyIndex, snapshot.lastApplyTerm); // do not use next index - // always send from new last index + 1 - SyncIndex lastIndex; - SyncTerm lastTerm; - ret = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm); - ASSERT(ret == 0); + // always send from snapshot.lastApplyIndex + 1, and wait for snapshot transfer finish - ret = syncNodeGetPreIndexTerm(pSyncNode, lastIndex + 1, &preLogIndex, &preLogTerm); - ASSERT(ret == 0); + preLogIndex = snapshot.lastApplyIndex; + preLogTerm = snapshot.lastApplyTerm; // to claim leader SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index e06f1a9ec4..e423d4d369 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -40,6 +40,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->finish = false; } else { sInfo("snapshotSenderCreate cannot create sender"); } @@ -270,6 +271,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); } cJSON *pJson = cJSON_CreateObject(); @@ -435,7 +437,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->pWriter = NULL; snapshotReceiverStop(pReceiver); pReceiver->ack = pMsg->seq; - needRsp = false; + needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); sTrace("snapshot recv end ack:%d recv msg:%s", pReceiver->ack, msgStr); @@ -506,6 +508,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { // receiver ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { + pSender->finish = true; snapshotSenderStop(pSender); return 0; } diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index ba7199f9b0..781c168da9 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -129,7 +129,7 @@ int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { sTrace("%s", logBuf); if (isApply) { - gSnapshotLastApplyIndex = 7; + gSnapshotLastApplyIndex = 10; gSnapshotLastApplyTerm = 1; } From 1ff40068e704f2443c3ec4fe9c8be86cdb5a8aad Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 2 Jun 2022 11:57:06 +0800 Subject: [PATCH 039/132] fix: send snapshot --- source/libs/sync/src/syncReplication.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 83fbb0aa97..bdb55f3a7c 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -178,15 +178,17 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { syncNodeAppendEntries(pSyncNode, pDestId, pMsg); syncAppendEntriesDestroy(pMsg); - SSyncSnapshotSender* pSender = NULL; - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - if (syncUtilSameId(&((pSyncNode->replicasId)[i]), pDestId)) { - pSender = (pSyncNode->senders)[i]; - break; + if (!snapshotSendingFinish) { + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncUtilSameId(&((pSyncNode->replicasId)[i]), pDestId)) { + pSender = (pSyncNode->senders)[i]; + break; + } } + ASSERT(pSender != NULL); + snapshotSenderStart(pSender); } - ASSERT(pSender != NULL); - snapshotSenderStart(pSender); } else { ret = syncNodeGetPreIndexTerm(pSyncNode, nextIndex, &preLogIndex, &preLogTerm); From ac90f61b6376c3bf365b109bd311728ebed8f480 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 2 Jun 2022 13:53:19 +0800 Subject: [PATCH 040/132] fix(sync): wal write from middle --- source/libs/sync/src/syncAppendEntries.c | 6 +++++- source/libs/sync/src/syncRaftLog.c | 10 +++++++--- source/libs/sync/src/syncSnapshot.c | 5 +++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4ff3b95890..4ebe142315 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -20,6 +20,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "wal.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -687,7 +688,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // execute fsm if (ths->pFsm != NULL) { for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - if (i != SYNC_INDEX_INVALID) { + // notice! wal maybe deleted, update firstVer + SyncIndex walFirstVer = walGetFirstVer(ths->pWal); + + if (i != SYNC_INDEX_INVALID && i >= walFirstVer) { SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, i); assert(pEntry != NULL); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a6397f8cba..a15786e758 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -208,9 +208,13 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddItemToObject(pRoot, "pEntries", pEntries); SyncIndex lastIndex = logStoreLastIndex(pLogStore); for (SyncIndex i = 0; i <= lastIndex; ++i) { - SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); + SyncIndex walFirstVer = walGetFirstVer(pData->pWal); + + if (i != SYNC_INDEX_INVALID && i >= walFirstVer) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } } } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index e423d4d369..c894a34fc0 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -16,6 +16,7 @@ #include "syncSnapshot.h" #include "syncRaftStore.h" #include "syncUtil.h" +#include "wal.h" static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver); @@ -434,6 +435,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // end, finish FSM pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + + walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); + sInfo("walRestoreFromSnapshot lastIndex:%ld", pMsg->lastIndex); + pReceiver->pWriter = NULL; snapshotReceiverStop(pReceiver); pReceiver->ack = pMsg->seq; From 44994201b6f143d7a292727d705001668bf8f0e9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 2 Jun 2022 19:47:06 +0800 Subject: [PATCH 041/132] fix(sync): sending snapshot --- source/libs/sync/inc/syncRaftLog.h | 2 ++ source/libs/sync/src/syncAppendEntries.c | 34 ++++++++++++++++++------ source/libs/sync/src/syncRaftLog.c | 6 +++++ source/libs/sync/src/syncReplication.c | 7 +++-- source/libs/sync/src/syncSnapshot.c | 2 ++ 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index df5cd3f36c..b54cd14ccd 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -39,6 +39,8 @@ char* logStore2Str(SSyncLogStore* pLogStore); cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); char* logStoreSimple2Str(SSyncLogStore* pLogStore); +SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); + // SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); // SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); // SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4ebe142315..fd31bfc148 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -454,16 +454,16 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs SyncIndex localPreLogIndex; SyncTerm localPreLogTerm; - ret = syncNodeGetPreIndexTerm(ths, pMsg->prevLogIndex + 1, &localPreLogIndex, &localPreLogTerm); - ASSERT(ret == 0); - - SyncIndex localLastIndex; - SyncTerm localLastTerm; - ret = syncNodeGetLastIndexTerm(ths, &localLastIndex, &localLastTerm); - ASSERT(ret == 0); bool logOK; - if (syncNodeIsIndexInSnapshot(ths, pMsg->prevLogIndex)) { + + SyncIndex logFirstIndex = logStoreFirstIndex(ths->pLogStore); + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + if (logFirstIndex > snapshot.lastApplyIndex) { + logOK = false; + + } else if (syncNodeIsIndexInSnapshot(ths, pMsg->prevLogIndex)) { SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); @@ -478,6 +478,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs logOK, pMsg->prevLogIndex, snapshot.lastApplyIndex, pMsg->prevLogTerm, snapshot.lastApplyTerm); } else { + ret = syncNodeGetPreIndexTerm(ths, pMsg->prevLogIndex + 1, &localPreLogIndex, &localPreLogTerm); + ASSERT(ret == 0); + logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && @@ -510,6 +513,21 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } + SyncIndex localLastIndex; + SyncTerm localLastTerm; + if (logFirstIndex == SYNC_INDEX_INVALID) { + localLastIndex = ths->pLogStore->getLastIndex(ths->pLogStore); + localLastTerm = ths->pLogStore->getLastTerm(ths->pLogStore); + + } else if (logFirstIndex > snapshot.lastApplyIndex) { + localLastIndex = ths->pLogStore->getLastIndex(ths->pLogStore); + localLastTerm = ths->pLogStore->getLastTerm(ths->pLogStore); + + } else { + ret = syncNodeGetLastIndexTerm(ths, &localLastIndex, &localLastTerm); + ASSERT(ret == 0); + } + // return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { sTrace( diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a15786e758..28b7a0bbf0 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -258,6 +258,12 @@ char* logStoreSimple2Str(SSyncLogStore* pLogStore) { return serialized; } +SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return walGetFirstVer(pWal); +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index bdb55f3a7c..4b9a8c51aa 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -151,8 +151,8 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { if ((syncNodeIsIndexInSnapshot(pSyncNode, nextIndex - 1) && !snapshotSendingFinish) || syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { // will send this msg until snapshot receive finish! - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SSnapshot snapshot = pSender->snapshot; sInfo("nextIndex:%ld in snapshot: , begin snapshot", nextIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm); @@ -162,6 +162,9 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { preLogIndex = snapshot.lastApplyIndex; preLogTerm = snapshot.lastApplyTerm; + // update next index! + syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, snapshot.lastApplyIndex + 1); + // to claim leader SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); assert(pMsg != NULL); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index c894a34fc0..5e7661b477 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -41,6 +41,8 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + pSender->finish = false; } else { sInfo("snapshotSenderCreate cannot create sender"); From eea343c5a6fe686697e0d5771bca7b836a5771f1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 4 Jun 2022 11:25:03 +0800 Subject: [PATCH 042/132] enh(sync): add syncTestTool --- source/libs/sync/inc/syncSnapshot.h | 14 +- source/libs/sync/src/syncIO.c | 14 +- source/libs/sync/src/syncSnapshot.c | 4 +- source/libs/sync/test/CMakeLists.txt | 14 + source/libs/sync/test/syncTest.cpp | 7 +- source/libs/sync/test/syncTestTool.cpp | 371 +++++++++++++++++++++++++ 6 files changed, 407 insertions(+), 17 deletions(-) create mode 100644 source/libs/sync/test/syncTestTool.cpp diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index f34c5fffb0..c028e732b8 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -56,14 +56,14 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; + void * pWriter; SyncTerm term; SSyncNode *pSyncNode; @@ -74,8 +74,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 3c69432837..4e814c41f3 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -90,8 +90,10 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t syncIOEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { int32_t ret = 0; - char logBuf[128] = {0}; - syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg); + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncIOEqMsg== msgType:%d", pMsg->msgType); + syncRpcMsgLog2(logBuf, pMsg); SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); @@ -240,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -255,7 +257,7 @@ static void *syncIOConsumerFunc(void *param) { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "==syncIOConsumerFunc== msgType:%d", pRpcMsg->msgType); + snprintf(logBuf, sizeof(logBuf), "==syncIOConsumMsg== msgType:%d", pRpcMsg->msgType); syncRpcMsgLog2(logBuf, pRpcMsg); // use switch case instead of if else diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 5e7661b477..6802fc0ed8 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -284,7 +284,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -398,7 +398,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 530d39a55f..b6def04eaa 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -43,6 +43,7 @@ add_executable(syncSnapshotSendTest "") add_executable(syncSnapshotRspTest "") add_executable(syncSnapshotSenderTest "") add_executable(syncSnapshotReceiverTest "") +add_executable(syncTestTool "") target_sources(syncTest @@ -225,6 +226,10 @@ target_sources(syncSnapshotReceiverTest PRIVATE "syncSnapshotReceiverTest.cpp" ) +target_sources(syncTestTool + PRIVATE + "syncTestTool.cpp" +) target_include_directories(syncTest @@ -452,6 +457,11 @@ target_include_directories(syncSnapshotReceiverTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncTestTool + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -634,6 +644,10 @@ target_link_libraries(syncSnapshotReceiverTest sync gtest_main ) +target_link_libraries(syncTestTool + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index ffe8b81571..97de81572a 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -50,14 +50,16 @@ void test4() { } int main(int argc, char** argv) { - // taosInitLog("tmp/syncTest.log", 100); + taosInitLog("/tmp/syncTest.log", 100); tsAsyncLog = 0; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; test1(); test2(); test3(); test4(); + /* if (argc == 2) { bool bTaosDirExist = taosDirExist(argv[1]); printf("%s bTaosDirExist:%d \n", argv[1], bTaosDirExist); @@ -65,7 +67,8 @@ int main(int argc, char** argv) { bool bTaosCheckExistFile = taosCheckExistFile(argv[1]); printf("%s bTaosCheckExistFile:%d \n", argv[1], bTaosCheckExistFile); } + */ - // taosCloseLog(); + taosCloseLog(); return 0; } diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp new file mode 100644 index 0000000000..b5b884b78b --- /dev/null +++ b/source/libs/sync/test/syncTestTool.cpp @@ -0,0 +1,371 @@ +#include +#include +#include "os.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftCfg.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7000, 7001, 7002, 7003, 7004}; +const char* gDir = "./syncTestTool"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; +int gIterTimes = 0; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + +int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) { + *ppReader = (void*)0xABCD; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartRead== pFsm:%p, *ppReader:%p", pFsm, *ppReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopRead== pFsm:%p, pReader:%p", pFsm, pReader); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { + static int readIter = 0; + + if (readIter == gIterTimes) { + *len = 0; + *ppBuf = NULL; + } else if (readIter < gIterTimes) { + *len = 20; + *ppBuf = taosMemoryMalloc(*len); + snprintf((char*)*ppBuf, *len, "data iter:%d", readIter); + } + + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d, *ppBuf:[%s], readIter:%d", pFsm, pReader, + *len, (char*)(*ppBuf), readIter); + sTrace("%s", logBuf); + + readIter++; + return 0; +} + +int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { + *ppWriter = (void*)0xCDEF; + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter); + sTrace("%s", logBuf); + return 0; +} + +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, + isApply); + sTrace("%s", logBuf); + + if (isApply) { + gSnapshotLastApplyIndex = 10; + gSnapshotLastApplyTerm = 1; + } + + return 0; +} + +int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoWrite== pFsm:%p, pWriter:%p, len:%d pBuf:[%s]", pFsm, + pWriter, len, (char*)pBuf); + sTrace("%s", logBuf); + return 0; +} + +void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb== pFsm:%p", pFsm); } + +void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) { + char* s = syncCfg2Str(&newCfg); + sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu, newCfg:%s", + cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term, s); + taosMemoryFree(s); +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + memset(pFsm, 0, sizeof(*pFsm)); + + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + + pFsm->FpReConfigCb = ReConfigCb; + pFsm->FpGetSnapshot = GetSnapshotCb; + pFsm->FpRestoreFinishCb = RestoreFinishCb; + + pFsm->FpSnapshotStartRead = SnapshotStartRead; + pFsm->FpSnapshotStopRead = SnapshotStopRead; + pFsm->FpSnapshotDoRead = SnapshotDoRead; + pFsm->FpSnapshotStartWrite = SnapshotStartWrite; + pFsm->FpSnapshotStopWrite = SnapshotStopWrite; + pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.msgcb = &gSyncIO->msgcb; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + syncInfo.isStandBy = isStandBy; + syncInfo.snapshotEnable = true; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + + gSyncIO->FpOnSyncSnapshotSend = pSyncNode->FpOnSnapshotSend; + gSyncIO->FpOnSyncSnapshotRsp = pSyncNode->FpOnSnapshotRsp; + + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void configChange(int64_t rid, int32_t newReplicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = newReplicaNum; + + for (int i = 0; i < newReplicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf( + "usage: %s replicaNum(1-5) myIndex(0-..) enableSnapshot(0/1) lastApplyIndex(>=-1) lastApplyTerm(>=0) " + "writeRecordNum(>=0) " + "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) \n", + exe); +} + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + sprintf(tsTempDir, "%s", "."); + tsAsyncLog = 0; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; + + if (argc != 10) { + usage(argv[0]); + exit(-1); + } + + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + bool enableSnapshot = atoi(argv[3]); + int32_t lastApplyIndex = atoi(argv[4]); + int32_t lastApplyTerm = atoi(argv[5]); + int32_t writeRecordNum = atoi(argv[6]); + bool isStandBy = atoi(argv[7]); + bool isConfigChange = atoi(argv[8]); + int32_t iterTimes = atoi(argv[9]); + + sTrace( + "args: replicaNum:%d, myIndex:%d, enableSnapshot:%d, lastApplyIndex:%d, lastApplyTerm:%d, writeRecordNum:%d, " + "isStandBy:%d, isConfigChange:%d, iterTimes:%d", + replicaNum, myIndex, enableSnapshot, lastApplyIndex, lastApplyTerm, writeRecordNum, isStandBy, isConfigChange, + iterTimes); + + // check parameter + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(lastApplyTerm >= 0); + assert(writeRecordNum >= 0); + assert(isConfigChange >= 0 && isConfigChange <= 5); + assert(iterTimes >= 0); + + char logFile[256]; + snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); + taosInitLog(logFile, 100); + sTrace("logFile : %s", logFile); + + gSnapshotLastApplyIndex = lastApplyIndex; + gSnapshotLastApplyTerm = lastApplyTerm; + gIterTimes = iterTimes; + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); + assert(rid > 0); + syncStart(rid); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + if (isConfigChange > 0) { + configChange(rid, isConfigChange, myIndex); + } + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* simpleStr = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", simpleStr, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", simpleStr, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", simpleStr); + } + + taosMsleep(1000); + taosMemoryFree(simpleStr); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + taosCloseLog(); + return 0; +} From ba8adb73cbd5788c9b4c106cbdaad2fb057327fc Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 4 Jun 2022 15:49:42 +0800 Subject: [PATCH 043/132] refactor(sync): restart elect timer when vote for --- source/libs/sync/src/syncRequestVote.c | 2 ++ source/libs/sync/test/syncTestTool.cpp | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 460362e728..bd0e684b86 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -62,6 +62,8 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { // maybe has already voted for pMsg->srcId // vote again, no harm raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + syncNodeResetElectTimer(ths); } SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index b5b884b78b..b5efb95057 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -184,7 +184,7 @@ SWal* createWal(char* path, int32_t vgId) { return pWal; } -int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy, bool enableSnapshot) { SSyncInfo syncInfo; syncInfo.vgId = vgId; syncInfo.msgcb = &gSyncIO->msgcb; @@ -194,7 +194,7 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); syncInfo.pWal = pWal; syncInfo.isStandBy = isStandBy; - syncInfo.snapshotEnable = true; + syncInfo.snapshotEnable = enableSnapshot; SSyncCfg* pCfg = &syncInfo.syncCfg; @@ -323,7 +323,7 @@ int main(int argc, char** argv) { snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); SWal* pWal = createWal(walPath, gVgId); - int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy, enableSnapshot); assert(rid > 0); syncStart(rid); From 5908631ec3a59368fcac46ff2e224cf5e12e98f1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 5 Jun 2022 19:47:54 +0800 Subject: [PATCH 044/132] enh(sync): add log store2 --- include/libs/sync/sync.h | 15 ++ source/libs/sync/inc/syncInt.h | 11 +- source/libs/sync/inc/syncRaftLog.h | 10 +- source/libs/sync/src/syncIO.c | 6 +- source/libs/sync/src/syncMain.c | 113 +++++++---- source/libs/sync/src/syncRaftLog.c | 221 ++++++++++++++++++++- source/libs/sync/test/CMakeLists.txt | 42 ++++ source/libs/sync/test/syncRaftLogTest.cpp | 172 ++++++++++++++++ source/libs/sync/test/syncRaftLogTest2.cpp | 98 +++++++++ source/libs/sync/test/syncRaftLogTest3.cpp | 98 +++++++++ source/libs/sync/test/syncTestTool.cpp | 3 +- 11 files changed, 730 insertions(+), 59 deletions(-) create mode 100644 source/libs/sync/test/syncRaftLogTest.cpp create mode 100644 source/libs/sync/test/syncRaftLogTest2.cpp create mode 100644 source/libs/sync/test/syncRaftLogTest3.cpp diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1f71c242ae..c8213160d2 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -141,6 +141,21 @@ typedef struct SSyncLogStore { // return commit index of log SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + // refactor, log[0 .. n] ==> log[m .. n] + int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); + SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); + SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); + bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); + int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); + bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + + SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); + SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore); + + int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); + int32_t (*syncLogGetEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); + int32_t (*syncLogTruncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); + } SSyncLogStore; typedef struct SSyncInfo { diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3f5c081f4a..4fedcdb3b1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -212,9 +212,14 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- -bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); -int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); -int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); +bool syncNodeHasSnapshot(SSyncNode* pSyncNode); +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); +SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); +SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); +int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); +SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); +SyncTerm syncNodeGetPreITerm(SSyncNode* pSyncNode, SyncIndex index); +int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index b54cd14ccd..aec1f77b42 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSyncLogStoreData { SSyncNode* pSyncNode; SWal* pWal; + SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); @@ -41,15 +42,6 @@ char* logStoreSimple2Str(SSyncLogStore* pLogStore); SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); -// SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); -// SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); -// SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); -// SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index); -// int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -// int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex); -// int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); -// SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); - // for debug void logStorePrint(SSyncLogStore* pLogStore); void logStorePrint2(char* s, SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 4e814c41f3..13d1a94b19 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -242,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ab696384a0..05e7bd34eb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1272,6 +1272,18 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { } // snapshot -------------- +bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { + bool ret = false; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN) { + ret = true; + } + } + return ret; +} + bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -1279,54 +1291,73 @@ bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { return b; } +SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + + SyncIndex lastIndex = logLastIndex > snapshot.lastApplyIndex ? logLastIndex : snapshot.lastApplyIndex; + return lastIndex; +} + +SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + + SyncTerm lastTerm = 0; + if (logLastIndex >= snapshot.lastApplyIndex) { + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); + } else { + lastTerm = snapshot.lastApplyTerm; + } + return lastTerm; +} + // get last index and term along with snapshot int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) { - SyncIndex logLastIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore); - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - SyncIndex snapshotLastIndex = snapshot.lastApplyIndex; - - if (logLastIndex > snapshotLastIndex) { - *pLastIndex = logLastIndex; - *pLastTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); - } else if (logLastIndex == snapshotLastIndex) { - *pLastIndex = snapshotLastIndex; - *pLastTerm = snapshot.lastApplyTerm; - } else if (logLastIndex < snapshotLastIndex) { - // maybe wal is deleted - *pLastIndex = snapshotLastIndex; - *pLastTerm = snapshot.lastApplyTerm; - } else { - ASSERT(0); - } + *pLastIndex = syncNodeGetLastIndex(pSyncNode); + *pLastTerm = syncNodeGetLastTerm(pSyncNode); return 0; } +SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex preIndex = index - 1; + return preIndex; +} + +SyncTerm syncNodeGetPreITerm(SSyncNode* pSyncNode, SyncIndex index) { + SyncTerm preTerm = 0; + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, index); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + if (syncNodeIsIndexInSnapshot(pSyncNode, preIndex) && preIndex == snapshot.lastApplyIndex) { + preTerm = snapshot.lastApplyTerm; + } else { + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); + ASSERT(code == 0); + if (pPreEntry != NULL) { + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + } + } + return preTerm; +} + // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { - ASSERT(index >= SYNC_INDEX_BEGIN); - // ASSERT(!syncNodeIsIndexInSnapshot(pSyncNode, index)); - int ret = 0; - - SyncIndex preIndex = index - 1; - if (syncNodeIsIndexInSnapshot(pSyncNode, preIndex)) { - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - ASSERT(preIndex == snapshot.lastApplyIndex); - - *pPreIndex = snapshot.lastApplyIndex; - *pPreTerm = snapshot.lastApplyTerm; - } else { - SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preIndex); - if (pPreEntry != NULL) { - *pPreIndex = pPreEntry->index; - *pPreTerm = pPreEntry->term; - } else { - *pPreIndex = SYNC_INDEX_INVALID; - *pPreTerm = 0; - } - } - + *pPreIndex = syncNodeGetPreIndex(pSyncNode, index); + *pPreTerm = syncNodeGetPreITerm(pSyncNode, index); return 0; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 28b7a0bbf0..0178755651 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -16,6 +16,22 @@ #include "syncRaftLog.h" #include "wal.h" +// refactor, log[0 .. n] ==> log[m .. n] +static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); +static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); +static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); +static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); +static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index); +static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); +static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); +static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); +static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); + +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry); + +//------------------------------- static SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); static SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); static SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); @@ -25,6 +41,168 @@ static int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex from static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); +// refactor, log[0 .. n] ==> log[m .. n] +static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { + // if beginIndex == 0, donot need call this funciton + ASSERT(beginIndex > 0); + + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + pData->beginIndex = beginIndex; + walRestoreFromSnapshot(pWal, beginIndex - 1); + return 0; +} + +static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return pData->beginIndex; +} + +static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } + +static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + return (endIndex >= beginIndex); +} + +static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + int32_t count = endIndex - beginIndex; + return count > 0 ? count : 0; +} + +static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) { + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + SyncIndex endIndex = raftLogEndIndex(pLogStore); + if (index >= beginIndex && index <= endIndex) { + return true; + } else { + return false; + } +} + +static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastVer = walGetLastVer(pWal); + return lastVer; +} + +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { + SyncTerm lastTerm = 0; + if (raftLogEntryCount == 0) { + lastTerm = 0; + } else { + SSyncRaftEntry* pLastEntry; + int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); + ASSERT(code == 0); + lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + } + return lastTerm; +} + +static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + + SyncIndex lastIndex = raftLogLastIndex(pLogStore); + ASSERT(pEntry->index == lastIndex + 1); + + int code = 0; + SSyncLogMeta syncMeta; + syncMeta.isWeek = pEntry->isWeak; + syncMeta.seqNum = pEntry->seqNum; + syncMeta.term = pEntry->term; + code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogAppendEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + ASSERT(0); + } + + walFsync(pWal, true); + return code; +} + +static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + int32_t code; + + *ppEntry = NULL; + if (raftLogInRange(pLogStore, index)) { + SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + ASSERT(pWalHandle != NULL); + + code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogGetEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + walCloseReadHandle(pWalHandle); + ASSERT(0); + return code; + } + + *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); + ASSERT(*ppEntry != NULL); + (*ppEntry)->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; + (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType; + (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; + (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; + (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term; + (*ppEntry)->index = index; + ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); + memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); + + // need to hold, do not new every time!! + walCloseReadHandle(pWalHandle); + + } else { + // index not in range + code = -2; + } + + return code; +} + +static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + int32_t code = walRollback(pWal, fromIndex); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("raftLogTruncate error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); + ASSERT(0); + } + return code; +} + +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { + if (raftLogEntryCount(pLogStore) == 0) { + return -1; + } + SyncIndex lastIndex = raftLogLastIndex(pLogStore); + int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); + return code; +} + +//------------------------------- SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); assert(pLogStore != NULL); @@ -36,6 +214,16 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pData->pSyncNode = pSyncNode; pData->pWal = pSyncNode->pWal; + SyncIndex firstVer = walGetFirstVer(pData->pWal); + SyncIndex lastVer = walGetLastVer(pData->pWal); + if (firstVer >= 0) { + pData->beginIndex = firstVer; + } else if (firstVer == -1) { + pData->beginIndex = lastVer + 1; + } else { + ASSERT(0); + } + pLogStore->appendEntry = logStoreAppendEntry; pLogStore->getEntry = logStoreGetEntry; pLogStore->truncate = logStoreTruncate; @@ -43,6 +231,19 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; + + pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + pLogStore->syncLogBeginIndex = raftLogBeginIndex; + pLogStore->syncLogEndIndex = raftLogEndIndex; + pLogStore->syncLogIsEmpty = raftLogIsEmpty; + pLogStore->syncLogEntryCount = raftLogEntryCount; + pLogStore->syncLogInRange = raftLogInRange; + pLogStore->syncLogLastIndex = raftLogLastIndex; + pLogStore->syncLogLastTerm = raftLogLastTerm; + pLogStore->syncLogAppendEntry = raftLogAppendEntry; + pLogStore->syncLogGetEntry = raftLogGetEntry; + pLogStore->syncLogTruncate = raftLogTruncate; + return pLogStore; } @@ -53,6 +254,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) { } } +//------------------------------- int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -136,7 +338,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { linuxErrMsg); ASSERT(0); } - return 0; // to avoid compiler error + return 0; } SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) { @@ -169,7 +371,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } - return 0; // to avoid compiler error + return 0; } SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) { @@ -204,9 +406,20 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + cJSON* pEntries = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "pEntries", pEntries); SyncIndex lastIndex = logStoreLastIndex(pLogStore); + + for (SyncIndex i = pData->beginIndex; i <= lastIndex; ++i) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + + /* for (SyncIndex i = 0; i <= lastIndex; ++i) { SyncIndex walFirstVer = walGetFirstVer(pData->pWal); @@ -216,6 +429,7 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { syncEntryDestory(pEntry); } } + */ } cJSON* pJson = cJSON_CreateObject(); @@ -244,6 +458,9 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index b6def04eaa..c68c6349fb 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -44,6 +44,9 @@ add_executable(syncSnapshotRspTest "") add_executable(syncSnapshotSenderTest "") add_executable(syncSnapshotReceiverTest "") add_executable(syncTestTool "") +add_executable(syncRaftLogTest "") +add_executable(syncRaftLogTest2 "") +add_executable(syncRaftLogTest3 "") target_sources(syncTest @@ -230,6 +233,18 @@ target_sources(syncTestTool PRIVATE "syncTestTool.cpp" ) +target_sources(syncRaftLogTest + PRIVATE + "syncRaftLogTest.cpp" +) +target_sources(syncRaftLogTest2 + PRIVATE + "syncRaftLogTest2.cpp" +) +target_sources(syncRaftLogTest3 + PRIVATE + "syncRaftLogTest3.cpp" +) target_include_directories(syncTest @@ -462,6 +477,21 @@ target_include_directories(syncTestTool "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncRaftLogTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftLogTest2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncRaftLogTest3 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -648,6 +678,18 @@ target_link_libraries(syncTestTool sync gtest_main ) +target_link_libraries(syncRaftLogTest + sync + gtest_main +) +target_link_libraries(syncRaftLogTest2 + sync + gtest_main +) +target_link_libraries(syncRaftLogTest3 + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp new file mode 100644 index 0000000000..7903e86749 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -0,0 +1,172 @@ +#include "syncRaftLog.h" +//#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +const char *gWalPath = "./syncLogStoreTest_wal"; + +void init() { walInit(); } + +void test1() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test2() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + for (int i = 0; i < 5; ++i) { + int code = walWrite(pWal, i, 100, "aa", 3); + if (code != 0) { + printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); + assert(0); + } + } + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test3() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test4() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + + for (int i = 6; i < 10; ++i) { + int code = walWrite(pWal, i, 100, "aa", 3); + if (code != 0) { + printf("code:%d terror:%d msg:%s i:%d \n", code, terrno, tstrerror(terrno), i); + assert(0); + } + } + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void test5() { + taosRemoveDir(gWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal *pWal = walOpen(gWalPath, &walCfg); + assert(pWal != NULL); + + walRestoreFromSnapshot(pWal, 5); + walRestoreFromSnapshot(pWal, 7); + + int64_t firstVer = walGetFirstVer(pWal); + int64_t lastVer = walGetLastVer(pWal); + printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + + walClose(pWal); +} + +void cleanup() { walCleanUp(); } + +int main(int argc, char **argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + init(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + cleanup(); + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp new file mode 100644 index 0000000000..27e1009335 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -0,0 +1,98 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncNode* pSyncNode; +SWal* pWal; +SSyncLogStore* pLogStore; +const char* pWalPath = "./syncLogStoreTest_wal"; + +void init() { + walInit(); + taosRemoveDir(pWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pWalPath, &walCfg); + assert(pWal != NULL); + + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; +} + +void cleanup() { + walClose(pWal); + walCleanUp(); + taosMemoryFree(pSyncNode); +} + +void logStoreTest() { + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_INVALID); + + logStoreLog2((char*)"logStoreTest", pLogStore); + + for (int i = 0; i < 5; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->getLastIndex(pLogStore) + 1; + snprintf(pEntry->data, dataLen, "value%d", i); + + syncEntryLog2((char*)"==write entry== :", pEntry); + pLogStore->appendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + + if (i == 0) { + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_BEGIN); + } + } + logStoreLog2((char*)"after appendEntry", pLogStore); + + pLogStore->truncate(pLogStore, 3); + logStoreLog2((char*)"after truncate 3", pLogStore); + + logStoreDestory(pLogStore); +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + + init(); + logStoreTest(); + + taosMsleep(2000); + cleanup(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp new file mode 100644 index 0000000000..27e1009335 --- /dev/null +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -0,0 +1,98 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SSyncNode* pSyncNode; +SWal* pWal; +SSyncLogStore* pLogStore; +const char* pWalPath = "./syncLogStoreTest_wal"; + +void init() { + walInit(); + taosRemoveDir(pWalPath); + + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = 1000; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pWalPath, &walCfg); + assert(pWal != NULL); + + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; +} + +void cleanup() { + walClose(pWal); + walCleanUp(); + taosMemoryFree(pSyncNode); +} + +void logStoreTest() { + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_INVALID); + + logStoreLog2((char*)"logStoreTest", pLogStore); + + for (int i = 0; i < 5; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->getLastIndex(pLogStore) + 1; + snprintf(pEntry->data, dataLen, "value%d", i); + + syncEntryLog2((char*)"==write entry== :", pEntry); + pLogStore->appendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + + if (i == 0) { + assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_BEGIN); + } + } + logStoreLog2((char*)"after appendEntry", pLogStore); + + pLogStore->truncate(pLogStore, 3); + logStoreLog2((char*)"after truncate 3", pLogStore); + + logStoreDestory(pLogStore); +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + + init(); + logStoreTest(); + + taosMsleep(2000); + cleanup(); + + return 0; +} diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index b5efb95057..2b69e41fcc 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -184,7 +184,8 @@ SWal* createWal(char* path, int32_t vgId) { return pWal; } -int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy, bool enableSnapshot) { +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy, + bool enableSnapshot) { SSyncInfo syncInfo; syncInfo.vgId = vgId; syncInfo.msgcb = &gSyncIO->msgcb; From 2adc0b8a76116d2a29a83648fc7a3e86cec503f2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 5 Jun 2022 21:48:50 +0800 Subject: [PATCH 045/132] enh(sync): add log index manager --- include/libs/sync/sync.h | 1 + source/libs/sync/src/syncRaftLog.c | 71 ++++++--- source/libs/sync/test/syncRaftLogTest2.cpp | 165 ++++++++++++++++++--- 3 files changed, 193 insertions(+), 44 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index c8213160d2..9a6d42e5bd 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -149,6 +149,7 @@ typedef struct SSyncLogStore { int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 0178755651..55900c1e89 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -20,6 +20,7 @@ static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index); @@ -70,7 +71,7 @@ static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); - int32_t count = endIndex - beginIndex; + int32_t count = endIndex - beginIndex + 1; return count > 0 ? count : 0; } @@ -85,22 +86,48 @@ static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) { } static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { + SyncIndex lastIndex; SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); - return lastVer; + SyncIndex firstVer = walGetFirstVer(pWal); + + if (lastVer < firstVer) { + // no record + lastIndex = -1; + + } else { + if (firstVer >= 0) { + lastIndex = lastVer; + } else if (firstVer == -1) { + lastIndex = -1; + } else { + ASSERT(0); + } + } + + return lastIndex; +} + +static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastVer = walGetLastVer(pWal); + return lastVer + 1; } static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SyncTerm lastTerm = 0; - if (raftLogEntryCount == 0) { + if (raftLogEntryCount(pLogStore) == 0) { lastTerm = 0; } else { SSyncRaftEntry* pLastEntry; int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); ASSERT(code == 0); - lastTerm = pLastEntry->term; - taosMemoryFree(pLastEntry); + if (pLastEntry != NULL) { + lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + } } return lastTerm; } @@ -109,8 +136,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - SyncIndex lastIndex = raftLogLastIndex(pLogStore); - ASSERT(pEntry->index == lastIndex + 1); + SyncIndex writeIndex = raftLogWriteIndex(pLogStore); + ASSERT(pEntry->index == writeIndex); int code = 0; SSyncLogMeta syncMeta; @@ -194,8 +221,9 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn } static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { + *ppLastEntry = NULL; if (raftLogEntryCount(pLogStore) == 0) { - return -1; + return 0; } SyncIndex lastIndex = raftLogLastIndex(pLogStore); int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); @@ -243,6 +271,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->syncLogAppendEntry = raftLogAppendEntry; pLogStore->syncLogGetEntry = raftLogGetEntry; pLogStore->syncLogTruncate = raftLogTruncate; + pLogStore->syncLogWriteIndex = raftLogWriteIndex; return pLogStore; } @@ -401,35 +430,29 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); cJSON* pEntries = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - SyncIndex lastIndex = logStoreLastIndex(pLogStore); - for (SyncIndex i = pData->beginIndex; i <= lastIndex; ++i) { + for (SyncIndex i = pData->beginIndex; i <= endIndex; ++i) { SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); syncEntryDestory(pEntry); } - - /* - for (SyncIndex i = 0; i <= lastIndex; ++i) { - SyncIndex walFirstVer = walGetFirstVer(pData->pWal); - - if (i != SYNC_INDEX_INVALID && i >= walFirstVer) { - SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); - } - } - */ } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 27e1009335..aaa59189f9 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -22,6 +22,16 @@ SWal* pWal; SSyncLogStore* pLogStore; const char* pWalPath = "./syncLogStoreTest_wal"; +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + void init() { walInit(); taosRemoveDir(pWalPath); @@ -41,6 +51,9 @@ void init() { pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); memset(pSyncNode, 0, sizeof(SSyncNode)); pSyncNode->pWal = pWal; + + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb; } void cleanup() { @@ -49,14 +62,37 @@ void cleanup() { taosMemoryFree(pSyncNode); } -void logStoreTest() { +void test1() { + init(); + pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); - assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_INVALID); + logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); + logStoreDestory(pLogStore); - logStoreLog2((char*)"logStoreTest", pLogStore); + cleanup(); +} - for (int i = 0; i < 5; ++i) { +void test2() { + init(); + + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); + logStoreDestory(pLogStore); + + cleanup(); +} + +void test3() { + init(); + + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); + + for (int i = 0; i <= 4; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); assert(pEntry != NULL); @@ -65,34 +101,123 @@ void logStoreTest() { pEntry->seqNum = 3; pEntry->isWeak = true; pEntry->term = 100 + i; - pEntry->index = pLogStore->getLastIndex(pLogStore) + 1; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); snprintf(pEntry->data, dataLen, "value%d", i); - syncEntryLog2((char*)"==write entry== :", pEntry); - pLogStore->appendEntry(pLogStore, pEntry); + pLogStore->syncLogAppendEntry(pLogStore, pEntry); syncEntryDestory(pEntry); - - if (i == 0) { - assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_BEGIN); - } } - logStoreLog2((char*)"after appendEntry", pLogStore); + logStoreLog2((char*)"test3 after appendEntry", pLogStore); + logStoreDestory(pLogStore); - pLogStore->truncate(pLogStore, 3); - logStoreLog2((char*)"after truncate 3", pLogStore); + cleanup(); +} + +void test4() { + init(); + + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test4 after appendEntry", pLogStore); + logStoreDestory(pLogStore); + + cleanup(); +} + +void test5() { + init(); + + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test5 after appendEntry", pLogStore); + + pLogStore->syncLogTruncate(pLogStore, 7); + logStoreLog2((char*)"after truncate 7", pLogStore); logStoreDestory(pLogStore); + + cleanup(); +} + +void test6() { + init(); + + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); + pLogStore->syncLogSetBeginIndex(pLogStore, 5); + + for (int i = 5; i <= 9; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test6 after appendEntry", pLogStore); + + pLogStore->syncLogTruncate(pLogStore, 5); + logStoreLog2((char*)"after truncate 5", pLogStore); + + logStoreDestory(pLogStore); + + cleanup(); } int main(int argc, char** argv) { tsAsyncLog = 0; - sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; - init(); - logStoreTest(); - - taosMsleep(2000); - cleanup(); + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); return 0; } From fb4d372255cca85b9e3d93ece38b6a8adc910d77 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 5 Jun 2022 22:51:15 +0800 Subject: [PATCH 046/132] enh(sync): add log index manager --- source/libs/sync/test/syncRaftLogTest2.cpp | 63 +++++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index aaa59189f9..911948363b 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -34,8 +34,7 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { void init() { walInit(); - taosRemoveDir(pWalPath); - + SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); walCfg.vgId = 1000; @@ -63,31 +62,48 @@ void cleanup() { } void test1() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest1 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } void test2() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pLogStore->syncLogSetBeginIndex(pLogStore, 5); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest2 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } void test3() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); @@ -109,13 +125,21 @@ void test3() { } logStoreLog2((char*)"test3 after appendEntry", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest3 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } void test4() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); @@ -138,13 +162,21 @@ void test4() { } logStoreLog2((char*)"test4 after appendEntry", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest4 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } void test5() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); @@ -171,13 +203,21 @@ void test5() { logStoreLog2((char*)"after truncate 7", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest5 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } void test6() { - init(); + taosRemoveDir(pWalPath); + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); @@ -204,7 +244,14 @@ void test6() { logStoreLog2((char*)"after truncate 5", pLogStore); logStoreDestory(pLogStore); + cleanup(); + // restart + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); + logStoreDestory(pLogStore); cleanup(); } From 5009b7822fff77ab0313fe1e1cf12b4f60e3eb92 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 11:24:25 +0800 Subject: [PATCH 047/132] enh(sync): add log index manager --- source/libs/sync/inc/syncInt.h | 10 +- source/libs/sync/src/syncMain.c | 118 ++++++++-- source/libs/sync/src/syncRaftLog.c | 4 +- source/libs/sync/test/syncRaftLogTest2.cpp | 2 +- source/libs/sync/test/syncRaftLogTest3.cpp | 262 +++++++++++++++++++-- 5 files changed, 345 insertions(+), 51 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 4fedcdb3b1..352649f16f 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -212,13 +212,17 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId); void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- -bool syncNodeHasSnapshot(SSyncNode* pSyncNode); -bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); +bool syncNodeHasSnapshot(SSyncNode* pSyncNode); +bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); + SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm); + +SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode); + SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); -SyncTerm syncNodeGetPreITerm(SSyncNode* pSyncNode, SyncIndex index); +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index); int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); // for debug -------------- diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 05e7bd34eb..04db39ec09 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1285,9 +1285,14 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { } bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(syncNodeHasSnapshot(pSyncNode)); + ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL); + ASSERT(index >= SYNC_INDEX_BEGIN); + SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); bool b = (index <= snapshot.lastApplyIndex); + return b; } @@ -1303,18 +1308,33 @@ SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { } SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshot != NULL) { - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncTerm lastTerm = 0; - if (logLastIndex >= snapshot.lastApplyIndex) { - lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); + if (syncNodeHasSnapshot(pSyncNode)) { + // has snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + if (pSyncNode->pLogStore->syncLogEntryCount(pSyncNode->pLogStore) > 0) { + // has log + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + if (logLastIndex > snapshot.lastApplyIndex) { + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); + } else { + lastTerm = snapshot.lastApplyTerm; + } + + } else { + // no log + lastTerm = snapshot.lastApplyTerm; + } + } else { - lastTerm = snapshot.lastApplyTerm; + // no snapshot + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); } + return lastTerm; } @@ -1325,39 +1345,89 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy return 0; } +SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) { + SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1; + return syncStartIndex; +} + SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { ASSERT(index >= SYNC_INDEX_BEGIN); - SyncIndex preIndex = index - 1; + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + ASSERT(index <= syncStartIndex); + + SyncIndex preIndex; + if (syncNodeHasSnapshot(pSyncNode)) { + // has snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + ASSERT(index > snapshot.lastApplyIndex); + preIndex = index - 1; + + } else { + // no snapshot + preIndex = index - 1; + } + return preIndex; } -SyncTerm syncNodeGetPreITerm(SSyncNode* pSyncNode, SyncIndex index) { - SyncTerm preTerm = 0; - SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, index); +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + ASSERT(index <= syncStartIndex); - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshot != NULL) { - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + if (index == SYNC_INDEX_BEGIN) { + return 0; } - if (syncNodeIsIndexInSnapshot(pSyncNode, preIndex) && preIndex == snapshot.lastApplyIndex) { - preTerm = snapshot.lastApplyTerm; - } else { - SSyncRaftEntry* pPreEntry = NULL; - int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); - ASSERT(code == 0); - if (pPreEntry != NULL) { + SyncTerm preTerm = 0; + if (syncNodeHasSnapshot(pSyncNode)) { + // has snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshot != NULL) { + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + } + + ASSERT(index > snapshot.lastApplyIndex); + if (index > snapshot.lastApplyIndex + 1) { + // should be log preTerm + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + preTerm = pPreEntry->term; taosMemoryFree(pPreEntry); + + } else if (index == snapshot.lastApplyIndex + 1) { + preTerm = snapshot.lastApplyTerm; + } else { + ASSERT(0); } + + } else { + // no snapshot + ASSERT(index > SYNC_INDEX_BEGIN); + + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); } + return preTerm; } // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { *pPreIndex = syncNodeGetPreIndex(pSyncNode, index); - *pPreTerm = syncNodeGetPreITerm(pSyncNode, index); + *pPreTerm = syncNodeGetPreTerm(pSyncNode, index); return 0; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 55900c1e89..b021c3bf59 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -437,11 +437,11 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - + SyncIndex endIndex = raftLogEndIndex(pLogStore); snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - + int32_t count = raftLogEntryCount(pLogStore); cJSON_AddNumberToObject(pRoot, "entryCount", count); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 911948363b..a23384df96 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -34,7 +34,7 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { void init() { walInit(); - + SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); walCfg.vgId = 1000; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 27e1009335..d5f9d91a51 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -22,9 +22,18 @@ SWal* pWal; SSyncLogStore* pLogStore; const char* pWalPath = "./syncLogStoreTest_wal"; +SyncIndex gSnapshotLastApplyIndex; +SyncIndex gSnapshotLastApplyTerm; + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; + return 0; +} + void init() { walInit(); - taosRemoveDir(pWalPath); SWalCfg walCfg; memset(&walCfg, 0, sizeof(SWalCfg)); @@ -41,6 +50,9 @@ void init() { pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); memset(pSyncNode, 0, sizeof(SSyncNode)); pSyncNode->pWal = pWal; + + pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pSyncNode->pFsm->FpGetSnapshot = GetSnapshotCb; } void cleanup() { @@ -49,14 +61,56 @@ void cleanup() { taosMemoryFree(pSyncNode); } -void logStoreTest() { +void test1() { + // no snapshot + // no log + + taosRemoveDir(pWalPath); + + init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); - assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_INVALID); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); - logStoreLog2((char*)"logStoreTest", pLogStore); + gSnapshotLastApplyIndex = -1; + gSnapshotLastApplyTerm = 0; - for (int i = 0; i < 5; ++i) { + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex testIndex = 0; + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, testIndex); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, testIndex); + + sTrace("test1"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("%ld's preIndex: %ld", testIndex, preIndex); + sTrace("%ld's preTerm: %lu", testIndex, preTerm); + + logStoreDestory(pLogStore); + cleanup(); +} + +void test2() { + // no snapshot + // whole log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); + + for (int i = 0; i <= 10; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); assert(pEntry != NULL); @@ -65,34 +119,200 @@ void logStoreTest() { pEntry->seqNum = 3; pEntry->isWeak = true; pEntry->term = 100 + i; - pEntry->index = pLogStore->getLastIndex(pLogStore) + 1; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); snprintf(pEntry->data, dataLen, "value%d", i); - syncEntryLog2((char*)"==write entry== :", pEntry); - pLogStore->appendEntry(pLogStore, pEntry); + pLogStore->syncLogAppendEntry(pLogStore, pEntry); syncEntryDestory(pEntry); - - if (i == 0) { - assert(pLogStore->getLastIndex(pLogStore) == SYNC_INDEX_BEGIN); - } } - logStoreLog2((char*)"after appendEntry", pLogStore); + logStoreLog2((char*)"test2 after appendEntry", pLogStore); - pLogStore->truncate(pLogStore, 3); - logStoreLog2((char*)"after truncate 3", pLogStore); + gSnapshotLastApplyIndex = -1; + gSnapshotLastApplyTerm = 0; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + sTrace("test2"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + + for (SyncIndex i = 11; i >= 0; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + } logStoreDestory(pLogStore); + cleanup(); + +} + +void test3() { + // has snapshot + // no log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, 6); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, 6); + + sTrace("test3"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + sTrace("%d's preIndex: %ld", 6, preIndex); + sTrace("%d's preTerm: %lu", 6, preTerm); + + logStoreDestory(pLogStore); + cleanup(); +} + +void test4() { + // has snapshot + // whole log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); + + for (int i = 0; i <= 10; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test4 after appendEntry", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + sTrace("test4"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + + for (SyncIndex i = 11; i >= 6; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + } + + logStoreDestory(pLogStore); + cleanup(); +} + +void test5() { + // has snapshot + // partial log + + taosRemoveDir(pWalPath); + + init(); + pLogStore = logStoreCreate(pSyncNode); + assert(pLogStore); + pSyncNode->pLogStore = pLogStore; + logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); + + pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + for (int i = 6; i <= 10; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100 + i; + pEntry->index = pLogStore->syncLogWriteIndex(pLogStore); + snprintf(pEntry->data, dataLen, "value%d", i); + + pLogStore->syncLogAppendEntry(pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStoreLog2((char*)"test5 after appendEntry", pLogStore); + + gSnapshotLastApplyIndex = 5; + gSnapshotLastApplyTerm = 100; + + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + + SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); + SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + + sTrace("test5"); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("lastIndex: %ld", lastIndex); + sTrace("lastTerm: %lu", lastTerm); + + for (SyncIndex i = 11; i >= 6; --i) { + SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); + SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); + + sTrace("%ld's preIndex: %ld", i, preIndex); + sTrace("%ld's preTerm: %lu", i, preTerm); + } + + logStoreDestory(pLogStore); + cleanup(); } int main(int argc, char** argv) { tsAsyncLog = 0; - sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; - init(); - logStoreTest(); - - taosMsleep(2000); - cleanup(); + test1(); + test2(); + test3(); + test4(); + test5(); return 0; } From 9c1e1b2f96fb5fc38b33057c2fbd087d8b016bd0 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 11:31:59 +0800 Subject: [PATCH 048/132] enh(sync): add log index manager --- source/libs/sync/test/syncRaftLogTest3.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index d5f9d91a51..4ada395016 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -87,10 +87,13 @@ void test1() { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, testIndex); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, testIndex); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + sTrace("test1"); sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); sTrace("%ld's preIndex: %ld", testIndex, preIndex); sTrace("%ld's preTerm: %lu", testIndex, preTerm); @@ -137,10 +140,13 @@ void test2() { SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + sTrace("test2"); sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); for (SyncIndex i = 11; i >= 0; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); @@ -180,10 +186,13 @@ void test3() { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, 6); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, 6); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + sTrace("test3"); sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); sTrace("%d's preIndex: %ld", 6, preIndex); sTrace("%d's preTerm: %lu", 6, preTerm); @@ -230,10 +239,13 @@ void test4() { SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + sTrace("test4"); sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); for (SyncIndex i = 11; i >= 6; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); @@ -287,10 +299,13 @@ void test5() { SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); SyncTerm lastTerm = syncNodeGetLastTerm(pSyncNode); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + sTrace("test5"); sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); + sTrace("syncStartIndex: %ld", syncStartIndex); for (SyncIndex i = 11; i >= 6; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); From 4b87154b0e05914888b2bce5029d8f42ebe877b9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 11:40:37 +0800 Subject: [PATCH 049/132] enh(sync): forbid elect for this round --- source/libs/sync/src/syncRequestVote.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index bd0e684b86..4d1e259575 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -63,6 +63,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { // vote again, no harm raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + // forbid elect for this round syncNodeResetElectTimer(ths); } From 01b04e03d9927568df3ab640e642fc861523b23d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 11:46:59 +0800 Subject: [PATCH 050/132] enh(sync): forbid elect for this round --- source/libs/sync/src/syncRequestVote.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 4d1e259575..3eef2e7b1e 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -105,6 +105,9 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { // maybe has already voted for pMsg->srcId // vote again, no harm raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); } SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); From af27303a4c0a8901b66781988ad56c4f0b7e66a1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 16:02:25 +0800 Subject: [PATCH 051/132] enh(sync): update raft core functions --- source/libs/sync/inc/syncElection.h | 1 + source/libs/sync/inc/syncInt.h | 2 + source/libs/sync/src/syncAppendEntries.c | 419 ++++++------------ source/libs/sync/src/syncAppendEntriesReply.c | 29 +- source/libs/sync/src/syncCommit.c | 5 + source/libs/sync/src/syncElection.c | 2 +- source/libs/sync/src/syncMain.c | 112 +++++ source/libs/sync/src/syncReplication.c | 137 ++---- source/libs/sync/src/syncRequestVote.c | 28 +- source/libs/sync/src/syncRequestVoteReply.c | 10 +- source/libs/sync/test/syncRaftLogTest3.cpp | 26 +- 11 files changed, 362 insertions(+), 409 deletions(-) diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 7a3d848d46..128dbf4050 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -40,6 +40,7 @@ extern "C" { // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode); int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode); + int32_t syncNodeElect(SSyncNode* pSyncNode); int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 352649f16f..3f269625e1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -225,6 +225,8 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index); SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index); int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm); +int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index fd31bfc148..0683e1d975 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -433,71 +433,133 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } +static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { + if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { + return true; + } + + SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); + SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + + if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { + return true; + } + + return false; +} + +static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg, SSyncRaftEntry** ppAppendEntry) { + int32_t code; + *ppAppendEntry = NULL; + + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, extraIndex, &pExtraEntry); + ASSERT(pExtraEntry != NULL); + + *ppAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ASSERT(*ppAppendEntry != NULL); + + // log not match, conflict, need delete + ASSERT(extraIndex == (*ppAppendEntry)->index); + if (pExtraEntry->term != (*ppAppendEntry)->term) { + conflict = true; + } + syncEntryDestory(pExtraEntry); + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->syncLogLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("entry conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry); + ASSERT(code == 0); + ASSERT(pRollBackEntry != NULL); + + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + code = ths->pLogStore->syncLogTruncate(ths->pLogStore, extraIndex); + ASSERT(code == 0); + } + + return 0; +} + +static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + return 0; +} + int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; + int32_t code = 0; + // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesSnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesLog2(logBuf, pMsg); + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); } - assert(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); // reset elect timer if (pMsg->term == ths->pRaftStore->currentTerm) { ths->leaderCache = pMsg->srcId; syncNodeResetElectTimer(ths); } - assert(pMsg->dataLen >= 0); + ASSERT(pMsg->dataLen >= 0); - SyncIndex localPreLogIndex; - SyncTerm localPreLogTerm; + bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); - bool logOK; - - SyncIndex logFirstIndex = logStoreFirstIndex(ths->pLogStore); - SSnapshot snapshot; - ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - if (logFirstIndex > snapshot.lastApplyIndex) { - logOK = false; - - } else if (syncNodeIsIndexInSnapshot(ths, pMsg->prevLogIndex)) { - SSnapshot snapshot; - ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - - // maybe this assert is error, because replica take takesnapshot separately - // leader will reset next index to newest - ASSERT(pMsg->prevLogIndex == snapshot.lastApplyIndex); - - logOK = (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->prevLogTerm == snapshot.lastApplyTerm); - sTrace( - "1 - logOK:%d, pMsg->prevLogIndex:%ld, snapshot.lastApplyIndex:%ld, pMsg->prevLogTerm:%lu, " - "snapshot.lastApplyTerm:%lu", - logOK, pMsg->prevLogIndex, snapshot.lastApplyIndex, pMsg->prevLogTerm, snapshot.lastApplyTerm); - - } else { - ret = syncNodeGetPreIndexTerm(ths, pMsg->prevLogIndex + 1, &localPreLogIndex, &localPreLogTerm); - ASSERT(ret == 0); - - logOK = (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || - ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && - (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && - (pMsg->prevLogTerm == localPreLogTerm)); - sTrace("2 - logOK:%d, pMsg->prevLogIndex:%ld, getLastIndex:%ld, pMsg->prevLogTerm:%lu, localPreLogTerm:%lu", logOK, - pMsg->prevLogIndex, ths->pLogStore->getLastIndex(ths->pLogStore), pMsg->prevLogTerm, localPreLogTerm); - } - - // reject request + // case1, reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { - sTrace( - "syncNodeOnAppendEntriesSnapshotCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "ths->state:%d, " - "logOK:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + sTrace("recv SyncAppendEntries, reject, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d", pMsg->term, + ths->pRaftStore->currentTerm, ths->state, logOK); + // send response SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; @@ -513,27 +575,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } - SyncIndex localLastIndex; - SyncTerm localLastTerm; - if (logFirstIndex == SYNC_INDEX_INVALID) { - localLastIndex = ths->pLogStore->getLastIndex(ths->pLogStore); - localLastTerm = ths->pLogStore->getLastTerm(ths->pLogStore); - - } else if (logFirstIndex > snapshot.lastApplyIndex) { - localLastIndex = ths->pLogStore->getLastIndex(ths->pLogStore); - localLastTerm = ths->pLogStore->getLastTerm(ths->pLogStore); - - } else { - ret = syncNodeGetLastIndexTerm(ths, &localLastIndex, &localLastTerm); - ASSERT(ret == 0); - } - - // return to follower state + // case 2, return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { - sTrace( - "syncNodeOnAppendEntriesSnapshotCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "ths->state:%d, logOK:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + sTrace("recv SyncAppendEntries, return to follower, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); syncNodeBecomeFollower(ths); @@ -541,99 +586,35 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } - // accept request + // case 3, accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - // preIndex = -1, or has preIndex entry in local log - assert(pMsg->prevLogIndex <= localLastIndex); - // has extra entries (> preIndex) in local log - bool hasExtraEntries = pMsg->prevLogIndex < localLastIndex; + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); + bool hasExtraEntries = myLastIndex > pMsg->prevLogIndex; // has entries in SyncAppendEntries msg bool hasAppendEntries = pMsg->dataLen > 0; sTrace( - "syncNodeOnAppendEntriesSnapshotCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "ths->state:%d, " - "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + "recv SyncAppendEntries, accept, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d, " + "hasExtraEntries:%d, hasAppendEntries:%d", pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); if (hasExtraEntries && hasAppendEntries) { - // not conflict by default - bool conflict = false; + // make log same + SSyncRaftEntry* pAppendEntry; + code = syncNodeMakeLogSame(ths, pMsg, &pAppendEntry); + ASSERT(code == 0); + ASSERT(pAppendEntry != NULL); - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); - assert(pExtraEntry != NULL); + // append new entries + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + ASSERT(code == 0); - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + // pre commit + code = syncNodePreCommit(ths, pAppendEntry); + ASSERT(code == 0); - // log not match, conflict - assert(extraIndex == pAppendEntry->index); - if (pExtraEntry->term != pAppendEntry->term) { - conflict = true; - } - - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; - - sTrace("syncNodeOnAppendEntriesSnapshotCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, - delEnd); - - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); - assert(pRollBackEntry != NULL); - - // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { - if (syncUtilUserRollback(pRollBackEntry->msgType)) { - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - - SFsmCbMeta cbMeta; - cbMeta.index = pRollBackEntry->index; - cbMeta.isWeak = pRollBackEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pRollBackEntry->seqNum; - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); - rpcFreeCont(rpcMsg.pCont); - } - - syncEntryDestory(pRollBackEntry); - } - } - - // delete confict entries - ths->pLogStore->truncate(ths->pLogStore, extraIndex); - - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { - if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { - SFsmCbMeta cbMeta; - cbMeta.index = pAppendEntry->index; - cbMeta.isWeak = pAppendEntry->isWeak; - cbMeta.code = 2; - cbMeta.state = ths->state; - cbMeta.seqNum = pAppendEntry->seqNum; - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); - } - } - rpcFreeCont(rpcMsg.pCont); - } - - // free memory - syncEntryDestory(pExtraEntry); syncEntryDestory(pAppendEntry); } else if (hasExtraEntries && !hasAppendEntries) { @@ -641,38 +622,26 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } else if (!hasExtraEntries && hasAppendEntries) { SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + ASSERT(pAppendEntry != NULL); // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + ASSERT(code == 0); // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { - if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { - SFsmCbMeta cbMeta; - cbMeta.index = pAppendEntry->index; - cbMeta.isWeak = pAppendEntry->isWeak; - cbMeta.code = 3; - cbMeta.state = ths->state; - cbMeta.seqNum = pAppendEntry->seqNum; - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); - } - } - rpcFreeCont(rpcMsg.pCont); + code = syncNodePreCommit(ths, pAppendEntry); + ASSERT(code == 0); - // free memory syncEntryDestory(pAppendEntry); } else if (!hasExtraEntries && !hasAppendEntries) { // do nothing } else { - assert(0); + ASSERT(0); } + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; @@ -685,6 +654,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->matchIndex = pMsg->prevLogIndex; } + // send response SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); @@ -693,7 +663,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // maybe update commit index from leader if (pMsg->commitIndex > ths->commitIndex) { // has commit entry in local - if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { SyncIndex beginIndex = ths->commitIndex + 1; SyncIndex endIndex = pMsg->commitIndex; @@ -701,122 +671,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->commitIndex = pMsg->commitIndex; // call back Wal - ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + ASSERT(code == 0); - // execute fsm - if (ths->pFsm != NULL) { - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - // notice! wal maybe deleted, update firstVer - SyncIndex walFirstVer = walGetFirstVer(ths->pWal); - - if (i != SYNC_INDEX_INVALID && i >= walFirstVer) { - SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, i); - assert(pEntry != NULL); - - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pEntry, &rpcMsg); - - if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { - SFsmCbMeta cbMeta; - cbMeta.index = pEntry->index; - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = ths->pRaftStore->currentTerm; - cbMeta.flag = 0x11; - - SSnapshot snapshot; - ASSERT(ths->pFsm->FpGetSnapshot != NULL); - ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - - bool needExecute = true; - if (cbMeta.index <= snapshot.lastApplyIndex) { - needExecute = false; - } - - if (needExecute) { - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); - } - } - - // config change - if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { - SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; - - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); - - // update new config myIndex - bool hit = false; - for (int i = 0; i < newSyncCfg.replicaNum; ++i) { - if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && - ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { - newSyncCfg.myIndex = i; - hit = true; - break; - } - } - - SReConfigCbMeta cbMeta = {0}; - bool isDrop; - - // I am in newConfig - if (hit) { - syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); - - // change isStandBy to normal - if (!isDrop) { - if (ths->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(ths); - } else { - syncNodeBecomeFollower(ths); - } - } - - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - } - - // always call FpReConfigCb - if (ths->pFsm->FpReConfigCb != NULL) { - cbMeta.code = 0; - cbMeta.currentTerm = ths->pRaftStore->currentTerm; - cbMeta.index = pEntry->index; - cbMeta.term = pEntry->term; - cbMeta.oldCfg = oldSyncCfg; - cbMeta.flag = 0x11; - cbMeta.isDrop = isDrop; - ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); - } - } - - // restore finish - if (pEntry->index == ths->pLogStore->getLastIndex(ths->pLogStore)) { - if (ths->restoreFinish == false) { - if (ths->pFsm->FpRestoreFinishCb != NULL) { - ths->pFsm->FpRestoreFinishCb(ths->pFsm); - } - ths->restoreFinish = true; - sInfo("==syncNodeOnAppendEntriesSnapshotCb== restoreFinish set true %p vgId:%d", ths, ths->vgId); - - /* - tsem_post(&ths->restoreSem); - sInfo("==syncNodeOnAppendEntriesSnapshotCb== RestoreFinish tsem_post %p", ths); - */ - } - } - - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); - } - } - } + code = syncNodeCommit(ths, beginIndex, endIndex, 0x11); + ASSERT(code == 0); } } } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 8ba169d093..b4520f55a9 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -98,18 +98,20 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplySnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); + // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, ths->pRaftStore->currentTerm); return ret; } - syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== before pNextIndex", ths->pNextIndex); - syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== before pMatchIndex", ths->pMatchIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply before pMatchIndex:", ths->pMatchIndex); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { @@ -118,14 +120,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "syncNodeOnAppendEntriesReplySnapshotCb error term, receive:%lu current:%lu", + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu", pMsg->term, ths->pRaftStore->currentTerm); syncNodeLog2(logBuf, ths); sError("%s", logBuf); return ret; } - assert(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); if (pMsg->success) { // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] @@ -143,14 +145,25 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // notice! int64, uint64 if (nextIndex > SYNC_INDEX_BEGIN) { --nextIndex; + + // has snapshot + if (syncNodeHasSnapshot(ths)) { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + if (nextIndex <= snapshot.lastApplyIndex) { + nextIndex = snapshot.lastApplyIndex + 1; + sInfo("reset new nextIndex %ld, snapshot.lastApplyIndex:%ld", nextIndex, snapshot.lastApplyIndex); + } + } + } else { nextIndex = SYNC_INDEX_BEGIN; } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); } - syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== after pNextIndex", ths->pNextIndex); - syncIndexMgrLog2("==syncNodeOnAppendEntriesReplySnapshotCb== after pMatchIndex", ths->pMatchIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b683ab12b4..74233ca4f3 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -94,6 +94,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { + int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, 0x1); + ASSERT(code == 0); + +#if 0 for (SyncIndex i = beginIndex; i <= endIndex; ++i) { if (i != SYNC_INDEX_INVALID) { SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i); @@ -201,6 +205,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { syncEntryDestory(pEntry); } } +#endif } } } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 4a470af9c1..fdebbe3990 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -51,7 +51,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { } int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 04db39ec09..fe6d2b672a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1704,3 +1704,115 @@ const char* syncStr(ESyncState state) { return "ERROR"; } } + +int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { + int32_t code = 0; + + // maybe execute by leader, skip snapshot + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (ths->pFsm->FpGetSnapshot != NULL) { + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + } + if (beginIndex <= snapshot.lastApplyIndex) { + beginIndex = snapshot.lastApplyIndex + 1; + } + + // execute fsm + if (ths->pFsm != NULL) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + if (i != SYNC_INDEX_INVALID) { + SSyncRaftEntry* pEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); + ASSERT(code == 0); + ASSERT(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { + SFsmCbMeta cbMeta; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = flag; + + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + + // config change + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; + + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); + + // update new config myIndex + bool hit = false; + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + hit = true; + break; + } + } + + SReConfigCbMeta cbMeta = {0}; + bool isDrop; + + // I am in newConfig + if (hit) { + syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop); + + // change isStandBy to normal + if (!isDrop) { + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths); + } else { + syncNodeBecomeFollower(ths); + } + } + + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); + taosMemoryFree(sOld); + taosMemoryFree(sNew); + } + + // always call FpReConfigCb + if (ths->pFsm->FpReConfigCb != NULL) { + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.index = pEntry->index; + cbMeta.term = pEntry->term; + cbMeta.oldCfg = oldSyncCfg; + cbMeta.flag = 0x11; + cbMeta.isDrop = isDrop; + ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta); + } + } + + // restore finish + if (pEntry->index == ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { + if (ths->restoreFinish == false) { + if (ths->pFsm->FpRestoreFinishCb != NULL) { + ths->pFsm->FpRestoreFinishCb(ths->pFsm); + } + ths->restoreFinish = true; + sInfo("restore finish %p vgId:%d", ths, ths->vgId); + } + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } + } + return 0; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 4b9a8c51aa..d26ed6bcb4 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -117,121 +117,64 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { } int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); - syncIndexMgrLog2("==syncNodeAppendEntriesPeersSnapshot== pNextIndex", pSyncNode->pNextIndex); - syncIndexMgrLog2("==syncNodeAppendEntriesPeersSnapshot== pMatchIndex", pSyncNode->pMatchIndex); - logStoreSimpleLog2("==syncNodeAppendEntriesPeersSnapshot==", pSyncNode->pLogStore); + syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex); + syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex); + logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore); int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); + // next index SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); - SyncIndex preLogIndex; - SyncTerm preLogTerm; + + // pre index, pre term + SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex); + SyncTerm preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex); // batch optimized // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); - // sending snapshot finish? - bool snapshotSendingFinish = false; - SSyncSnapshotSender* pSender = NULL; - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - if (syncUtilSameId(pDestId, &((pSyncNode->replicasId)[i]))) { - pSender = (pSyncNode->senders)[i]; - } - } - ASSERT(pSender != NULL); - snapshotSendingFinish = (pSender->finish) && (pSender->term == pSyncNode->pRaftStore->currentTerm); - if (snapshotSendingFinish) { - sInfo("snapshotSendingFinish! term:%lu", pSender->term); - } + // prepare entry + SyncAppendEntries* pMsg = NULL; - if ((syncNodeIsIndexInSnapshot(pSyncNode, nextIndex - 1) && !snapshotSendingFinish) || - syncNodeIsIndexInSnapshot(pSyncNode, nextIndex)) { - // will send this msg until snapshot receive finish! + SSyncRaftEntry* pEntry; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, nextIndex, &pEntry); + ASSERT(code == 0); - SSnapshot snapshot = pSender->snapshot; - sInfo("nextIndex:%ld in snapshot: , begin snapshot", nextIndex, - snapshot.lastApplyIndex, snapshot.lastApplyTerm); + if (pEntry != NULL) { + pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); + ASSERT(pMsg != NULL); - // do not use next index - // always send from snapshot.lastApplyIndex + 1, and wait for snapshot transfer finish + // add pEntry into msg + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(len == pEntry->bytes); + memcpy(pMsg->data, serialized, len); - preLogIndex = snapshot.lastApplyIndex; - preLogTerm = snapshot.lastApplyTerm; - - // update next index! - syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, snapshot.lastApplyIndex + 1); - - // to claim leader - SyncAppendEntries* pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); - assert(pMsg != NULL); - pMsg->srcId = pSyncNode->myRaftId; - pMsg->destId = *pDestId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; - pMsg->prevLogIndex = preLogIndex; - pMsg->prevLogTerm = preLogTerm; - pMsg->commitIndex = pSyncNode->commitIndex; - - syncAppendEntriesLog2("==syncNodeAppendEntriesPeersSnapshot==", pMsg); - - // send AppendEntries - syncNodeAppendEntries(pSyncNode, pDestId, pMsg); - syncAppendEntriesDestroy(pMsg); - - if (!snapshotSendingFinish) { - SSyncSnapshotSender* pSender = NULL; - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - if (syncUtilSameId(&((pSyncNode->replicasId)[i]), pDestId)) { - pSender = (pSyncNode->senders)[i]; - break; - } - } - ASSERT(pSender != NULL); - snapshotSenderStart(pSender); - } + taosMemoryFree(serialized); + syncEntryDestory(pEntry); } else { - ret = syncNodeGetPreIndexTerm(pSyncNode, nextIndex, &preLogIndex, &preLogTerm); - ASSERT(ret == 0); - - SyncAppendEntries* pMsg = NULL; - SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, nextIndex); - if (pEntry != NULL) { - pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); - assert(pMsg != NULL); - - // add pEntry into msg - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - assert(len == pEntry->bytes); - memcpy(pMsg->data, serialized, len); - - taosMemoryFree(serialized); - syncEntryDestory(pEntry); - - } else { - // maybe overflow, send empty record - pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); - assert(pMsg != NULL); - } - - assert(pMsg != NULL); - pMsg->srcId = pSyncNode->myRaftId; - pMsg->destId = *pDestId; - pMsg->term = pSyncNode->pRaftStore->currentTerm; - pMsg->prevLogIndex = preLogIndex; - pMsg->prevLogTerm = preLogTerm; - pMsg->commitIndex = pSyncNode->commitIndex; - - syncAppendEntriesLog2("==syncNodeAppendEntriesPeersSnapshot==", pMsg); - - // send AppendEntries - syncNodeAppendEntries(pSyncNode, pDestId, pMsg); - syncAppendEntriesDestroy(pMsg); + // no entry in log + pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); + ASSERT(pMsg != NULL); } + + // prepare msg + ASSERT(pMsg != NULL); + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + + // send msg + syncNodeAppendEntries(pSyncNode, pDestId, pMsg); + syncAppendEntriesDestroy(pMsg); } return ret; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 3eef2e7b1e..4b2d444596 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -81,24 +81,35 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } +static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { + SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); + SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + + if (pMsg->lastLogTerm > myLastTerm) { + return true; + } + if (pMsg->lastLogTerm == myLastTerm && pMsg->lastLogIndex >= myLastIndex) { + return true; + } + + return false; +} + int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; + // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteSnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); } - assert(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); - SyncIndex lastIndex; - SyncTerm lastTerm; - ret = syncNodeGetLastIndexTerm(ths, &lastIndex, &lastTerm); - ASSERT(ret == 0); - - bool logOK = (pMsg->lastLogTerm > lastTerm) || ((pMsg->lastLogTerm == lastTerm) && (pMsg->lastLogIndex >= lastIndex)); + bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); if (grant) { @@ -110,6 +121,7 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { syncNodeResetElectTimer(ths); } + // send msg SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index cff0167f1a..d2009fb355 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -96,12 +96,14 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; + // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplySnapshotCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); + // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, ths->pRaftStore->currentTerm); return ret; } @@ -114,14 +116,14 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplySnapshotCb error term, receive:%lu current:%lu", + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, error term, receive_term:%lu current_term:%lu", pMsg->term, ths->pRaftStore->currentTerm); syncNodePrint2(logBuf, ths); sError("%s", logBuf); return ret; } - assert(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); // This tallies votes even when the current state is not Candidate, // but they won't be looked at, so it doesn't matter. diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 4ada395016..9dbc201966 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -76,7 +76,7 @@ void test1() { gSnapshotLastApplyIndex = -1; gSnapshotLastApplyTerm = 0; - bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -90,7 +90,8 @@ void test1() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test1"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); @@ -133,7 +134,7 @@ void test2() { gSnapshotLastApplyIndex = -1; gSnapshotLastApplyTerm = 0; - bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -143,7 +144,8 @@ void test2() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test2"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); @@ -158,7 +160,6 @@ void test2() { logStoreDestory(pLogStore); cleanup(); - } void test3() { @@ -176,7 +177,7 @@ void test3() { gSnapshotLastApplyIndex = 5; gSnapshotLastApplyTerm = 100; - bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -189,7 +190,8 @@ void test3() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test3"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); @@ -232,7 +234,7 @@ void test4() { gSnapshotLastApplyIndex = 5; gSnapshotLastApplyTerm = 100; - bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -242,7 +244,8 @@ void test4() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test4"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); @@ -292,7 +295,7 @@ void test5() { gSnapshotLastApplyIndex = 5; gSnapshotLastApplyTerm = 100; - bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); + bool hasSnapshot = syncNodeHasSnapshot(pSyncNode); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); @@ -302,7 +305,8 @@ void test5() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test5"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + snapshot.lastApplyTerm); sTrace("lastIndex: %ld", lastIndex); sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); From 6b329f791bda6bee971036807eb4d45a4123dc08 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 16:35:44 +0800 Subject: [PATCH 052/132] enh(sync): update raft core functions --- source/libs/sync/inc/syncInt.h | 2 -- source/libs/sync/src/syncAppendEntriesReply.c | 23 ++++++++++++++++++- source/libs/sync/src/syncMain.c | 18 +-------------- source/libs/sync/src/syncSnapshot.c | 19 ++++----------- 4 files changed, 27 insertions(+), 35 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3f269625e1..ba7e35ec5b 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -152,11 +152,9 @@ typedef struct SSyncNode { SSyncRespMgr* pSyncRespMgr; // restore state - // sem_t restoreSem; bool restoreFinish; // SSnapshot* pSnapshot; SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; - SSyncSnapshotReceiver* receivers[TSDB_MAX_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; } SSyncNode; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index b4520f55a9..f246e4fb29 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -19,6 +19,7 @@ #include "syncInt.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -137,7 +138,9 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); // maybe commit - syncMaybeAdvanceCommitIndex(ths); + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncMaybeAdvanceCommitIndex(ths); + } } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); @@ -151,8 +154,26 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); if (nextIndex <= snapshot.lastApplyIndex) { + ASSERT(nextIndex == snapshot.lastApplyIndex); + nextIndex = snapshot.lastApplyIndex + 1; sInfo("reset new nextIndex %ld, snapshot.lastApplyIndex:%ld", nextIndex, snapshot.lastApplyIndex); + + // start send snapshot + // get sender + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(&(pMsg->srcId), &((ths->replicasId)[i]))) { + pSender = (ths->senders)[i]; + } + } + ASSERT(pSender != NULL); + + if (!(pSender->term == ths->pRaftStore->currentTerm && pSender->finish == true)) { + snapshotSenderStart(pSender); + } else { + sInfo("snapshot send finish, send_term:%lu, current_term:%lu", pSender->term, ths->pRaftStore->currentTerm); + } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fe6d2b672a..4656a8e579 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -598,12 +598,6 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { } // snapshot receivers - for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { - SSyncSnapshotReceiver* pReceiver = snapshotReceiverCreate(pSyncNode, i); - // ASSERT(pReceiver != NULL); - (pSyncNode->receivers)[i] = pReceiver; - } - pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, 100); // start in syncNodeStart @@ -705,13 +699,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { } } - for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { - if ((pSyncNode->receivers)[i] != NULL) { - snapshotReceiverDestroy((pSyncNode->receivers)[i]); - (pSyncNode->receivers)[i] = NULL; - } - } - if (pSyncNode->pNewNodeReceiver != NULL) { snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; @@ -1025,10 +1012,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // snapshot receivers cJSON* pReceivers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "receivers", pReceivers); - for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { - cJSON_AddItemToArray(pReceivers, snapshotReceiver2Json((pSyncNode->receivers)[i])); - } + cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 6802fc0ed8..19f14e8e1f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -284,7 +284,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -398,7 +398,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -406,19 +406,8 @@ char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { // receiver do something int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // get receiver - SSyncSnapshotReceiver *pReceiver = NULL; - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { - pReceiver = (pSyncNode->receivers)[i]; - } - } - - // add new replica - if (pReceiver == NULL) { - pReceiver = pSyncNode->pNewNodeReceiver; - } - - bool needRsp = false; + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + bool needRsp = false; // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { From 84a67e850e9ed2cfe9d1713191f40973210d0ef7 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 17:23:59 +0800 Subject: [PATCH 053/132] fix(sync): sending snapshot --- include/libs/sync/sync.h | 1 + source/libs/sync/src/syncAppendEntries.c | 5 ++++- source/libs/sync/src/syncRaftLog.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 9a6d42e5bd..8f83e21ce4 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -91,6 +91,7 @@ typedef struct SSnapshot { void* data; SyncIndex lastApplyIndex; SyncTerm lastApplyTerm; + SyncIndex lastConfigIndex; } SSnapshot; typedef struct SSyncFSM { diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 0683e1d975..97600c3f15 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -438,9 +438,12 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries return true; } - SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + if (pMsg->prevLogIndex > myLastIndex) { + return false; + } + SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { return true; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index b021c3bf59..95f5926a6f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -198,7 +198,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, } else { // index not in range - code = -2; + code = 0; } return code; From 0d7272a369a9f472949bf2250cb248cbfd51aa6f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 18:23:37 +0800 Subject: [PATCH 054/132] fix(sync): sending snapshot --- source/libs/sync/src/syncSnapshot.c | 4 +-- source/libs/sync/test/syncTestTool.cpp | 35 ++++++++++++++++++-------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 19f14e8e1f..bfb60e448c 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -284,7 +284,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -398,7 +398,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 2b69e41fcc..f733b6f007 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -24,6 +24,9 @@ SyncIndex gSnapshotLastApplyIndex; SyncIndex gSnapshotLastApplyTerm; int gIterTimes = 0; +SyncIndex gFinishLastApplyIndex; +SyncIndex gFinishLastApplyTerm; + void init() { int code = walInit(); assert(code == 0); @@ -111,22 +114,25 @@ int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32 int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) { *ppWriter = (void*)0xCDEF; char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter); sTrace("%s", logBuf); return 0; } int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { - char logBuf[256] = {0}; - snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, - isApply); - sTrace("%s", logBuf); - if (isApply) { - gSnapshotLastApplyIndex = 10; - gSnapshotLastApplyTerm = 1; + gSnapshotLastApplyIndex = gFinishLastApplyIndex; + gSnapshotLastApplyTerm = gFinishLastApplyTerm; } + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d, gSnapshotLastApplyIndex:%ld, " + "gSnapshotLastApplyTerm:%ld", + pFsm, pWriter, isApply, gSnapshotLastApplyIndex, gSnapshotLastApplyTerm); + sTrace("%s", logBuf); + return 0; } @@ -258,7 +264,7 @@ void usage(char* exe) { printf( "usage: %s replicaNum(1-5) myIndex(0-..) enableSnapshot(0/1) lastApplyIndex(>=-1) lastApplyTerm(>=0) " "writeRecordNum(>=0) " - "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) \n", + "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) \n", exe); } @@ -277,7 +283,7 @@ int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; - if (argc != 10) { + if (argc != 12) { usage(argv[0]); exit(-1); } @@ -291,12 +297,14 @@ int main(int argc, char** argv) { bool isStandBy = atoi(argv[7]); bool isConfigChange = atoi(argv[8]); int32_t iterTimes = atoi(argv[9]); + int32_t finishLastApplyIndex = atoi(argv[10]); + int32_t finishLastApplyTerm = atoi(argv[11]); sTrace( "args: replicaNum:%d, myIndex:%d, enableSnapshot:%d, lastApplyIndex:%d, lastApplyTerm:%d, writeRecordNum:%d, " - "isStandBy:%d, isConfigChange:%d, iterTimes:%d", + "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d", replicaNum, myIndex, enableSnapshot, lastApplyIndex, lastApplyTerm, writeRecordNum, isStandBy, isConfigChange, - iterTimes); + iterTimes, finishLastApplyIndex, finishLastApplyTerm); // check parameter assert(replicaNum >= 1 && replicaNum <= 5); @@ -306,6 +314,8 @@ int main(int argc, char** argv) { assert(writeRecordNum >= 0); assert(isConfigChange >= 0 && isConfigChange <= 5); assert(iterTimes >= 0); + assert(finishLastApplyIndex >= -1); + assert(finishLastApplyTerm >= 0); char logFile[256]; snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); @@ -316,6 +326,9 @@ int main(int argc, char** argv) { gSnapshotLastApplyTerm = lastApplyTerm; gIterTimes = iterTimes; + gFinishLastApplyIndex = finishLastApplyIndex; + gFinishLastApplyTerm = finishLastApplyTerm; + init(); int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); assert(ret == 0); From 2071c5a74e7a47b10f658627c7c73b2b5cb0fa06 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 20:02:27 +0800 Subject: [PATCH 055/132] fix(sync): sending snapshot --- source/libs/sync/src/syncAppendEntries.c | 28 ++++++++++++++++-------- source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncRaftLog.c | 11 ++++++++-- source/libs/sync/src/syncSnapshot.c | 14 ++++++++---- source/libs/sync/test/syncTestTool.cpp | 4 ++-- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 97600c3f15..4fb3df3d9a 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -451,9 +451,11 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries return false; } -static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg, SSyncRaftEntry** ppAppendEntry) { +static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg, SSyncRaftEntry** ppAppendEntry, + bool* pEntryAlreadyWritten) { int32_t code; *ppAppendEntry = NULL; + *pEntryAlreadyWritten = false; // not conflict by default bool conflict = false; @@ -466,10 +468,15 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg, SSyn *ppAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); ASSERT(*ppAppendEntry != NULL); - // log not match, conflict, need delete ASSERT(extraIndex == (*ppAppendEntry)->index); if (pExtraEntry->term != (*ppAppendEntry)->term) { + // log not match, conflict, need delete conflict = true; + } else { + // log match, already written + ASSERT(extraIndex == (*ppAppendEntry)->index && pExtraEntry->term == (*ppAppendEntry)->term); + *pEntryAlreadyWritten = true; + sInfo("entry already written, term:%lu, index:%ld", pExtraEntry->term, pExtraEntry->index); } syncEntryDestory(pExtraEntry); @@ -606,17 +613,20 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (hasExtraEntries && hasAppendEntries) { // make log same SSyncRaftEntry* pAppendEntry; - code = syncNodeMakeLogSame(ths, pMsg, &pAppendEntry); + bool entryAlreadyWritten; + code = syncNodeMakeLogSame(ths, pMsg, &pAppendEntry, &entryAlreadyWritten); ASSERT(code == 0); ASSERT(pAppendEntry != NULL); - // append new entries - code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - ASSERT(code == 0); + if (!entryAlreadyWritten) { + // append new entries + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + ASSERT(code == 0); - // pre commit - code = syncNodePreCommit(ths, pAppendEntry); - ASSERT(code == 0); + // pre commit + code = syncNodePreCommit(ths, pAppendEntry); + ASSERT(code == 0); + } syncEntryDestory(pAppendEntry); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4656a8e579..2aafbb1fb5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1612,7 +1612,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); - SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); assert(pEntry != NULL); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 95f5926a6f..a375152ae3 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -477,13 +477,20 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index bfb60e448c..36388e6d6a 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,6 +14,7 @@ */ #include "syncSnapshot.h" +#include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "wal.h" @@ -284,7 +285,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -398,7 +399,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -427,8 +428,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); - walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); - sInfo("walRestoreFromSnapshot lastIndex:%ld", pMsg->lastIndex); + pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + sInfo("snapshot receive finish, update log begin index:%ld, raft log:%s", pMsg->lastIndex + 1, logSimpleStr); + taosMemoryFree(logSimpleStr); + + // walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); + // sInfo("walRestoreFromSnapshot lastIndex:%ld", pMsg->lastIndex); pReceiver->pWriter = NULL; snapshotReceiverStop(pReceiver); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index f733b6f007..f772b225fe 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -295,7 +295,7 @@ int main(int argc, char** argv) { int32_t lastApplyTerm = atoi(argv[5]); int32_t writeRecordNum = atoi(argv[6]); bool isStandBy = atoi(argv[7]); - bool isConfigChange = atoi(argv[8]); + int32_t isConfigChange = atoi(argv[8]); int32_t iterTimes = atoi(argv[9]); int32_t finishLastApplyIndex = atoi(argv[10]); int32_t finishLastApplyTerm = atoi(argv[11]); @@ -308,7 +308,7 @@ int main(int argc, char** argv) { // check parameter assert(replicaNum >= 1 && replicaNum <= 5); - assert(myIndex >= 0 && myIndex < replicaNum); + //assert(myIndex >= 0 && myIndex < replicaNum); assert(lastApplyIndex >= -1); assert(lastApplyTerm >= 0); assert(writeRecordNum >= 0); From 9baaf2cd0c0a9f54bdf7de7868b8660f314b6230 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 6 Jun 2022 20:55:59 +0800 Subject: [PATCH 056/132] enh(sync): add log index manager --- source/libs/sync/src/syncAppendEntriesReply.c | 18 +++++++++++++++++- source/libs/sync/src/syncReplication.c | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f246e4fb29..f5bcda1955 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -112,7 +112,13 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); - syncIndexMgrLog2("recv SyncAppendEntriesReply before pMatchIndex:", ths->pMatchIndex); + syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex); + { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { @@ -133,6 +139,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pMsg->success) { // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + sTrace("update next index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); @@ -144,6 +151,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + sTrace("begin to update next index:%ld, success:%d", nextIndex, pMsg->success); // notice! int64, uint64 if (nextIndex > SYNC_INDEX_BEGIN) { @@ -180,11 +188,19 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { nextIndex = SYNC_INDEX_BEGIN; } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + sTrace("update next index:%ld, success:%d", nextIndex, pMsg->success); } syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); + { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index d26ed6bcb4..ab3614bac3 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -122,6 +122,12 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex); syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex); logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore); + { + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + snapshot.lastApplyIndex, snapshot.lastApplyTerm); + } int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { From a89e45526759e46dce516ed7cafe9f8d79211342 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 09:55:18 +0800 Subject: [PATCH 057/132] fix(sync): delete some assert temporary --- source/libs/sync/inc/syncInt.h | 2 ++ source/libs/sync/src/syncAppendEntriesReply.c | 8 +++++++- source/libs/sync/src/syncMain.c | 17 ++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index ba7e35ec5b..076526e6c9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -225,6 +225,8 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncInd int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); +bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f5bcda1955..1e587dac92 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -104,6 +104,12 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + sInfo("maybe already dropped"); + return ret; + } + // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, @@ -162,7 +168,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); if (nextIndex <= snapshot.lastApplyIndex) { - ASSERT(nextIndex == snapshot.lastApplyIndex); + // ASSERT(nextIndex == snapshot.lastApplyIndex); nextIndex = snapshot.lastApplyIndex + 1; sInfo("reset new nextIndex %ld, snapshot.lastApplyIndex:%ld", nextIndex, snapshot.lastApplyIndex); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 2aafbb1fb5..951da64b7c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1347,7 +1347,7 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); } - ASSERT(index > snapshot.lastApplyIndex); + // ASSERT(index > snapshot.lastApplyIndex); preIndex = index - 1; } else { @@ -1375,7 +1375,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); } - ASSERT(index > snapshot.lastApplyIndex); + // ASSERT(index > snapshot.lastApplyIndex); if (index > snapshot.lastApplyIndex + 1) { // should be log preTerm SSyncRaftEntry* pPreEntry = NULL; @@ -1389,7 +1389,9 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { } else if (index == snapshot.lastApplyIndex + 1) { preTerm = snapshot.lastApplyTerm; } else { - ASSERT(0); + // ASSERT(0); + // maybe snapshot change + preTerm = snapshot.lastApplyTerm; } } else { @@ -1799,4 +1801,13 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } } return 0; +} + +bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) { + return true; + } + } + return false; } \ No newline at end of file From a56f149af961ad0213bfe71dde87735f6169980c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 13:01:48 +0800 Subject: [PATCH 058/132] refactor: get last commit index to mnode --- source/dnode/mnode/impl/src/mndSync.c | 4 ++-- source/dnode/mnode/sdb/inc/sdb.h | 4 +++- source/dnode/mnode/sdb/src/sdb.c | 8 ++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index c52679fab4..bdf0940828 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -57,8 +57,8 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { SMnode *pMnode = pFsm->data; - pSnapshot->lastApplyIndex = sdbGetApplyIndex(pMnode->pSdb); - pSnapshot->lastApplyTerm = sdbGetApplyTerm(pMnode->pSdb); + pSnapshot->lastApplyIndex = sdbGetCommitIndex(pMnode->pSdb); + pSnapshot->lastApplyTerm = sdbGetCommitTerm(pMnode->pSdb); return 0; } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 4a00befa1e..bf8dd514e0 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -358,9 +358,11 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type); * @return int32_t The current index of sdb */ void sdbSetApplyIndex(SSdb *pSdb, int64_t index); -int64_t sdbGetApplyIndex(SSdb *pSdb); void sdbSetApplyTerm(SSdb *pSdb, int64_t term); +int64_t sdbGetApplyIndex(SSdb *pSdb); int64_t sdbGetApplyTerm(SSdb *pSdb); +int64_t sdbGetCommitIndex(SSdb *pSdb); +int64_t sdbGetCommitTerm(SSdb *pSdb); SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 0526ea5c2d..ccd5018aca 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -159,8 +159,12 @@ static int32_t sdbCreateDir(SSdb *pSdb) { void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; } -int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } - void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; } +int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } + int64_t sdbGetApplyTerm(SSdb *pSdb) { return pSdb->curTerm; } + +int64_t sdbGetCommitIndex(SSdb *pSdb) { return pSdb->lastCommitVer; } + +int64_t sdbGetCommitTerm(SSdb *pSdb) { return pSdb->lastCommitTerm; } \ No newline at end of file From 44a893a1c13cb712aa7959574bbcfff3c34967d4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 13:48:40 +0800 Subject: [PATCH 059/132] fix(sync): snapshot maybe change when sending --- source/libs/sync/src/syncAppendEntriesReply.c | 36 ++++++++++--------- source/libs/sync/src/syncMain.c | 1 + 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 1e587dac92..82b462eacd 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -165,29 +165,33 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // has snapshot if (syncNodeHasSnapshot(ths)) { - SSnapshot snapshot; - ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - if (nextIndex <= snapshot.lastApplyIndex) { - // ASSERT(nextIndex == snapshot.lastApplyIndex); - - nextIndex = snapshot.lastApplyIndex + 1; - sInfo("reset new nextIndex %ld, snapshot.lastApplyIndex:%ld", nextIndex, snapshot.lastApplyIndex); - - // start send snapshot - // get sender - SSyncSnapshotSender* pSender = NULL; - for (int i = 0; i < ths->replicaNum; ++i) { - if (syncUtilSameId(&(pMsg->srcId), &((ths->replicasId)[i]))) { - pSender = (ths->senders)[i]; - } + // get sender + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(&(pMsg->srcId), &((ths->replicasId)[i]))) { + pSender = (ths->senders)[i]; } - ASSERT(pSender != NULL); + } + ASSERT(pSender != NULL); + // calculate sentryIndex + SyncIndex sentryIndex; + if (pSender->start) { + sentryIndex = pSender->snapshot.lastApplyIndex; + + } else { + // start send snapshot if (!(pSender->term == ths->pRaftStore->currentTerm && pSender->finish == true)) { snapshotSenderStart(pSender); } else { sInfo("snapshot send finish, send_term:%lu, current_term:%lu", pSender->term, ths->pRaftStore->currentTerm); } + sentryIndex = pSender->snapshot.lastApplyIndex; + } + + // update nextIndex to sentryIndex + 1 + if (nextIndex <= sentryIndex) { + nextIndex = sentryIndex + 1; } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 951da64b7c..c343ccc3a1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1693,6 +1693,7 @@ const char* syncStr(ESyncState state) { int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; + sInfo("sync commit from %ld to %ld, flag:0x%lX", beginIndex, endIndex, flag); // maybe execute by leader, skip snapshot SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; From 326714a0772e75858c2539242fd935cd6dd9569c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 14:35:20 +0800 Subject: [PATCH 060/132] fix(sync): snapshot maybe change when sending --- source/libs/sync/inc/syncSnapshot.h | 16 +++++++++------- source/libs/sync/src/syncAppendEntriesReply.c | 13 +++++-------- source/libs/sync/src/syncSnapshot.c | 4 ++-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index c028e732b8..f9f890aa8a 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -50,20 +50,22 @@ typedef struct SSyncSnapshotSender { bool finish; } SSyncSnapshotSender; +void snapshotSenderDoStart(SSyncSnapshotSender *pSender); + SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void * pWriter; + void *pWriter; SyncTerm term; SSyncNode *pSyncNode; @@ -74,8 +76,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 82b462eacd..47e76dab4c 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -174,18 +174,15 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } ASSERT(pSender != NULL); - // calculate sentryIndex SyncIndex sentryIndex; - if (pSender->start) { + if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { + // already start sentryIndex = pSender->snapshot.lastApplyIndex; } else { - // start send snapshot - if (!(pSender->term == ths->pRaftStore->currentTerm && pSender->finish == true)) { - snapshotSenderStart(pSender); - } else { - sInfo("snapshot send finish, send_term:%lu, current_term:%lu", pSender->term, ths->pRaftStore->currentTerm); - } + // start send snapshot, first time + snapshotSenderDoStart(pSender); + pSender->start = true; sentryIndex = pSender->snapshot.lastApplyIndex; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 36388e6d6a..4df955083b 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -19,7 +19,7 @@ #include "syncUtil.h" #include "wal.h" -static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); +// static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { @@ -59,7 +59,7 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { } // begin send snapshot (current term, seq begin) -static void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { +void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; From 24a6b352ef028731e83ed4be77ad3820168dad68 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 15:11:16 +0800 Subject: [PATCH 061/132] refactor: adjust some logs --- source/dnode/mnode/impl/src/mndSync.c | 4 ++-- source/dnode/mnode/sdb/src/sdbFile.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index bdf0940828..6f805d8c1d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -34,7 +34,7 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); pMgmt->errCode = cbMeta.code; - mTrace("trans:%d, is proposed, savedTransId:%d code:0x%x, ver:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId, + mDebug("trans:%d, is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId, pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term, syncStr(cbMeta.state), pRaw); if (pMgmt->errCode == 0) { @@ -78,7 +78,7 @@ void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) SSyncMgmt *pMgmt = &pMnode->syncMgmt; pMgmt->errCode = cbMeta.code; - mInfo("trans:-1, sync reconfig is proposed, savedTransId:%d code:0x%x, curTerm:%" PRId64 " term:%" PRId64, + mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term); if (pMgmt->transId == -1) { diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 83135491a9..c0b27c33e8 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -496,6 +496,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); taosThreadMutexLock(&pSdb->filelock); + int64_t commitIndex = pSdb->lastCommitVer; + int64_t commitTerm = pSdb->lastCommitTerm; if (taosCopyFile(datafile, pIter->name) < 0) { taosThreadMutexUnlock(&pSdb->filelock); terrno = TAOS_SYSTEM_ERROR(errno); @@ -514,7 +516,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mInfo("sdbiter:%p, is created to read snapshot, file:%s", pIter, pIter->name); + mInfo("sdbiter:%p, is created to read snapshot, index:%" PRId64 " term:%" PRId64 " file:%s", pIter, commitIndex, + commitTerm, pIter->name); return 0; } From 8680da282a54487945d0f175e402a47a61973d17 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 15:36:41 +0800 Subject: [PATCH 062/132] fix(sync): snapshot maybe change when sending --- source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncAppendEntriesReply.c | 5 +++++ source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncMain.c | 11 +++++++---- source/libs/sync/src/syncRaftLog.c | 5 +++++ 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4fb3df3d9a..4ed64c43df 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -687,7 +687,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); ASSERT(code == 0); - code = syncNodeCommit(ths, beginIndex, endIndex, 0x11); + code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); ASSERT(code == 0); } } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 47e76dab4c..841ee15899 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -178,9 +178,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { // already start sentryIndex = pSender->snapshot.lastApplyIndex; + sTrace("sending snapshot already start: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu", pSender->term, + ths->pRaftStore->currentTerm); } else { // start send snapshot, first time + sTrace("sending snapshot start first: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu", pSender->term, + ths->pRaftStore->currentTerm); + snapshotSenderDoStart(pSender); pSender->start = true; sentryIndex = pSender->snapshot.lastApplyIndex; diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 74233ca4f3..f8d980d235 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -94,7 +94,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { - int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, 0x1); + int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); ASSERT(code == 0); #if 0 diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c343ccc3a1..c22b59212e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1558,7 +1558,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { assert(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); syncNodeReplicate(ths); } @@ -1620,7 +1621,8 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { assert(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); // start replicate right now! syncNodeReplicate(ths); @@ -1692,8 +1694,9 @@ const char* syncStr(ESyncState state) { } int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { - int32_t code = 0; - sInfo("sync commit from %ld to %ld, flag:0x%lX", beginIndex, endIndex, flag); + int32_t code = 0; + ESyncState state = flag; + sInfo("sync event commit from %ld to %ld, %s", beginIndex, endIndex, syncUtilState2String(state)); // maybe execute by leader, skip snapshot SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a375152ae3..ce6734ba6b 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -156,6 +156,9 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr } walFsync(pWal, true); + + sTrace("sync event write wal: %ld", pEntry->index); + return code; } @@ -309,6 +312,8 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { // assert(code == 0); walFsync(pWal, true); + + sTrace("sync event old write wal: %ld", pEntry->index); return code; } From 48cf259181a7e2c92601733785d976a57afc0429 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 16:30:50 +0800 Subject: [PATCH 063/132] refactor: adjust some logs --- source/dnode/mnode/impl/src/mndTrans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c2097c069b..7a32fac7c3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -922,7 +922,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio char detail[1024] = {0}; int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType), pAction->epSet.numOfEps, pAction->epSet.inUse); - for (int32_t i = 0; i < pTrans->lastErrorEpset.numOfEps; ++i) { + for (int32_t i = 0; i < pAction->epSet.numOfEps; ++i) { len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn, pAction->epSet.eps[i].port); } From 305f4a858098b0ca1d9bfb978e7695ebcb6b90f1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 16:33:37 +0800 Subject: [PATCH 064/132] enh(sync): add term into index mgr --- source/libs/sync/inc/syncIndexMgr.h | 8 ++- source/libs/sync/src/syncIndexMgr.c | 54 ++++++++++++--- source/libs/sync/test/syncIndexMgrTest.cpp | 81 ++++++---------------- 3 files changed, 73 insertions(+), 70 deletions(-) diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 0a6e2428fe..c56dedb7ec 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SSyncIndexMgr { SRaftId (*replicas)[TSDB_MAX_REPLICA]; SyncIndex index[TSDB_MAX_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function int32_t replicaNum; SSyncNode *pSyncNode; } SSyncIndexMgr; @@ -40,8 +41,11 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); -char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); + +void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); +SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); // for debug ------------------- void syncIndexMgrPrint(SSyncIndexMgr *pObj); diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 4d556d21dd..eec6adf32f 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -46,6 +46,7 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); + memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm)); /* for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { pSyncIndexMgr->index[i] = 0; @@ -62,7 +63,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, } // maybe config change - // assert(0); + assert(0); } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { @@ -86,14 +87,27 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } - int respondNum = 0; - int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); - for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { - arr[i] = pSyncIndexMgr->index[i]; + + { + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->index[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "index", pIndex); } - cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "index", pIndex); + + { + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + arr[i] = pSyncIndexMgr->privateTerm[i]; + } + cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "privateTerm", pIndex); + } + snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); } @@ -105,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -135,4 +149,26 @@ void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); +} + +void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->privateTerm)[i] = term; + return; + } + } + + // maybe config change + assert(0); +} + +SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + SyncTerm term = (pSyncIndexMgr->privateTerm)[i]; + return term; + } + } + assert(0); } \ No newline at end of file diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 7fcce2bc4f..34a6eeb461 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -22,55 +22,24 @@ int32_t replicaNum = 3; int32_t myIndex = 0; SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; SSyncNode* pSyncNode; SSyncNode* syncNodeInit() { - syncInfo.vgId = 1234; - syncInfo.msgcb = &gSyncIO->msgcb; - syncInfo.FpSendMsg = syncIOSendMsg; - syncInfo.FpEqMsg = syncIOEqMsg; - syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); - - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->myIndex = myIndex; - pCfg->replicaNum = replicaNum; + pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->replicaNum = replicaNum; for (int i = 0; i < replicaNum; ++i) { - pCfg->nodeInfo[i].nodePort = ports[i]; - snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + pSyncNode->replicasId[i].addr = syncUtilAddr2U64("127.0.0.1", ports[i]); + pSyncNode->replicasId[i].vgId = 1234; + + ids[i].addr = pSyncNode->replicasId[i].addr; + ids[i].vgId = pSyncNode->replicasId[i].vgId; } - pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); - - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; - gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; - gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; - gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->pSyncNode = pSyncNode; - return pSyncNode; } -SSyncNode* syncInitTest() { return syncNodeInit(); } - -void initRaftId(SSyncNode* pSyncNode) { - for (int i = 0; i < replicaNum; ++i) { - ids[i] = pSyncNode->replicasId[i]; - char* s = syncUtilRaftId2Str(&ids[i]); - printf("raftId[%d] : %s\n", i, s); - taosMemoryFree(s); - } -} - int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; @@ -80,58 +49,52 @@ int main(int argc, char** argv) { myIndex = atoi(argv[1]); } - int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); - assert(ret == 0); - - ret = syncEnvStart(); - assert(ret == 0); - - SSyncNode* pSyncNode = syncInitTest(); + SSyncNode* pSyncNode = syncNodeInit(); assert(pSyncNode != NULL); - char* serialized = syncNode2Str(pSyncNode); - printf("%s\n", serialized); - taosMemoryFree(serialized); - - initRaftId(pSyncNode); - + printf("---------------------------------------\n"); SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode); assert(pSyncIndexMgr != NULL); - - printf("---------------------------------------\n"); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); + printf("---------------------------------------\n"); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); - - printf("---------------------------------------\n"); + syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); + syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); + syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); printf("---------------------------------------\n"); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); - printf("index %d : %lu \n", i, idx); + SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); + printf("%d: index:%ld term:%lu \n", i, idx, term); } - - syncIndexMgrClear(pSyncIndexMgr); printf("---------------------------------------\n"); + + printf("---------------------------------------\n"); + syncIndexMgrClear(pSyncIndexMgr); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); taosMemoryFree(serialized); } + printf("---------------------------------------\n"); syncIndexMgrDestroy(pSyncIndexMgr); return 0; From 7852188394c43326cee9bd05ae9d81a00c3c1e3b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 16:49:07 +0800 Subject: [PATCH 065/132] enh(sync): add privateTerm into SSyncSnapshotSender, SyncAppendEntries, SyncAppendEntriesReply --- include/libs/sync/syncTools.h | 2 ++ source/libs/sync/inc/syncSnapshot.h | 1 + source/libs/sync/src/syncMessage.c | 6 ++++++ source/libs/sync/src/syncSnapshot.c | 2 ++ source/libs/sync/test/syncAppendEntriesReplyTest.cpp | 2 ++ source/libs/sync/test/syncAppendEntriesTest.cpp | 1 + source/libs/sync/test/syncSnapshotSenderTest.cpp | 2 ++ 7 files changed, 16 insertions(+) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 21ce75c28e..9176d7b317 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -301,6 +301,7 @@ typedef struct SyncAppendEntries { SyncIndex prevLogIndex; SyncTerm prevLogTerm; SyncIndex commitIndex; + SyncTerm privateTerm; uint32_t dataLen; char data[]; } SyncAppendEntries; @@ -332,6 +333,7 @@ typedef struct SyncAppendEntriesReply { SRaftId destId; // private data SyncTerm term; + SyncTerm privateTerm; bool success; SyncIndex matchIndex; } SyncAppendEntriesReply; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index f9f890aa8a..a297217b74 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -47,6 +47,7 @@ typedef struct SSyncSnapshotSender { SSyncNode *pSyncNode; int32_t replicaIndex; SyncTerm term; + SyncTerm privateTerm; bool finish; } SSyncSnapshotSender; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index e8fa146271..126229af42 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1343,6 +1343,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); @@ -1504,6 +1507,9 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 4df955083b..36d9225b30 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -275,6 +275,8 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); } diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a90259bc3a..d41e99a3cd 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -22,6 +22,8 @@ SyncAppendEntriesReply *createMsg() { pMsg->destId.vgId = 100; pMsg->success = true; pMsg->matchIndex = 77; + pMsg->term = 33; + pMsg->privateTerm = 44; return pMsg; } diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index bb9f306a1c..98b392274e 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -23,6 +23,7 @@ SyncAppendEntries *createMsg() { pMsg->prevLogIndex = 11; pMsg->prevLogTerm = 22; pMsg->commitIndex = 33; + pMsg->privateTerm = 44; strcpy(pMsg->data, "hello world"); return pMsg; } diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 76d021004f..404ba2acae 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -40,6 +40,7 @@ SSyncSnapshotSender* createSender() { pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead; pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead; pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead; + pSyncNode->pFsm->FpGetSnapshot = GetSnapshot; SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2); pSender->start = true; @@ -54,6 +55,7 @@ SSyncSnapshotSender* createSender() { pSender->snapshot.lastApplyTerm = 88; pSender->sendingMS = 77; pSender->term = 66; + pSender->privateTerm = 99; return pSender; } From 4e99992cffb28dc2c456ae2140dc14fa72bb8ad0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 18:00:50 +0800 Subject: [PATCH 066/132] fix: change quorum on sync reconfig successed --- source/libs/sync/src/syncMain.c | 5 ++++- source/libs/sync/src/syncRaftLog.c | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5404591c7f..80a3fb478e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1074,6 +1074,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDro voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode); votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode); + pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + // isDrop *isDrop = true; bool IamInOld, IamInNew; @@ -1696,7 +1698,8 @@ const char* syncStr(ESyncState state) { int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sInfo("sync event commit from %ld to %ld, %s", beginIndex, endIndex, syncUtilState2String(state)); + sInfo("sync event commit from index:%" PRId64 " to index:%" PRId64 ", %s", beginIndex, endIndex, + syncUtilState2String(state)); // maybe execute by leader, skip snapshot SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 279b99428c..ff70202f87 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -157,7 +157,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sTrace("sync event write wal: %ld", pEntry->index); + sTrace("sync event write index:%" PRId64, pEntry->index); return code; } From 31b54e7462a4eb3497e0ac0c86e6d9a3c89021a5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 18:33:06 +0800 Subject: [PATCH 067/132] enh(sync): add privateTerm into SyncSnapshotRsp, SyncSnapshotSend, SSyncSnapshotReceiver --- include/libs/sync/syncTools.h | 2 ++ source/libs/sync/inc/syncIndexMgr.h | 4 ++-- source/libs/sync/inc/syncSnapshot.h | 23 +++++++++--------- source/libs/sync/src/syncAppendEntries.c | 2 ++ source/libs/sync/src/syncAppendEntriesReply.c | 24 +++++++++++++++---- source/libs/sync/src/syncIndexMgr.c | 2 +- source/libs/sync/src/syncMessage.c | 6 +++++ source/libs/sync/src/syncReplication.c | 1 + source/libs/sync/src/syncSnapshot.c | 14 +++++++++++ source/libs/sync/test/syncIndexMgrTest.cpp | 3 +-- .../sync/test/syncSnapshotReceiverTest.cpp | 1 + source/libs/sync/test/syncSnapshotRspTest.cpp | 1 + .../libs/sync/test/syncSnapshotSendTest.cpp | 1 + source/libs/sync/test/syncTestTool.cpp | 2 +- 14 files changed, 64 insertions(+), 22 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 9176d7b317..dcbb76fe49 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -398,6 +398,7 @@ typedef struct SyncSnapshotSend { SyncTerm term; SyncIndex lastIndex; // lastIndex of snapshot SyncTerm lastTerm; // lastTerm of snapshot + SyncTerm privateTerm; int32_t seq; uint32_t dataLen; char data[]; @@ -432,6 +433,7 @@ typedef struct SyncSnapshotRsp { SyncTerm term; SyncIndex lastIndex; SyncTerm lastTerm; + SyncTerm privateTerm; int32_t ack; } SyncSnapshotRsp; diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index c56dedb7ec..1983d407df 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -41,8 +41,8 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); -char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index a297217b74..383f731ab9 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -55,19 +55,20 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); -void snapshotSenderStart(SSyncSnapshotSender *pSender); -void snapshotSenderStop(SSyncSnapshotSender *pSender); -int32_t snapshotSend(SSyncSnapshotSender *pSender); -int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); +// void snapshotSenderStart(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender); +int32_t snapshotSend(SSyncSnapshotSender *pSender); +int32_t snapshotReSend(SSyncSnapshotSender *pSender); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; + void * pWriter; SyncTerm term; + SyncTerm privateTerm; SSyncNode *pSyncNode; int32_t replicaIndex; @@ -77,8 +78,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4ed64c43df..ec377d6e92 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -576,6 +576,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->term = ths->pRaftStore->currentTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + pReply->privateTerm = pMsg->privateTerm; SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -660,6 +661,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; pReply->success = true; + pReply->privateTerm = pMsg->privateTerm; if (hasAppendEntries) { pReply->matchIndex = pMsg->prevLogIndex + 1; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 841ee15899..1a74b0dd72 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -175,6 +175,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ASSERT(pSender != NULL); SyncIndex sentryIndex; +#if 0 + SyncIndex sentryIndex; if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { // already start sentryIndex = pSender->snapshot.lastApplyIndex; @@ -182,14 +184,26 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ths->pRaftStore->currentTerm); } else { - // start send snapshot, first time - sTrace("sending snapshot start first: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu", pSender->term, - ths->pRaftStore->currentTerm); + if (pMsg->privateTerm == pSender->privateTerm) { + sTrace("same privateTerm, pMsg->privateTerm:%lu, pSender->privateTerm:%lu, do not start snapshot again", + pMsg->privateTerm, pSender->privateTerm); + } else { + // start send snapshot, first time + sTrace( + "sending snapshot start first: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "pMsg->privateTerm:%lu, pSender->privateTerm:%lu", + pSender->term, ths->pRaftStore->currentTerm, pMsg->privateTerm, pSender->privateTerm); + + snapshotSenderDoStart(pSender); + pSender->start = true; + + // update snapshot private term + syncIndexMgrSetTerm(ths->pNextIndex, &(pMsg->srcId), pSender->privateTerm); + } - snapshotSenderDoStart(pSender); - pSender->start = true; sentryIndex = pSender->snapshot.lastApplyIndex; } +#endif // update nextIndex to sentryIndex + 1 if (nextIndex <= sentryIndex) { diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index eec6adf32f..06906195dc 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -119,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 126229af42..7fb6263612 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1815,6 +1815,9 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); @@ -1978,6 +1981,9 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index ab3614bac3..598260f390 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -177,6 +177,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { pMsg->prevLogIndex = preLogIndex; pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; + pMsg->privateTerm = syncIndexMgrGetTerm(pSyncNode->pNextIndex, pDestId); // send msg syncNodeAppendEntries(pSyncNode, pDestId, pMsg); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 36d9225b30..4b60939edf 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,6 +14,7 @@ */ #include "syncSnapshot.h" +#include "syncIndexMgr.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" @@ -42,6 +43,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = pSyncNode->pRaftStore->currentTerm; + pSender->privateTerm = 0; pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; @@ -90,9 +92,13 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { sTrace("snapshot send begin seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); taosMemoryFree(msgStr); + // when start, increase term + ++(pSender->privateTerm); + syncSnapshotSendDestroy(pMsg); } +#if 0 // when entry in snapshot, start sender void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (!(pSender->start)) { @@ -142,6 +148,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { sInfo("snapshotSenderStart %s", s); taosMemoryFree(s); } +#endif void snapshotSenderStop(SSyncSnapshotSender *pSender) { if (pSender->pReader != NULL) { @@ -392,6 +399,9 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex); snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term); cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); } cJSON *pJson = cJSON_CreateObject(); @@ -514,6 +524,10 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { pSender->finish = true; snapshotSenderStop(pSender); + + // update nextIndex private term + syncIndexMgrSetTerm(pSyncNode->pNextIndex, &(pMsg->srcId), pSender->privateTerm); + return 0; } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 34a6eeb461..e920ae3ac6 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -25,7 +25,6 @@ SRaftId ids[TSDB_MAX_REPLICA]; SSyncNode* pSyncNode; SSyncNode* syncNodeInit() { - pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); memset(pSyncNode, 0, sizeof(SSyncNode)); pSyncNode->replicaNum = replicaNum; @@ -81,7 +80,7 @@ int main(int argc, char** argv) { printf("---------------------------------------\n"); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); - SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); + SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); printf("%d: index:%ld term:%lu \n", i, idx, term); } printf("---------------------------------------\n"); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index a9e87ba12d..69670f09a6 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -46,6 +46,7 @@ SSyncSnapshotReceiver* createReceiver() { pReceiver->ack = 20; pReceiver->pWriter = (void*)0x11; pReceiver->term = 66; + pReceiver->privateTerm = 99; return pReceiver; } diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 3c1ce11456..d624d45c94 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -21,6 +21,7 @@ SyncSnapshotRsp *createMsg() { pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; pMsg->term = 11; + pMsg->privateTerm = 99; pMsg->lastIndex = 22; pMsg->lastTerm = 33; pMsg->ack = 44; diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index ca4ac2abeb..01d3264693 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -21,6 +21,7 @@ SyncSnapshotSend *createMsg() { pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; pMsg->term = 11; + pMsg->privateTerm = 99; pMsg->lastIndex = 22; pMsg->lastTerm = 33; pMsg->seq = 44; diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index f772b225fe..c23d06ee38 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -308,7 +308,7 @@ int main(int argc, char** argv) { // check parameter assert(replicaNum >= 1 && replicaNum <= 5); - //assert(myIndex >= 0 && myIndex < replicaNum); + // assert(myIndex >= 0 && myIndex < replicaNum); assert(lastApplyIndex >= -1); assert(lastApplyTerm >= 0); assert(writeRecordNum >= 0); From eba8f2b543d0eaab35a573c8557734e766a624b7 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 7 Jun 2022 19:20:05 +0800 Subject: [PATCH 068/132] fix(sync) send snapshot multi-times --- source/libs/sync/inc/syncIndexMgr.h | 8 ++++---- source/libs/sync/src/syncAppendEntries.c | 5 +++-- source/libs/sync/src/syncAppendEntriesReply.c | 20 +++++++++++++++++++ source/libs/sync/src/syncReplication.c | 3 ++- source/libs/sync/src/syncSnapshot.c | 17 +++++++++++----- source/libs/sync/test/syncIndexMgrTest.cpp | 10 +++++----- 6 files changed, 46 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 1983d407df..e11094e73a 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -41,11 +41,11 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); -char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); -void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); -SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); +// void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); +// SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); // for debug ------------------- void syncIndexMgrPrint(SSyncIndexMgr *pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 00331b3782..41bf7ac778 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -18,6 +18,7 @@ #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" +#include "syncSnapshot.h" #include "syncUtil.h" #include "syncVoteMgr.h" #include "wal.h" @@ -576,7 +577,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->term = ths->pRaftStore->currentTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; - pReply->privateTerm = pMsg->privateTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); @@ -661,7 +662,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; pReply->success = true; - pReply->privateTerm = pMsg->privateTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; if (hasAppendEntries) { pReply->matchIndex = pMsg->prevLogIndex + 1; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 1a74b0dd72..94406c458e 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -175,6 +175,26 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ASSERT(pSender != NULL); SyncIndex sentryIndex; + if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { + // already start + sentryIndex = pSender->snapshot.lastApplyIndex; + sTrace( + "sending snapshot already start: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "pSender->privateTerm:%lu", + pSender->term, ths->pRaftStore->currentTerm, pSender->privateTerm); + + } else { + if (pMsg->privateTerm >= pSender->privateTerm) { + // donot start again + sentryIndex = pSender->snapshot.lastApplyIndex; + + } else { + // start first time + snapshotSenderDoStart(pSender); + pSender->start = true; + sentryIndex = pSender->snapshot.lastApplyIndex; + } + } #if 0 SyncIndex sentryIndex; if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 598260f390..ff39b0b13d 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -177,7 +177,8 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { pMsg->prevLogIndex = preLogIndex; pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; - pMsg->privateTerm = syncIndexMgrGetTerm(pSyncNode->pNextIndex, pDestId); + pMsg->privateTerm = 0; + // pMsg->privateTerm = syncIndexMgrGetTerm(pSyncNode->pNextIndex, pDestId); // send msg syncNodeAppendEntries(pSyncNode, pDestId, pMsg); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 4b60939edf..5715354de0 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -43,7 +43,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = pSyncNode->pRaftStore->currentTerm; - pSender->privateTerm = 0; + pSender->privateTerm = taosGetTimestampMs() + 100; pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; @@ -62,6 +62,9 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { // begin send snapshot (current term, seq begin) void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { + // when start, increase term + ++(pSender->privateTerm); + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -82,6 +85,7 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN + pMsg->privateTerm = pSender->privateTerm; // send SRpcMsg rpcMsg; @@ -92,9 +96,6 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { sTrace("snapshot send begin seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); taosMemoryFree(msgStr); - // when start, increase term - ++(pSender->privateTerm); - syncSnapshotSendDestroy(pMsg); } @@ -198,6 +199,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->lastIndex = pSender->snapshot.lastApplyIndex; pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->seq = pSender->seq; + pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); SRpcMsg rpcMsg; @@ -316,6 +318,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl pReceiver->pSyncNode = pSyncNode; pReceiver->replicaIndex = replicaIndex; pReceiver->term = pSyncNode->pRaftStore->currentTerm; + pReceiver->privateTerm = 0; + } else { sInfo("snapshotReceiverCreate cannot create receiver"); } @@ -377,6 +381,7 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; + ++(pReceiver->privateTerm); char *s = snapshotReceiver2Str(pReceiver); sInfo("snapshotReceiverStop %s", s); @@ -429,6 +434,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // begin snapshotReceiverStart(pReceiver); pReceiver->ack = pMsg->seq; + pReceiver->privateTerm = pMsg->privateTerm; needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); @@ -490,6 +496,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->ack = pReceiver->ack; + pRspMsg->privateTerm = pReceiver->privateTerm; SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); @@ -526,7 +533,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { snapshotSenderStop(pSender); // update nextIndex private term - syncIndexMgrSetTerm(pSyncNode->pNextIndex, &(pMsg->srcId), pSender->privateTerm); + // syncIndexMgrSetTerm(pSyncNode->pNextIndex, &(pMsg->srcId), pSender->privateTerm); return 0; } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index e920ae3ac6..7fb80e9d58 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -66,9 +66,9 @@ int main(int argc, char** argv) { syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); - syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); - syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); - syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); + //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); + //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); + //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); @@ -80,8 +80,8 @@ int main(int argc, char** argv) { printf("---------------------------------------\n"); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); - SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); - printf("%d: index:%ld term:%lu \n", i, idx, term); + //SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); + //printf("%d: index:%ld term:%lu \n", i, idx, term); } printf("---------------------------------------\n"); From ee757b00fb56dd4a5e83d13ed72b6cb232652676 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 19:43:12 +0800 Subject: [PATCH 069/132] test: reproduce coredump --- tests/script/tsim/mnode/basic3.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index bc70cd7a85..37eda973b9 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,8 +39,10 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 +system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 +return $x = 0 step2: $x = $x + 1 From fbbfb0dbe6ddbfe1b2102a151a56bdb855ad7f2c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 20:25:42 +0800 Subject: [PATCH 070/132] feat: drop dnode --- source/dnode/mnode/impl/src/mndDnode.c | 2 + tests/script/jenkins/basic.txt | 1 + tests/script/tsim/dnode/drop_dnode_mnode.sim | 52 ++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 tests/script/tsim/dnode/drop_dnode_mnode.sim diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7be4939337..3fab870277 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -566,9 +566,11 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM pRaw = NULL; if (pMObj != NULL) { + mDebug("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER; } if (numOfVnodes > 0) { + mDebug("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER; } if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3249c02e88..f9d59f5a4f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -22,6 +22,7 @@ # ---- dnode ./test.sh -f tsim/dnode/create_dnode.sim +./test.sh -f tsim/dnode/drop_dnode_mnode.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim diff --git a/tests/script/tsim/dnode/drop_dnode_mnode.sim b/tests/script/tsim/dnode/drop_dnode_mnode.sim new file mode 100644 index 0000000000..e0a85b9803 --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_mnode.sim @@ -0,0 +1,52 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +sql create dnode $hostname port 7300 +sql drop dnode 3 +sql_error drop dnode 1 + +print =============== step2: create mnode +sql create mnode on dnode 2 + +print =============== step3: drop dnode 3 +sql drop dnode 2 +sql show dnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT From bf3c767dccf500488621208238dd6630191fe83e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Jun 2022 20:27:45 +0800 Subject: [PATCH 071/132] test: restore mnode basic test --- tests/script/tsim/mnode/basic3.sim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index 37eda973b9..bc70cd7a85 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,10 +39,8 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 -system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 -return $x = 0 step2: $x = $x + 1 From 2547e8d02ebaabbea14f5653197a39c6a43dbe0c Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 11:03:28 +0800 Subject: [PATCH 072/132] refactor: sync index --- source/libs/sync/inc/syncInt.h | 3 +- source/libs/sync/src/syncAppendEntries.c | 40 ++++++- source/libs/sync/src/syncAppendEntriesReply.c | 31 +---- source/libs/sync/src/syncCommit.c | 110 ------------------ source/libs/sync/src/syncMain.c | 14 +++ source/libs/sync/src/syncRaftLog.c | 2 + source/libs/sync/src/syncRequestVote.c | 6 + source/libs/sync/src/syncRequestVoteReply.c | 6 + source/libs/sync/src/syncSnapshot.c | 14 ++- 9 files changed, 79 insertions(+), 147 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 076526e6c9..bd44718c58 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -225,7 +225,8 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncInd int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); -bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); +bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); +SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 41bf7ac778..2fc1cc07ed 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -549,6 +549,15 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesLog2(logBuf, pMsg); + // if I am standby, be added into a raft group, I should process SyncAppendEntries msg + /* + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + sInfo("recv SyncAppendEntries maybe replica already dropped"); + return ret; + } + */ + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); @@ -598,7 +607,36 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } - // case 3, accept request + // case 3, index in my snapshot + if (pMsg->term == ths->pRaftStore->currentTerm && syncNodeHasSnapshot(ths)) { + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); + if (pMsg->prevLogIndex < snapshot.lastApplyIndex) { + sTrace( + "recv SyncAppendEntries, accept, in snapshot, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d, " + "snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->matchIndex = snapshot.lastApplyIndex - 1; + + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + } + + // case 4, accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { // has extra entries (> preIndex) in local log SyncIndex myLastIndex = syncNodeGetLastIndex(ths); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 94406c458e..16b0f79e3d 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -106,7 +106,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { - sInfo("maybe already dropped"); + sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); return ret; } @@ -195,35 +195,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries sentryIndex = pSender->snapshot.lastApplyIndex; } } -#if 0 - SyncIndex sentryIndex; - if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { - // already start - sentryIndex = pSender->snapshot.lastApplyIndex; - sTrace("sending snapshot already start: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu", pSender->term, - ths->pRaftStore->currentTerm); - - } else { - if (pMsg->privateTerm == pSender->privateTerm) { - sTrace("same privateTerm, pMsg->privateTerm:%lu, pSender->privateTerm:%lu, do not start snapshot again", - pMsg->privateTerm, pSender->privateTerm); - } else { - // start send snapshot, first time - sTrace( - "sending snapshot start first: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "pMsg->privateTerm:%lu, pSender->privateTerm:%lu", - pSender->term, ths->pRaftStore->currentTerm, pMsg->privateTerm, pSender->privateTerm); - - snapshotSenderDoStart(pSender); - pSender->start = true; - - // update snapshot private term - syncIndexMgrSetTerm(ths->pNextIndex, &(pMsg->srcId), pSender->privateTerm); - } - - sentryIndex = pSender->snapshot.lastApplyIndex; - } -#endif // update nextIndex to sentryIndex + 1 if (nextIndex <= sentryIndex) { diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index a64ca36c72..c092b31adf 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -96,116 +96,6 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); ASSERT(code == 0); - -#if 0 - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - if (i != SYNC_INDEX_INVALID) { - SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i); - assert(pEntry != NULL); - - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pEntry, &rpcMsg); - - if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { - SFsmCbMeta cbMeta; - cbMeta.index = pEntry->index; - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = pSyncNode->state; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm; - cbMeta.flag = 0x1; - - SSnapshot snapshot; - ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL); - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - - bool needExecute = true; - if (cbMeta.index <= snapshot.lastApplyIndex) { - needExecute = false; - } - - if (needExecute) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta); - } - } - - // config change - if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { - SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg; - - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); - - // update new config myIndex - bool hit = false; - for (int i = 0; i < newSyncCfg.replicaNum; ++i) { - if (strcmp(pSyncNode->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && - pSyncNode->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { - newSyncCfg.myIndex = i; - hit = true; - break; - } - } - - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - ASSERT(hit == true); - } - - bool isDrop; - syncNodeUpdateConfig(pSyncNode, &newSyncCfg, &isDrop); - - // change isStandBy to normal - if (!isDrop) { - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(pSyncNode); - } else { - syncNodeBecomeFollower(pSyncNode); - } - } - - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x1 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - - if (pSyncNode->pFsm->FpReConfigCb != NULL) { - SReConfigCbMeta cbMeta = {0}; - cbMeta.code = 0; - cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm; - cbMeta.index = pEntry->index; - cbMeta.term = pEntry->term; - cbMeta.oldCfg = oldSyncCfg; - cbMeta.flag = 0x1; - cbMeta.isDrop = isDrop; - pSyncNode->pFsm->FpReConfigCb(pSyncNode->pFsm, newSyncCfg, cbMeta); - } - } - - // restore finish - if (pEntry->index == pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore)) { - if (pSyncNode->restoreFinish == false) { - if (pSyncNode->pFsm->FpRestoreFinishCb != NULL) { - pSyncNode->pFsm->FpRestoreFinishCb(pSyncNode->pFsm); - } - pSyncNode->restoreFinish = true; - sInfo("==syncMaybeAdvanceCommitIndex== restoreFinish set true %p vgId:%d", pSyncNode, pSyncNode->vgId); - - /* - tsem_post(&pSyncNode->restoreSem); - sInfo("==syncMaybeAdvanceCommitIndex== RestoreFinish tsem_post %p", pSyncNode); - */ - } - } - - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); - } - } -#endif } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 80a3fb478e..8db4d6bd11 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -835,6 +835,10 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); if (pSyncNode->FpSendMsg != NULL) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeSendMsgById== msgType:%d", pMsg->msgType); + syncRpcMsgLog2(logBuf, pMsg); + // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1817,4 +1821,14 @@ bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { } } return false; +} + +SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) { + SSyncSnapshotSender* pSender = NULL; + for (int i = 0; i < ths->replicaNum; ++i) { + if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { + pSender = (ths->senders)[i]; + } + } + return pSender; } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ff70202f87..d55f0b80d5 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -44,6 +44,8 @@ static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { + sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex); + // if beginIndex == 0, donot need call this funciton ASSERT(beginIndex > 0); diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 4b2d444596..d939244906 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -103,6 +103,12 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + sInfo("recv SyncRequestVote maybe replica already dropped"); + return ret; + } + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index d2009fb355..9cc905d847 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -101,6 +101,12 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); + return ret; + } + // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 5715354de0..021306397b 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -438,7 +438,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv begin ack:%d recv msg:%s", pReceiver->ack, msgStr); + sTrace("snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -448,7 +448,9 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); - sInfo("snapshot receive finish, update log begin index:%ld, raft log:%s", pMsg->lastIndex + 1, logSimpleStr); + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + sInfo("snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu, raft log:%s", pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); taosMemoryFree(logSimpleStr); // walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); @@ -460,7 +462,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv end ack:%d recv msg:%s", pReceiver->ack, msgStr); + sTrace("snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -469,7 +471,9 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = false; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv force close ack:%d recv msg:%s", pReceiver->ack, msgStr); + sTrace("snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + + taosMemoryFree(msgStr); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { @@ -481,7 +485,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv receiving ack:%d recv msg:%s", pReceiver->ack, msgStr); + sTrace("snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else { From 77cd6f44f443c587b7c4a65a6c25292d8a283f91 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 11:25:24 +0800 Subject: [PATCH 073/132] refactor(sync): syncUtilJson2Line --- source/libs/sync/inc/syncIndexMgr.h | 4 ++-- source/libs/sync/inc/syncUtil.h | 1 + source/libs/sync/src/syncSnapshot.c | 24 ++++++++++++++-------- source/libs/sync/src/syncUtil.c | 22 ++++++++++++++++++++ source/libs/sync/test/syncIndexMgrTest.cpp | 10 ++++----- source/libs/sync/test/syncTimeoutTest.cpp | 21 +++++++++++++++++++ 6 files changed, 66 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index e11094e73a..1f60a9d57e 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -41,8 +41,8 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); -char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); // void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); // SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 1b08d3f7a1..7ecff7ae97 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -61,6 +61,7 @@ bool syncUtilIsData(tmsg_t msgType); bool syncUtilUserPreCommit(tmsg_t msgType); bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); +void syncUtilJson2Line(char* jsonStr); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 021306397b..5b28e5531f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -296,7 +296,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -416,7 +416,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -438,7 +438,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -447,10 +448,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); - char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + char * logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - sInfo("snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu, raft log:%s", pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); + sInfo( + "snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " + "snapshot.lastApplyTerm:%lu, raft log:%s", + pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); taosMemoryFree(logSimpleStr); // walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); @@ -462,7 +466,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -471,9 +476,9 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = false; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, msgStr); - taosMemoryFree(msgStr); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { @@ -485,7 +490,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 48567b75c2..f6ff521e01 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -240,4 +240,26 @@ bool syncUtilUserRollback(tmsg_t msgType) { return true; } return false; +} + +void syncUtilJson2Line(char* jsonStr) { + int p, q, len; + p = 0; + q = 1; + len = strlen(jsonStr); + while (1) { + if (jsonStr[q] == '\0') { + jsonStr[p + 1] = '\0'; + break; + } + + if (jsonStr[q] == '\n' || jsonStr[q] == ' ' || jsonStr[q] == '\t') { + q++; + continue; + } else { + jsonStr[p + 1] = jsonStr[q]; + p++; + q++; + } + } } \ No newline at end of file diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 7fb80e9d58..0ad69f0f51 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -66,9 +66,9 @@ int main(int argc, char** argv) { syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200); syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300); - //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); - //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); - //syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800); + // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900); { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); @@ -80,8 +80,8 @@ int main(int argc, char** argv) { printf("---------------------------------------\n"); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); - //SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); - //printf("%d: index:%ld term:%lu \n", i, idx, term); + // SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); + // printf("%d: index:%ld term:%lu \n", i, idx, term); } printf("---------------------------------------\n"); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 30f25bd1d8..6125b24c1a 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -78,6 +78,26 @@ void test5() { syncTimeoutDestroy(pMsg2); } +void test6() { + SyncTimeout *pMsg = createMsg(); + char * jsonStr = syncTimeout2Str(pMsg); + sTrace("jsonStr: %s", jsonStr); + + syncUtilJson2Line(jsonStr); + sTrace("jsonStr: %s", jsonStr); + + char str[10]; + snprintf(str, sizeof(str), "%s", "{}"); + sTrace("str: %s", str); + syncUtilJson2Line(str); + sTrace("str: %s", str); + + snprintf(str, sizeof(str), "%s", ""); + sTrace("str: %s", str); + syncUtilJson2Line(str); + sTrace("str: %s", str); +} + int main() { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; @@ -88,6 +108,7 @@ int main() { test3(); test4(); test5(); + test6(); return 0; } From f6aed44076dbcf146686b7d9079f8da86b1d5db4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 13:52:33 +0800 Subject: [PATCH 074/132] refactor(sync): syncNodeSendMsgById to one line --- source/libs/sync/src/syncMain.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8db4d6bd11..ad76b0d6a8 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -835,9 +835,10 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); if (pSyncNode->FpSendMsg != NULL) { - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeSendMsgById== msgType:%d", pMsg->msgType); - syncRpcMsgLog2(logBuf, pMsg); + char* JsonStr = syncRpcMsg2Str(pMsg); + syncUtilJson2Line(JsonStr); + sTrace("sync send msg, vgId:%d, type:%d, msg:%s", pSyncNode->vgId, pMsg->msgType, JsonStr); + taosMemoryFree(JsonStr); // htonl syncUtilMsgHtoN(pMsg->pCont); From 687caac2bfe9c2c1a0b6a8cb250cb3a8ed896b6f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 16:45:40 +0800 Subject: [PATCH 075/132] refactor(sync): add raft log test --- source/libs/sync/src/syncAppendEntries.c | 382 ++++++++++----------- source/libs/sync/src/syncMain.c | 45 +-- source/libs/sync/src/syncRaftLog.c | 30 +- source/libs/sync/test/syncRaftLogTest2.cpp | 167 +++++++++ source/libs/sync/test/syncTimeoutTest.cpp | 2 +- 5 files changed, 388 insertions(+), 238 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 2fc1cc07ed..6d0971a1fa 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -434,92 +434,43 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } -static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { - if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { - return true; - } - - SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); - if (pMsg->prevLogIndex > myLastIndex) { - return false; - } - - SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); - if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { - return true; - } - - return false; -} - -static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg, SSyncRaftEntry** ppAppendEntry, - bool* pEntryAlreadyWritten) { +static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; - *ppAppendEntry = NULL; - *pEntryAlreadyWritten = false; - // not conflict by default - bool conflict = false; + SyncIndex delBegin = pMsg->prevLogIndex + 1; + SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore); - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry; - code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, extraIndex, &pExtraEntry); - ASSERT(pExtraEntry != NULL); + // invert roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry; + code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry); + ASSERT(code == 0); + ASSERT(pRollBackEntry != NULL); - *ppAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - ASSERT(*ppAppendEntry != NULL); + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - ASSERT(extraIndex == (*ppAppendEntry)->index); - if (pExtraEntry->term != (*ppAppendEntry)->term) { - // log not match, conflict, need delete - conflict = true; - } else { - // log match, already written - ASSERT(extraIndex == (*ppAppendEntry)->index && pExtraEntry->term == (*ppAppendEntry)->term); - *pEntryAlreadyWritten = true; - sInfo("entry already written, term:%lu, index:%ld", pExtraEntry->term, pExtraEntry->index); - } - syncEntryDestory(pExtraEntry); - - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->syncLogLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; - - sTrace("entry conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); - - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry; - code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry); - ASSERT(code == 0); - ASSERT(pRollBackEntry != NULL); - - if (syncUtilUserRollback(pRollBackEntry->msgType)) { - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - - SFsmCbMeta cbMeta; - cbMeta.index = pRollBackEntry->index; - cbMeta.isWeak = pRollBackEntry->isWeak; - cbMeta.code = 0; - cbMeta.state = ths->state; - cbMeta.seqNum = pRollBackEntry->seqNum; - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); - rpcFreeCont(rpcMsg.pCont); - } - - syncEntryDestory(pRollBackEntry); + SFsmCbMeta cbMeta; + cbMeta.index = pRollBackEntry->index; + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); } - } - // delete confict entries - code = ths->pLogStore->syncLogTruncate(ths->pLogStore, extraIndex); - ASSERT(code == 0); + syncEntryDestory(pRollBackEntry); + } } - return 0; + // delete confict entries + code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); + ASSERT(code == 0); + + return code; } static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { @@ -540,6 +491,26 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { return 0; } +// really pre log match +// prevLogIndex == -1 +static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { + if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { + return true; + } + + SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + if (pMsg->prevLogIndex > myLastIndex) { + return false; + } + + SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); + if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { + return true; + } + + return false; +} + int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; int32_t code = 0; @@ -549,7 +520,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesLog2(logBuf, pMsg); - // if I am standby, be added into a raft group, I should process SyncAppendEntries msg + // if I am standby, to be added into a raft group, I should process SyncAppendEntries msg /* // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { @@ -573,57 +544,55 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); - // case1, reject request - if ((pMsg->term < ths->pRaftStore->currentTerm) || - ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { - sTrace("recv SyncAppendEntries, reject, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d", pMsg->term, - ths->pRaftStore->currentTerm, ths->state, logOK); + // candidate to follower + // + // operation: + // to follower + do { + bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; + if (condition) { + syncNodeBecomeFollower(ths); + // do not reply? + return ret; + } + } while (0); - // send response - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; - pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; - - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); - - return ret; - } - - // case 2, return to follower state - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { - sTrace("recv SyncAppendEntries, return to follower, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); - - syncNodeBecomeFollower(ths); - - // ret or reply? - return ret; - } - - // case 3, index in my snapshot - if (pMsg->term == ths->pRaftStore->currentTerm && syncNodeHasSnapshot(ths)) { + // fake match + // + // condition1: + // I have snapshot, no log, preIndex > myLastIndex + // + // condition2: + // I have snapshot, have log, log <= snapshot, preIndex > myLastIndex + // + // condition3: + // I have snapshot, preIndex <= snapshot.lastApplyIndex + // + // operation: + // match snapshot.lastApplyIndex - 1; + // no operation on log + do { + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); SSnapshot snapshot; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - if (pMsg->prevLogIndex < snapshot.lastApplyIndex) { - sTrace( - "recv SyncAppendEntries, accept, in snapshot, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d, " - "snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, snapshot.lastApplyIndex, snapshot.lastApplyTerm); + bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && + syncNodeHasSnapshot(ths); + bool condition1 = + condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); + bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && + (pMsg->prevLogIndex > myLastIndex); + bool condition3 = condition0 && (pMsg->prevLogIndex <= snapshot.lastApplyIndex); + bool condition = condition1 || condition2 || condition3; + + if (condition) { // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = true; pReply->matchIndex = snapshot.lastApplyIndex - 1; // send response @@ -634,105 +603,120 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } - } + } while (0); - // case 4, accept request - if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - // has extra entries (> preIndex) in local log - SyncIndex myLastIndex = syncNodeGetLastIndex(ths); - bool hasExtraEntries = myLastIndex > pMsg->prevLogIndex; + // not match + // + // condition1: + // term < myTerm + // + // condition2: + // !logOK + // + // operation: + // not match + // no operation on log + do { + bool condition1 = pMsg->term < ths->pRaftStore->currentTerm; + bool condition2 = + (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK; + bool condition = condition1 || condition2; - // has entries in SyncAppendEntries msg - bool hasAppendEntries = pMsg->dataLen > 0; + if (condition) { + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; - sTrace( - "recv SyncAppendEntries, accept, receive_term:%lu, current_term:%lu, ths->state:%d, logOK:%d, " - "hasExtraEntries:%d, hasAppendEntries:%d", - pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); - if (hasExtraEntries && hasAppendEntries) { - // make log same - SSyncRaftEntry* pAppendEntry; - bool entryAlreadyWritten; - code = syncNodeMakeLogSame(ths, pMsg, &pAppendEntry, &entryAlreadyWritten); - ASSERT(code == 0); - ASSERT(pAppendEntry != NULL); + return ret; + } + } while (0); + + // really match + // + // condition: + // logOK + // + // operation: + // match + // make log same + do { + bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && logOK; + if (condition) { + // has extra entries (> preIndex) in local log + SyncIndex myLastIndex = syncNodeGetLastIndex(ths); + bool hasExtraEntries = myLastIndex > pMsg->prevLogIndex; + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + if (hasExtraEntries) { + // make log same, rollback deleted entries + code = syncNodeMakeLogSame(ths, pMsg); + ASSERT(code == 0); + } + + if (hasAppendEntries) { + // append entry + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ASSERT(pAppendEntry != NULL); - if (!entryAlreadyWritten) { - // append new entries code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); ASSERT(code == 0); // pre commit code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); + + syncEntryDestory(pAppendEntry); } - syncEntryDestory(pAppendEntry); + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = true; + pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex; - } else if (hasExtraEntries && !hasAppendEntries) { - // do nothing + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); - } else if (!hasExtraEntries && hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - ASSERT(pAppendEntry != NULL); + // maybe update commit index, leader notice me + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; - // append new entries - code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - ASSERT(code == 0); + // update commit index + ths->commitIndex = pMsg->commitIndex; - // pre commit - code = syncNodePreCommit(ths, pAppendEntry); - ASSERT(code == 0); + // call back Wal + code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + ASSERT(code == 0); - syncEntryDestory(pAppendEntry); - - } else if (!hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else { - ASSERT(0); - } - - // prepare response msg - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; - pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; - - if (hasAppendEntries) { - pReply->matchIndex = pMsg->prevLogIndex + 1; - } else { - pReply->matchIndex = pMsg->prevLogIndex; - } - - // send response - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); - - // maybe update commit index from leader - if (pMsg->commitIndex > ths->commitIndex) { - // has commit entry in local - if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { - SyncIndex beginIndex = ths->commitIndex + 1; - SyncIndex endIndex = pMsg->commitIndex; - - // update commit index - ths->commitIndex = pMsg->commitIndex; - - // call back Wal - code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); - ASSERT(code == 0); - - code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); - ASSERT(code == 0); + code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } } + return ret; } - } + } while (0); return ret; -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ad76b0d6a8..acbe658914 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1283,7 +1283,6 @@ bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); bool b = (index <= snapshot.lastApplyIndex); - return b; } @@ -1307,17 +1306,10 @@ SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); } - if (pSyncNode->pLogStore->syncLogEntryCount(pSyncNode->pLogStore) > 0) { - // has log - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - if (logLastIndex > snapshot.lastApplyIndex) { - lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); - } else { - lastTerm = snapshot.lastApplyTerm; - } - + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + if (logLastIndex > snapshot.lastApplyIndex) { + lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore); } else { - // no log lastTerm = snapshot.lastApplyTerm; } @@ -1346,22 +1338,7 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); ASSERT(index <= syncStartIndex); - SyncIndex preIndex; - if (syncNodeHasSnapshot(pSyncNode)) { - // has snapshot - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshot != NULL) { - pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); - } - - // ASSERT(index > snapshot.lastApplyIndex); - preIndex = index - 1; - - } else { - // no snapshot - preIndex = index - 1; - } - + SyncIndex preIndex = index - 1; return preIndex; } @@ -1382,7 +1359,6 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); } - // ASSERT(index > snapshot.lastApplyIndex); if (index > snapshot.lastApplyIndex + 1) { // should be log preTerm SSyncRaftEntry* pPreEntry = NULL; @@ -1395,10 +1371,19 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { } else if (index == snapshot.lastApplyIndex + 1) { preTerm = snapshot.lastApplyTerm; + } else { - // ASSERT(0); // maybe snapshot change - preTerm = snapshot.lastApplyTerm; + sError("sync get pre term, bad scene. index:%ld", index); + logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore); + + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); + ASSERT(code == 0); + ASSERT(pPreEntry != NULL); + + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); } } else { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d55f0b80d5..e03f8ba65f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -437,10 +437,6 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); @@ -452,6 +448,17 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { int32_t count = raftLogEntryCount(pLogStore); cJSON_AddNumberToObject(pRoot, "entryCount", count); + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + cJSON* pEntries = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "pEntries", pEntries); @@ -484,10 +491,6 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); @@ -498,6 +501,17 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { int32_t count = raftLogEntryCount(pLogStore); cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index a23384df96..d05401b0f3 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -32,6 +32,8 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { return 0; } +bool gAssert = true; + void init() { walInit(); @@ -68,6 +70,17 @@ void test1() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); @@ -76,6 +89,17 @@ void test1() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest1 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); } @@ -88,6 +112,17 @@ void test2() { assert(pLogStore); pLogStore->syncLogSetBeginIndex(pLogStore, 5); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); @@ -96,6 +131,17 @@ void test2() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest2 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); } @@ -108,6 +154,16 @@ void test3() { assert(pLogStore); logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + for (int i = 0; i <= 4; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); @@ -124,6 +180,17 @@ void test3() { syncEntryDestory(pEntry); } logStoreLog2((char*)"test3 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == 4); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 4); + assert(pLogStore->syncLogLastTerm(pLogStore) == 104); + } + logStoreDestory(pLogStore); cleanup(); @@ -132,6 +199,17 @@ void test3() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest3 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 0); + assert(pLogStore->syncLogEndIndex(pLogStore) == 4); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 4); + assert(pLogStore->syncLogLastTerm(pLogStore) == 104); + } + logStoreDestory(pLogStore); cleanup(); } @@ -161,6 +239,17 @@ void test4() { syncEntryDestory(pEntry); } logStoreLog2((char*)"test4 after appendEntry", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + logStoreDestory(pLogStore); cleanup(); @@ -169,6 +258,17 @@ void test4() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest4 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + logStoreDestory(pLogStore); cleanup(); } @@ -199,9 +299,29 @@ void test5() { } logStoreLog2((char*)"test5 after appendEntry", pLogStore); + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + pLogStore->syncLogTruncate(pLogStore, 7); logStoreLog2((char*)"after truncate 7", pLogStore); + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 6); + assert(pLogStore->syncLogEntryCount(pLogStore) == 2); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 6); + assert(pLogStore->syncLogLastTerm(pLogStore) == 106); + } + logStoreDestory(pLogStore); cleanup(); @@ -210,6 +330,17 @@ void test5() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest5 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 6); + assert(pLogStore->syncLogEntryCount(pLogStore) == 2); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 6); + assert(pLogStore->syncLogLastTerm(pLogStore) == 106); + } + logStoreDestory(pLogStore); cleanup(); } @@ -240,9 +371,29 @@ void test6() { } logStoreLog2((char*)"test6 after appendEntry", pLogStore); + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == 9); + assert(pLogStore->syncLogEntryCount(pLogStore) == 5); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogLastIndex(pLogStore) == 9); + assert(pLogStore->syncLogLastTerm(pLogStore) == 109); + } + pLogStore->syncLogTruncate(pLogStore, 5); logStoreLog2((char*)"after truncate 5", pLogStore); + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); @@ -251,6 +402,17 @@ void test6() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); + + if (gAssert) { + assert(pLogStore->syncLogBeginIndex(pLogStore) == 5); + assert(pLogStore->syncLogEndIndex(pLogStore) == -1); + assert(pLogStore->syncLogEntryCount(pLogStore) == 0); + assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogLastIndex(pLogStore) == -1); + assert(pLogStore->syncLogLastTerm(pLogStore) == 0); + } + logStoreDestory(pLogStore); cleanup(); } @@ -259,6 +421,11 @@ int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + if (argc == 2) { + gAssert = atoi(argv[1]); + } + sTrace("gAssert : %d", gAssert); + test1(); test2(); test3(); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 6125b24c1a..e60fabe38b 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -80,7 +80,7 @@ void test5() { void test6() { SyncTimeout *pMsg = createMsg(); - char * jsonStr = syncTimeout2Str(pMsg); + char * jsonStr = syncTimeout2Str(pMsg); sTrace("jsonStr: %s", jsonStr); syncUtilJson2Line(jsonStr); From f1ea9905aaa5ea7428c60d67a7367bea167857f9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Jun 2022 16:50:02 +0800 Subject: [PATCH 076/132] fix: error in schemaless_insert if insert with multi threads --- source/client/src/clientSml.c | 45 ++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bcdba69692..b6db3b7081 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -303,7 +303,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); - taosMsleep(500); + taosMsleep(10); } break; } @@ -327,7 +327,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); - taosMsleep(500); + taosMsleep(10); } break; } @@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); - taosMsleep(500); + taosMsleep(10); } break; } @@ -373,7 +373,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); - taosMsleep(500); + taosMsleep(10); } break; } @@ -424,7 +424,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); - taosMsleep(500); + taosMsleep(10); } break; } @@ -461,18 +461,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle* info, SSchema* schemaField, SH static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); SSmlSTableMeta** tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, NULL); while (tableMetaSml) { SSmlSTableMeta* sTableData = *tableMetaSml; - STableMeta *pTableMeta = NULL; - SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); size_t superTableLen = 0; void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memcpy(pName.tname, superTable, superTableLen); code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); @@ -487,7 +487,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = smlApplySchemaAction(info, &schemaAction); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName); - return code; + goto end; } info->cost.numOfCreateSTables++; }else if (code == TSDB_CODE_SUCCESS) { @@ -502,7 +502,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &schemaAction, true); if (code != TSDB_CODE_SUCCESS) { taosHashCleanup(hashTmp); - return code; + goto end; } taosHashClear(hashTmp); @@ -512,29 +512,33 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false); taosHashCleanup(hashTmp); if (code != TSDB_CODE_SUCCESS) { - return code; + goto end; } - code = catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, -1); + code = catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, 1); if (code != TSDB_CODE_SUCCESS) { - return code; + goto end; } } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); - return code; + goto end; } if(pTableMeta) taosMemoryFree(pTableMeta); code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, (char*)superTable); - return code; + goto end; } sTableData->tableMeta = pTableMeta; tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, tableMetaSml); } return 0; + +end: + catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, 1); + return code; } //========================================================================= @@ -2298,7 +2302,13 @@ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) { info->cost.numOfCTables = taosHashGetSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); - code = smlModifyDBSchemas(info); + + int32_t retryNum = 0; + do{ + code = smlModifyDBSchemas(info); + if (code == 0) break; + } while (retryNum++ < taosHashGetSize(info->superTables)); + if (code != 0) { uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); goto cleanup; @@ -2407,6 +2417,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr info->pRequest->code = smlProcess(info, lines, numLines); end: + info->taos->schemalessType = 0; uDebug("result:%s", info->msgBuf.buf); smlDestroyInfo(info); return (TAOS_RES*)request; From 48b487dd9d42d5a42f389f10d34e854fd2e8f3e9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Jun 2022 17:01:20 +0800 Subject: [PATCH 077/132] fix: error in schemaless_insert if insert with multi threads --- source/client/src/clientSml.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a99c5a4deb..995722a49f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2289,6 +2289,8 @@ static int32_t smlParseLine(SSmlHandle *info, char* lines[], int numLines){ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) { int32_t code = TSDB_CODE_SUCCESS; + int32_t retryNum = 0; + info->cost.parseTime = taosGetTimestampUs(); code = smlParseLine(info, lines, numLines); @@ -2303,7 +2305,6 @@ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) { info->cost.schemaTime = taosGetTimestampUs(); - int32_t retryNum = 0; do{ code = smlModifyDBSchemas(info); if (code == 0) break; From 39d0b652baef3380678991d4bd9bec60a577c290 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 17:18:26 +0800 Subject: [PATCH 078/132] refactor(sync): add raft log test --- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/sync/test/syncRaftLogTest2.cpp | 30 ++++++------- source/libs/sync/test/syncRaftLogTest3.cpp | 51 ++++++++++++++++++++++ 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index e03f8ba65f..7feb7d293f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -67,7 +67,7 @@ static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftL static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); - return (endIndex >= beginIndex); + return (endIndex < beginIndex); } static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index d05401b0f3..64e1da51a1 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -76,7 +76,7 @@ void test1() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -95,7 +95,7 @@ void test1() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -118,7 +118,7 @@ void test2() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -137,7 +137,7 @@ void test2() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -159,7 +159,7 @@ void test3() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 0); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -186,7 +186,7 @@ void test3() { assert(pLogStore->syncLogEndIndex(pLogStore) == 4); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 4); assert(pLogStore->syncLogLastTerm(pLogStore) == 104); } @@ -205,7 +205,7 @@ void test3() { assert(pLogStore->syncLogEndIndex(pLogStore) == 4); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 4); assert(pLogStore->syncLogLastTerm(pLogStore) == 104); } @@ -245,7 +245,7 @@ void test4() { assert(pLogStore->syncLogEndIndex(pLogStore) == 9); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 9); assert(pLogStore->syncLogLastTerm(pLogStore) == 109); } @@ -264,7 +264,7 @@ void test4() { assert(pLogStore->syncLogEndIndex(pLogStore) == 9); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 9); assert(pLogStore->syncLogLastTerm(pLogStore) == 109); } @@ -304,7 +304,7 @@ void test5() { assert(pLogStore->syncLogEndIndex(pLogStore) == 9); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 9); assert(pLogStore->syncLogLastTerm(pLogStore) == 109); } @@ -317,7 +317,7 @@ void test5() { assert(pLogStore->syncLogEndIndex(pLogStore) == 6); assert(pLogStore->syncLogEntryCount(pLogStore) == 2); assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 6); assert(pLogStore->syncLogLastTerm(pLogStore) == 106); } @@ -336,7 +336,7 @@ void test5() { assert(pLogStore->syncLogEndIndex(pLogStore) == 6); assert(pLogStore->syncLogEntryCount(pLogStore) == 2); assert(pLogStore->syncLogWriteIndex(pLogStore) == 7); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 6); assert(pLogStore->syncLogLastTerm(pLogStore) == 106); } @@ -376,7 +376,7 @@ void test6() { assert(pLogStore->syncLogEndIndex(pLogStore) == 9); assert(pLogStore->syncLogEntryCount(pLogStore) == 5); assert(pLogStore->syncLogWriteIndex(pLogStore) == 10); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); assert(pLogStore->syncLogLastIndex(pLogStore) == 9); assert(pLogStore->syncLogLastTerm(pLogStore) == 109); } @@ -389,7 +389,7 @@ void test6() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } @@ -408,7 +408,7 @@ void test6() { assert(pLogStore->syncLogEndIndex(pLogStore) == -1); assert(pLogStore->syncLogEntryCount(pLogStore) == 0); assert(pLogStore->syncLogWriteIndex(pLogStore) == 5); - assert(pLogStore->syncLogIsEmpty(pLogStore) == 0); + assert(pLogStore->syncLogIsEmpty(pLogStore) == 1); assert(pLogStore->syncLogLastIndex(pLogStore) == -1); assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 9dbc201966..b47f8c96c5 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -17,6 +17,8 @@ void logTest() { sFatal("--- sync log test: fatal"); } +bool gAssert = true; + SSyncNode* pSyncNode; SWal* pWal; SSyncLogStore* pLogStore; @@ -98,6 +100,14 @@ void test1() { sTrace("%ld's preIndex: %ld", testIndex, preIndex); sTrace("%ld's preTerm: %lu", testIndex, preTerm); + if (gAssert) { + assert(lastIndex == -1); + assert(lastTerm == 0); + assert(syncStartIndex == 0); + assert(preIndex == -1); + assert(preTerm == 0); + } + logStoreDestory(pLogStore); cleanup(); } @@ -150,12 +160,26 @@ void test2() { sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); + if (gAssert) { + assert(lastIndex == 10); + assert(lastTerm == 110); + assert(syncStartIndex == 11); + } + for (SyncIndex i = 11; i >= 0; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); sTrace("%ld's preIndex: %ld", i, preIndex); sTrace("%ld's preTerm: %lu", i, preTerm); + + if (gAssert) { + SyncIndex preIndexArr[12] = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + SyncTerm preTermArr[12] = {0, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110}; + + assert(preIndex == preIndexArr[i]); + assert(preTerm == preTermArr[i]); + } } logStoreDestory(pLogStore); @@ -198,6 +222,14 @@ void test3() { sTrace("%d's preIndex: %ld", 6, preIndex); sTrace("%d's preTerm: %lu", 6, preTerm); + if (gAssert) { + assert(lastIndex == 5); + assert(lastTerm == 100); + assert(syncStartIndex == 6); + assert(preIndex == 5); + assert(preTerm == 100); + } + logStoreDestory(pLogStore); cleanup(); } @@ -250,6 +282,12 @@ void test4() { sTrace("lastTerm: %lu", lastTerm); sTrace("syncStartIndex: %ld", syncStartIndex); + if (gAssert) { + assert(lastIndex == 10); + assert(lastTerm == 110); + assert(syncStartIndex == 11); + } + for (SyncIndex i = 11; i >= 6; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); @@ -317,6 +355,14 @@ void test5() { sTrace("%ld's preIndex: %ld", i, preIndex); sTrace("%ld's preTerm: %lu", i, preTerm); + + if (gAssert) { + SyncIndex preIndexArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 5, 6, 7, 8, 9, 10}; + SyncTerm preTermArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 100, 106, 107, 108, 109, 110}; + + assert(preIndex == preIndexArr[i]); + assert(preTerm == preTermArr[i]); + } } logStoreDestory(pLogStore); @@ -327,6 +373,11 @@ int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + if (argc == 2) { + gAssert = atoi(argv[1]); + } + sTrace("gAssert : %d", gAssert); + test1(); test2(); test3(); From d72d2d698c20d92cd464e2f399456b866b6fcb0a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 17:44:32 +0800 Subject: [PATCH 079/132] refactor(sync): SyncAppendEntries --- include/libs/sync/sync.h | 1 + source/libs/sync/inc/syncRaftStore.h | 4 +-- source/libs/sync/src/syncAppendEntries.c | 33 +++++++++++++++++++++++- source/libs/sync/src/syncRaftLog.c | 2 ++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 8f83e21ce4..166a363ada 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -144,6 +144,7 @@ typedef struct SSyncLogStore { // refactor, log[0 .. n] ==> log[m .. n] int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); + int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index e0cbcf0744..9f03ac3e55 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -49,8 +49,8 @@ void raftStoreClearVote(SRaftStore *pRaftStore); void raftStoreNextTerm(SRaftStore *pRaftStore); void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson); -cJSON * raftStore2Json(SRaftStore *pRaftStore); -char * raftStore2Str(SRaftStore *pRaftStore); +cJSON *raftStore2Json(SRaftStore *pRaftStore); +char *raftStore2Str(SRaftStore *pRaftStore); // for debug ------------------- void raftStorePrint(SRaftStore *pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 6d0971a1fa..6f725e123e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -469,6 +469,8 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); + sInfo("syncNodeMakeLogSame, from %ld to %ld", delBegin, delEnd); + logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; } @@ -495,19 +497,30 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { // prevLogIndex == -1 static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { + sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); return true; } SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { + sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, + myLastIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { + sTrace( + "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); return true; } + sTrace( + "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); return false; } @@ -517,7 +530,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId, + ths->pRaftStore->currentTerm); syncAppendEntriesLog2(logBuf, pMsg); // if I am standby, to be added into a raft group, I should process SyncAppendEntries msg @@ -551,6 +565,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs do { bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; if (condition) { + sTrace("recv SyncAppendEntries, candidate to follower"); + syncNodeBecomeFollower(ths); // do not reply? return ret; @@ -586,6 +602,12 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2 || condition3; if (condition) { + sTrace( + "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " + "condition1:%d, condition2:%d, condition3:%d", + myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), + ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3); + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; @@ -623,6 +645,12 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2; if (condition) { + sTrace( + "recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, " + "condition2:%d, logOK:%d", + ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore), + condition1, condition2, logOK); + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; @@ -660,6 +688,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // has entries in SyncAppendEntries msg bool hasAppendEntries = pMsg->dataLen > 0; + sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex, + hasExtraEntries, hasAppendEntries); + if (hasExtraEntries) { // make log same, rollback deleted entries code = syncNodeMakeLogSame(ths, pMsg); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 7feb7d293f..d7a8828d94 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -56,6 +56,8 @@ static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex b return 0; } +int32_t raftLogResetBeginIndex(struct SSyncLogStore* pLogStore) { return 0; } + static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; From 76185e1742d54eac8cb077611cd8bb9625630efe Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 18:07:50 +0800 Subject: [PATCH 080/132] refactor(sync): SSyncSnapshotSender --- source/libs/sync/inc/syncSnapshot.h | 27 +++++------ source/libs/sync/src/syncAppendEntriesReply.c | 2 +- source/libs/sync/src/syncSnapshot.c | 47 ++++++++++++------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 383f731ab9..3c22f03bf2 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void * pReader; - void * pCurrentBlock; + void *pReader; + void *pCurrentBlock; int32_t blockLen; SSnapshot snapshot; int64_t sendingMS; @@ -48,25 +48,24 @@ typedef struct SSyncSnapshotSender { int32_t replicaIndex; SyncTerm term; SyncTerm privateTerm; - bool finish; + bool apply; } SSyncSnapshotSender; -void snapshotSenderDoStart(SSyncSnapshotSender *pSender); - SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); -// void snapshotSenderStart(SSyncSnapshotSender *pSender); -void snapshotSenderStop(SSyncSnapshotSender *pSender); -int32_t snapshotSend(SSyncSnapshotSender *pSender); -int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); -char * snapshotSender2Str(SSyncSnapshotSender *pSender); +bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); +void snapshotSenderStart(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender); +int32_t snapshotSend(SSyncSnapshotSender *pSender); +int32_t snapshotReSend(SSyncSnapshotSender *pSender); +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); +char *snapshotSender2Str(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void * pWriter; + void *pWriter; SyncTerm term; SyncTerm privateTerm; @@ -78,8 +77,8 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); -cJSON * snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 16b0f79e3d..e6bef88a4c 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -190,7 +190,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { // start first time - snapshotSenderDoStart(pSender); + snapshotSenderStart(pSender); pSender->start = true; sentryIndex = pSender->snapshot.lastApplyIndex; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 5b28e5531f..b06f8d9779 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -20,7 +20,6 @@ #include "syncUtil.h" #include "wal.h" -// static void snapshotSenderDoStart(SSyncSnapshotSender *pSender); static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { @@ -45,27 +44,28 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->term = pSyncNode->pRaftStore->currentTerm; pSender->privateTerm = taosGetTimestampMs() + 100; pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); - - pSender->finish = false; + pSender->apply = false; } else { - sInfo("snapshotSenderCreate cannot create sender"); + sError("snapshotSenderCreate cannot create sender"); } - return pSender; } void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { if (pSender != NULL) { + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + } taosMemoryFree(pSender); } } -// begin send snapshot (current term, seq begin) -void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { - // when start, increase term - ++(pSender->privateTerm); +bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } + +// begin send snapshot (current term, seq begin) +void snapshotSenderStart(SSyncSnapshotSender *pSender) { + ASSERT(!snapshotSenderIsStart(pSender)); - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -74,9 +74,21 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); ASSERT(ret == 0); + if (pSender->pCurrentBlock != NULL) { + taosMemoryFree(pSender->pCurrentBlock); + } + + pSender->blockLen = 0; + // get current snapshot info pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); + pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + ++(pSender->privateTerm); + pSender->apply = false; + pSender->start = true; + // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; @@ -87,7 +99,7 @@ void snapshotSenderDoStart(SSyncSnapshotSender *pSender) { pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN pMsg->privateTerm = pSender->privateTerm; - // send + // send msg SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); @@ -192,6 +204,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pSender->seq = SYNC_SNAPSHOT_SEQ_END; } + // build msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId); pMsg->srcId = pSender->pSyncNode->myRaftId; pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; @@ -202,6 +215,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + // send msg SRpcMsg rpcMsg; syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); @@ -215,7 +229,6 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); - return 0; } @@ -286,7 +299,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + cJSON_AddNumberToObject(pRoot, "apply", pSender->apply); } cJSON *pJson = cJSON_CreateObject(); @@ -296,7 +309,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -416,7 +429,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -448,7 +461,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); - char * logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); + char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); sInfo( @@ -539,7 +552,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { // receiver ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - pSender->finish = true; + pSender->apply = true; snapshotSenderStop(pSender); // update nextIndex private term From aa107d03c35852c5774e53274293a5b8925c4c63 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 18:17:17 +0800 Subject: [PATCH 081/132] refactor(sync): SSyncSnapshotReceiver --- source/libs/sync/inc/syncSnapshot.h | 1 + source/libs/sync/src/syncSnapshot.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 3c22f03bf2..63e69f77ca 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -76,6 +76,7 @@ typedef struct SSyncSnapshotReceiver { SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index b06f8d9779..1d52d9be33 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -346,6 +346,8 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { } } +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } + // begin receive snapshot msg (current term, seq begin) static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver) { pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; @@ -359,7 +361,7 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver) { // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { - if (!(pReceiver->start)) { + if (!snapshotReceiverIsStart(pReceiver)) { // start snapshotReceiverDoStart(pReceiver); pReceiver->start = true; From 29b97fa0ad843c12f3a5d84cf1966cb4c1398e10 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 19:53:07 +0800 Subject: [PATCH 082/132] refactor(sync): syncNodeOnAppendEntriesReplySnapshotCb --- include/libs/sync/syncTools.h | 5 +- source/libs/sync/inc/syncSnapshot.h | 6 +- source/libs/sync/src/syncAppendEntriesReply.c | 61 ++++++++----------- source/libs/sync/src/syncMessage.c | 1 + source/libs/sync/src/syncSnapshot.c | 60 +++++++++--------- source/libs/sync/test/syncSnapshotRspTest.cpp | 1 + 6 files changed, 62 insertions(+), 72 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index dcbb76fe49..27437ae3c2 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -398,7 +398,7 @@ typedef struct SyncSnapshotSend { SyncTerm term; SyncIndex lastIndex; // lastIndex of snapshot SyncTerm lastTerm; // lastTerm of snapshot - SyncTerm privateTerm; + SyncTerm privateTerm; int32_t seq; uint32_t dataLen; char data[]; @@ -433,8 +433,9 @@ typedef struct SyncSnapshotRsp { SyncTerm term; SyncIndex lastIndex; SyncTerm lastTerm; - SyncTerm privateTerm; + SyncTerm privateTerm; int32_t ack; + int32_t code; } SyncSnapshotRsp; SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 63e69f77ca..b16e47b51e 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -48,7 +48,7 @@ typedef struct SSyncSnapshotSender { int32_t replicaIndex; SyncTerm term; SyncTerm privateTerm; - bool apply; + bool finish; } SSyncSnapshotSender; SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); @@ -75,9 +75,9 @@ typedef struct SSyncSnapshotReceiver { SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index e6bef88a4c..a14ce3607f 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -101,7 +101,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId, + ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); // if already drop replica, do not process @@ -145,7 +146,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pMsg->success) { // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); - sTrace("update next index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); + sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); @@ -157,49 +158,35 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); - sTrace("begin to update next index:%ld, success:%d", nextIndex, pMsg->success); + sTrace("update next not match, begin, index:%ld, success:%d", nextIndex, pMsg->success); // notice! int64, uint64 if (nextIndex > SYNC_INDEX_BEGIN) { --nextIndex; - // has snapshot - if (syncNodeHasSnapshot(ths)) { - // get sender - SSyncSnapshotSender* pSender = NULL; - for (int i = 0; i < ths->replicaNum; ++i) { - if (syncUtilSameId(&(pMsg->srcId), &((ths->replicasId)[i]))) { - pSender = (ths->senders)[i]; - } - } - ASSERT(pSender != NULL); + // get sender + SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); + ASSERT(pSender != NULL); + bool hasSnapshot = syncNodeHasSnapshot(ths); + SSnapshot snapshot; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot); - SyncIndex sentryIndex; - if (pSender->start && pSender->term == ths->pRaftStore->currentTerm) { - // already start - sentryIndex = pSender->snapshot.lastApplyIndex; - sTrace( - "sending snapshot already start: pSender->term:%lu, ths->pRaftStore->currentTerm:%lu, " - "pSender->privateTerm:%lu", - pSender->term, ths->pRaftStore->currentTerm, pSender->privateTerm); + // start sending snapshot first time + // start here, stop by receiver + if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && + pMsg->privateTerm < pSender->privateTerm) { + snapshotSenderStart(pSender); - } else { - if (pMsg->privateTerm >= pSender->privateTerm) { - // donot start again - sentryIndex = pSender->snapshot.lastApplyIndex; + char* s = snapshotSender2Str(pSender); + sInfo("snapshot send, start sender first time, sender:%s", s); + taosMemoryFree(s); + } - } else { - // start first time - snapshotSenderStart(pSender); - pSender->start = true; - sentryIndex = pSender->snapshot.lastApplyIndex; - } - } + SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; - // update nextIndex to sentryIndex + 1 - if (nextIndex <= sentryIndex) { - nextIndex = sentryIndex + 1; - } + // update nextIndex to sentryIndex + if (nextIndex <= sentryIndex) { + nextIndex = sentryIndex; } } else { @@ -207,7 +194,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); - sTrace("update next index:%ld, success:%d", nextIndex, pMsg->success); + sTrace("update next not match, end, index:%ld, success:%d", nextIndex, pMsg->success); } syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 221a8e1af2..056c2d5c5b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1991,6 +1991,7 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); + cJSON_AddNumberToObject(pRoot, "code", pMsg->code); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 1d52d9be33..6e546ecedc 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -20,8 +20,9 @@ #include "syncUtil.h" #include "wal.h" -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver); +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm); +//---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); @@ -44,7 +45,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->term = pSyncNode->pRaftStore->currentTerm; pSender->privateTerm = taosGetTimestampMs() + 100; pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot)); - pSender->apply = false; + pSender->finish = false; } else { sError("snapshotSenderCreate cannot create sender"); } @@ -86,7 +87,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS; pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; ++(pSender->privateTerm); - pSender->apply = false; + pSender->finish = false; pSender->start = true; // build begin msg @@ -299,7 +300,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - cJSON_AddNumberToObject(pRoot, "apply", pSender->apply); + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); } cJSON *pJson = cJSON_CreateObject(); @@ -349,8 +350,9 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } // begin receive snapshot msg (current term, seq begin) -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver) { +static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; ASSERT(pReceiver->pWriter == NULL); @@ -360,10 +362,10 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver) { // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again -void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { +void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) { if (!snapshotReceiverIsStart(pReceiver)) { // start - snapshotReceiverDoStart(pReceiver); + snapshotReceiverDoStart(pReceiver, privateTerm); pReceiver->start = true; } else { @@ -376,7 +378,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { pReceiver->pWriter = NULL; // start again - snapshotReceiverDoStart(pReceiver); + snapshotReceiverDoStart(pReceiver, privateTerm); pReceiver->start = true; ASSERT(0); @@ -387,7 +389,7 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver) { taosMemoryFree(s); } -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { if (pReceiver->pWriter != NULL) { int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); @@ -396,7 +398,10 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; - ++(pReceiver->privateTerm); + + if (apply) { + ++(pReceiver->privateTerm); + } char *s = snapshotReceiver2Str(pReceiver); sInfo("snapshotReceiverStop %s", s); @@ -441,15 +446,15 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // get receiver SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; bool needRsp = false; + int32_t writeCode = 0; // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { // begin - snapshotReceiverStart(pReceiver); + snapshotReceiverStart(pReceiver, pMsg->privateTerm); pReceiver->ack = pMsg->seq; - pReceiver->privateTerm = pMsg->privateTerm; needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); @@ -459,7 +464,9 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // end, finish FSM - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + writeCode = pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + ASSERT(writeCode == 0); + pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); @@ -472,11 +479,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); taosMemoryFree(logSimpleStr); - // walRestoreFromSnapshot(pSyncNode->pWal, pMsg->lastIndex); - // sInfo("walRestoreFromSnapshot lastIndex:%ld", pMsg->lastIndex); - pReceiver->pWriter = NULL; - snapshotReceiverStop(pReceiver); + snapshotReceiverStop(pReceiver, true); pReceiver->ack = pMsg->seq; needRsp = true; @@ -487,7 +491,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); - snapshotReceiverStop(pReceiver); + snapshotReceiverStop(pReceiver, false); needRsp = false; char *msgStr = syncSnapshotSend2Str(pMsg); @@ -499,7 +503,9 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering if (pMsg->seq == pReceiver->ack + 1) { - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + writeCode = + pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); + ASSERT(writeCode == 0); pReceiver->ack = pMsg->seq; } needRsp = true; @@ -521,6 +527,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; pRspMsg->ack = pReceiver->ack; + pRspMsg->code = writeCode; pRspMsg->privateTerm = pReceiver->privateTerm; SRpcMsg rpcMsg; @@ -541,12 +548,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { // get sender - SSyncSnapshotSender *pSender = NULL; - for (int i = 0; i < pSyncNode->replicaNum; ++i) { - if (syncUtilSameId(&(pMsg->srcId), &((pSyncNode->replicasId)[i]))) { - pSender = (pSyncNode->senders)[i]; - } - } + SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); ASSERT(pSender != NULL); // state, term, seq/ack @@ -554,12 +556,8 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { // receiver ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - pSender->apply = true; + pSender->finish = true; snapshotSenderStop(pSender); - - // update nextIndex private term - // syncIndexMgrSetTerm(pSyncNode->pNextIndex, &(pMsg->srcId), pSender->privateTerm); - return 0; } @@ -569,8 +567,10 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { pSender->ack = pMsg->ack; (pSender->seq)++; snapshotSend(pSender); + } else if (pMsg->ack == pSender->seq - 1) { snapshotReSend(pSender); + } else { ASSERT(0); } diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index d624d45c94..f689d47aaf 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -25,6 +25,7 @@ SyncSnapshotRsp *createMsg() { pMsg->lastIndex = 22; pMsg->lastTerm = 33; pMsg->ack = 44; + pMsg->code = 55; return pMsg; } From 0147e4984894ff8d132790a46afee1b5b0546dda Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Jun 2022 21:01:01 +0800 Subject: [PATCH 083/132] fix:add test case --- source/client/test/smlTest.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 1358d45f30..46c451f164 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1272,14 +1272,40 @@ TEST(testCase, sml_params_Test) { }; TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); ASSERT_EQ(taos_errno(res), TSDB_CODE_PAR_DB_NOT_SPECIFIED); - taos_free_result(pRes); + taos_free_result(res); pRes = taos_query(taos, "use param"); - taos_free_result(pRes); + taos_free_result(res); res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); ASSERT_EQ(taos_errno(res), TSDB_CODE_SML_INVALID_DB_CONF); + taos_free_result(res); +} + +TEST(testCase, sml_16384_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists d16384 schemaless 1"); taos_free_result(pRes); + + const char *sql[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=t,c1=127i8 1626006833639000000", + }; + + pRes = taos_query(taos, "use d16384"); + taos_free_result(pRes); + + TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + const char *sql1[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + }; + TAOS_RES* res1 = taos_schemaless_insert(taos, (char**)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(res1), 0); + taos_free_result(res1); } TEST(testCase, sml_oom_Test) { From 98a2451c15abea6c34f2a2eb51b6f57bd3a348ce Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 21:55:36 +0800 Subject: [PATCH 084/132] refactor(sync): sync snapshot --- source/libs/sync/src/syncAppendEntries.c | 23 +++++++++++-------- source/libs/sync/src/syncAppendEntriesReply.c | 2 +- source/libs/sync/test/syncTestTool.cpp | 14 +++++++++++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 6f725e123e..7f654975bb 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -469,7 +469,7 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sInfo("syncNodeMakeLogSame, from %ld to %ld", delBegin, delEnd); + sInfo("sync event log truncate, from %ld to %ld", delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; @@ -556,8 +556,6 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } ASSERT(pMsg->dataLen >= 0); - bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); - // candidate to follower // // operation: @@ -582,7 +580,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // I have snapshot, have log, log <= snapshot, preIndex > myLastIndex // // condition3: - // I have snapshot, preIndex <= snapshot.lastApplyIndex + // I have snapshot, preIndex < snapshot.lastApplyIndex + // + // condition4: + // I have snapshot, preIndex == snapshot.lastApplyIndex, no data // // operation: // match snapshot.lastApplyIndex - 1; @@ -598,15 +599,16 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && (pMsg->prevLogIndex > myLastIndex); - bool condition3 = condition0 && (pMsg->prevLogIndex <= snapshot.lastApplyIndex); - bool condition = condition1 || condition2 || condition3; + bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex); + bool condition4 = condition0 && (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->dataLen == 0); + bool condition = condition1 || condition2 || condition3 || condition4; if (condition) { sTrace( "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " - "condition1:%d, condition2:%d, condition3:%d", + "condition1:%d, condition2:%d, condition3:%d, condition4:%d", myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), - ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3); + ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -615,7 +617,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->term = ths->pRaftStore->currentTerm; pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; - pReply->matchIndex = snapshot.lastApplyIndex - 1; + pReply->matchIndex = snapshot.lastApplyIndex; // send response SRpcMsg rpcMsg; @@ -627,6 +629,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } } while (0); + // calculate logOK here, before will coredump, due to fake match + bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); + // not match // // condition1: diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index a14ce3607f..947116f965 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -178,7 +178,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries snapshotSenderStart(pSender); char* s = snapshotSender2Str(pSender); - sInfo("snapshot send, start sender first time, sender:%s", s); + sInfo("snapshot send start sender first time, sender:%s", s); taosMemoryFree(s); } diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index c23d06ee38..782baf3c97 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -205,6 +205,19 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* SSyncCfg* pCfg = &syncInfo.syncCfg; +#if 0 + { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } +#endif + if (isStandBy) { pCfg->myIndex = 0; pCfg->replicaNum = 1; @@ -222,6 +235,7 @@ int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* } } + int64_t rid = syncOpen(&syncInfo); assert(rid > 0); From e86ee68f8b85f84ac1c65ceb0fd030fddb5855a0 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 22:43:58 +0800 Subject: [PATCH 085/132] refactor(sync): sync snapshot --- source/libs/sync/src/syncAppendEntries.c | 13 +++++-------- source/libs/sync/src/syncAppendEntriesReply.c | 3 ++- source/libs/sync/src/syncMain.c | 4 ++-- source/libs/sync/src/syncRequestVote.c | 3 ++- source/libs/sync/src/syncRequestVoteReply.c | 3 ++- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7f654975bb..170b50cfed 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -534,14 +534,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->pRaftStore->currentTerm); syncAppendEntriesLog2(logBuf, pMsg); - // if I am standby, to be added into a raft group, I should process SyncAppendEntries msg - /* - // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { - sInfo("recv SyncAppendEntries maybe replica already dropped"); - return ret; - } - */ + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncAppendEntries maybe replica already dropped"); + return ret; + } // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 947116f965..150b6f2410 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -17,6 +17,7 @@ #include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncSnapshot.h" @@ -106,7 +107,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncAppendEntriesReplyLog2(logBuf, pMsg); // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); return ret; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index acbe658914..6287172e76 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1039,10 +1039,10 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " "electTimerLogicClock:%lu, " "electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, replicaNum:%d", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftCfg->isStandBy, pSyncNode->electTimerLogicClock, - pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, pSyncNode->replicaNum); return s; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index d939244906..9ed7f00982 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -15,6 +15,7 @@ #include "syncRequestVote.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -104,7 +105,7 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { syncRequestVoteLog2(logBuf, pMsg); // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { sInfo("recv SyncRequestVote maybe replica already dropped"); return ret; } diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 9cc905d847..5d041cefcd 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -15,6 +15,7 @@ #include "syncRequestVoteReply.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" @@ -102,7 +103,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl syncRequestVoteReplyLog2(logBuf, pMsg); // if already drop replica, do not process - if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); return ret; } From 61cc736ca022e3549d9af12c9f2bb34a7f4f78d5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 23:08:40 +0800 Subject: [PATCH 086/132] fix(sync): create db core due to snapshot --- source/libs/sync/src/syncSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 6e546ecedc..a4dc89ad78 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -320,7 +320,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t repl bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); - SSyncSnapshotReceiver *pReceiver; + SSyncSnapshotReceiver *pReceiver = NULL; if (condition) { pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); ASSERT(pReceiver != NULL); From e0706577784bcd593ea1b709ffe564caa9b2616a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 8 Jun 2022 23:20:07 +0800 Subject: [PATCH 087/132] refactor(sync): add syncGetSnapshotMeta --- include/libs/sync/sync.h | 6 ++++++ source/libs/sync/inc/syncInt.h | 2 ++ source/libs/sync/src/syncMain.c | 15 +++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 166a363ada..10ece0b219 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -94,6 +94,10 @@ typedef struct SSnapshot { SyncIndex lastConfigIndex; } SSnapshot; +typedef struct SSnapshotMeta { + SyncIndex lastConfigIndex; +} SSnapshotMeta; + typedef struct SSyncFSM { void* data; @@ -191,6 +195,8 @@ bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); +int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); + // to be moved to static void syncStartNormal(int64_t rid); void syncStartStandBy(int64_t rid); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index bd44718c58..dba2266285 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -157,6 +157,8 @@ typedef struct SSyncNode { SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; + SSnapshotMeta sMeta; + } SSyncNode; // open/close -------------- diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 6287172e76..d549e3bd9d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -214,6 +214,18 @@ bool syncIsRestoreFinish(int64_t rid) { return b; } +int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return -1; + } + assert(rid == pSyncNode->rid); + *sMeta = pSyncNode->sMeta; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +} + const char* syncGetMyRoleStr(int64_t rid) { const char* s = syncUtilState2String(syncGetMyRole(rid)); return s; @@ -604,6 +616,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // start raft // syncNodeBecomeFollower(pSyncNode); + // snapshot meta + pSyncNode->sMeta.lastConfigIndex = -1; + return pSyncNode; } From 23c0bdf0529e9711b21886d6e92b171205bc6364 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 10:57:32 +0800 Subject: [PATCH 088/132] enh(query): add hll function distributed splitting --- include/libs/function/functionMgt.h | 2 ++ source/libs/function/src/builtins.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 77376a05d9..d77b914fae 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -129,6 +129,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_SPREAD_MERGE, FUNCTION_TYPE_HISTOGRAM_PARTIAL, FUNCTION_TYPE_HISTOGRAM_MERGE, + FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, + FUNCTION_TYPE_HYPERLOGLOG_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index eca4587c03..bc5f56784a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1480,6 +1480,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = hllFunction, .finalizeFunc = hllFinalize }, + { + .name = "_hyperloglog_partial", + .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateHLL, + .getEnvFunc = getHLLFuncEnv, + .initFunc = functionSetup, + .processFunc = hllFunction, + .finalizeFunc = hllFinalize + }, + { + .name = "_hyperloglog_merge", + .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateHLL, + .getEnvFunc = getHLLFuncEnv, + .initFunc = functionSetup, + .processFunc = hllFunction, + .finalizeFunc = hllFinalize + }, { .name = "diff", .type = FUNCTION_TYPE_DIFF, From 89e2c2d47a349a2ced330342e782eb3eead41899 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 11:00:28 +0800 Subject: [PATCH 089/132] fix(sync): update sender when become leader --- source/libs/sync/inc/syncInt.h | 2 ++ source/libs/sync/src/syncMain.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index dba2266285..10218f69e6 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -58,6 +58,8 @@ typedef struct SSyncRespMgr SSyncRespMgr; typedef struct SSyncSnapshotSender SSyncSnapshotSender; typedef struct SSyncSnapshotReceiver SSyncSnapshotReceiver; +extern bool gRaftDetailLog; + typedef struct SSyncNode { // init by SSyncInfo SyncGroupId vgId; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d549e3bd9d..fe4ff61aad 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,6 +35,8 @@ #include "syncVoteMgr.h" #include "tref.h" +bool gRaftDetailLog = false; + static int32_t tsNodeRefId = -1; // ------ local funciton --------- @@ -1203,6 +1205,16 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID; } + // update sender private term + SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); + ASSERT(pMySender != NULL); + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { + pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + } + } + (pMySender->privateTerm) += 100; + // stop elect timer syncNodeStopElectTimer(pSyncNode); From ac5f09251d3de44154c90523d44d8cc0be73647c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 9 Jun 2022 11:09:23 +0800 Subject: [PATCH 090/132] fix:err in tag scan --- source/libs/executor/src/scanoperator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cc76cc29e4..b6ec37982b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1756,6 +1756,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi ; pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; + pInfo->pFilterNode = pPhyNode->node.pConditions; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; pOperator->blocking = false; From 94720bbbb1990c80ad6de2cab6604753c9430201 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 11:15:50 +0800 Subject: [PATCH 091/132] fix: should not execute trans if not master --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 42fa7b718e..53943b61b0 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -71,7 +71,7 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7a32fac7c3..aa4950bfba 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1085,6 +1085,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } if (code == 0) { + if (!pMnode->deploy && !mndIsMaster(pMnode)) break; + pTrans->code = 0; pTrans->redoActionPos++; mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), From d81f5e9f77704d26f8c1e4b8c8a63a171c60f440 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 11:18:48 +0800 Subject: [PATCH 092/132] fix(sync): add detail log flag --- source/libs/sync/src/syncAppendEntries.c | 31 +++++--- source/libs/sync/src/syncIndexMgr.c | 10 ++- source/libs/sync/src/syncMain.c | 10 ++- source/libs/sync/src/syncMessage.c | 96 +++++++++++++++--------- source/libs/sync/src/syncRaftLog.c | 8 +- 5 files changed, 97 insertions(+), 58 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 170b50cfed..3c558b60c8 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -497,30 +497,39 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { // prevLogIndex == -1 static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { - sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); + if (gRaftDetailLog) { + sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); + } return true; } SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, - myLastIndex); + if (gRaftDetailLog) { + sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, + myLastIndex); + } return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { - sTrace( - "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + if (gRaftDetailLog) { + sTrace( + "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + } return true; } - sTrace( - "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + if (gRaftDetailLog) { + sTrace( + "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " + "myPreLogTerm:%lu", + pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); + } + return false; } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 06906195dc..ecc1c8f1e2 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -119,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -146,9 +146,11 @@ void syncIndexMgrLog(SSyncIndexMgr *pObj) { } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { - char *serialized = syncIndexMgr2Str(pObj); - sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char *serialized = syncIndexMgr2Str(pObj); + sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fe4ff61aad..fedbb5c1b4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -852,10 +852,12 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); if (pSyncNode->FpSendMsg != NULL) { - char* JsonStr = syncRpcMsg2Str(pMsg); - syncUtilJson2Line(JsonStr); - sTrace("sync send msg, vgId:%d, type:%d, msg:%s", pSyncNode->vgId, pMsg->msgType, JsonStr); - taosMemoryFree(JsonStr); + if (gRaftDetailLog) { + char* JsonStr = syncRpcMsg2Str(pMsg); + syncUtilJson2Line(JsonStr); + sTrace("sync send msg, vgId:%d, type:%d, msg:%s", pSyncNode->vgId, pMsg->msgType, JsonStr); + taosMemoryFree(JsonStr); + } // htonl syncUtilMsgHtoN(pMsg->pCont); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 056c2d5c5b..af04a0f649 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -145,9 +145,11 @@ void syncRpcMsgLog(SRpcMsg* pMsg) { } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncTimeout---- @@ -274,9 +276,11 @@ void syncTimeoutLog(const SyncTimeout* pMsg) { } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { - char* serialized = syncTimeout2Str(pMsg); - sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncTimeout2Str(pMsg); + sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncPing---- @@ -534,9 +538,11 @@ void syncPingLog(const SyncPing* pMsg) { } void syncPingLog2(char* s, const SyncPing* pMsg) { - char* serialized = syncPing2Str(pMsg); - sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncPing2Str(pMsg); + sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncPingReply---- @@ -794,9 +800,11 @@ void syncPingReplyLog(const SyncPingReply* pMsg) { } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { - char* serialized = syncPingReply2Str(pMsg); - sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncPingReply2Str(pMsg); + sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncClientRequest---- @@ -935,9 +943,11 @@ void syncClientRequestLog(const SyncClientRequest* pMsg) { } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncRequestVote---- @@ -1084,9 +1094,11 @@ void syncRequestVoteLog(const SyncRequestVote* pMsg) { } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { - char* serialized = syncRequestVote2Str(pMsg); - sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRequestVote2Str(pMsg); + sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncRequestVoteReply---- @@ -1230,9 +1242,11 @@ void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { - char* serialized = syncRequestVoteReply2Str(pMsg); - sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncRequestVoteReply2Str(pMsg); + sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncAppendEntries---- @@ -1399,9 +1413,11 @@ void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { - char* serialized = syncAppendEntries2Str(pMsg); - sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncAppendEntries2Str(pMsg); + sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncAppendEntriesReply---- @@ -1551,9 +1567,11 @@ void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { - char* serialized = syncAppendEntriesReply2Str(pMsg); - sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncAppendEntriesReply2Str(pMsg); + sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // ---- message process SyncApplyMsg---- @@ -1702,9 +1720,11 @@ void syncApplyMsgLog(const SyncApplyMsg* pMsg) { } void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) { - char* serialized = syncApplyMsg2Str(pMsg); - sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncApplyMsg2Str(pMsg); + sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // --------------------------------------------- @@ -1870,9 +1890,11 @@ void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { } void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } // --------------------------------------------- @@ -2028,7 +2050,9 @@ void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg) { } void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) { - char* serialized = syncSnapshotRsp2Str(pMsg); - sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = syncSnapshotRsp2Str(pMsg); + sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d7a8828d94..466863c0db 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -583,7 +583,9 @@ void logStoreSimpleLog(SSyncLogStore* pLogStore) { } void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); + if (gRaftDetailLog) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } } From d1a0ab2445350f2dc12e9f1e9c53494a34b7810b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 11:36:28 +0800 Subject: [PATCH 093/132] add hll partial/merge translate functions --- source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 25 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 3a9b94d71e..9de1580c73 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -127,6 +127,7 @@ int32_t getHistogramInfoSize(); bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t hllFunction(SqlFunctionCtx* pCtx); int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t getHLLInfoSize(); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bc5f56784a..2158b8bca9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -313,6 +313,7 @@ static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int static int32_t translateApercentilePartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateApercentileImpl(pFunc, pErrBuf, len, true); } + static int32_t translateApercentileMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateApercentileImpl(pFunc, pErrBuf, len, false); } @@ -401,6 +402,7 @@ static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t static int32_t translateSpreadPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateSpreadImpl(pFunc, pErrBuf, len, true); } + static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateSpreadImpl(pFunc, pErrBuf, len, false); } @@ -551,6 +553,7 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32 static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateHistogramImpl(pFunc, pErrBuf, len, true); } + static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return translateHistogramImpl(pFunc, pErrBuf, len, false); } @@ -564,6 +567,28 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return TSDB_CODE_SUCCESS; } +static int32_t translateHLLImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (isPartial) { + pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + } else { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateHLLPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateHLLImpl(pFunc, pErrBuf, len, true); +} + +static int32_t translateHLLMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateHLLImpl(pFunc, pErrBuf, len, false); +} + static bool validateStateOper(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e0302f9598..683d7e6342 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3428,6 +3428,10 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return 1; } +int32_t getHLLInfoSize() { + return (int32_t)sizeof(SHLLInfo); +} + bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SHLLInfo); return true; From d95b7a3dc19333d4e41fd969e424647a174b466d Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 9 Jun 2022 11:48:41 +0800 Subject: [PATCH 094/132] fix(os): add win32 taosd case --- source/libs/scalar/src/sclvector.c | 2 +- source/os/src/osTime.c | 427 ++++++++---------- tests/pytest/util/dnodes.py | 3 + tests/system-test/0-others/taosdlog.py | 2 +- tests/system-test/0-others/udf_cluster.py | 2 +- tests/system-test/0-others/udf_create.py | 24 +- tests/system-test/0-others/user_control.py | 3 +- .../1-insert/insertWithMoreVgroup.py | 2 +- .../1-insert/mutipythonnodebugtaosd.py | 2 +- .../1-insert/test_stmt_insert_query_ex.py | 2 +- .../1-insert/test_stmt_muti_insert_query.py | 2 +- .../1-insert/test_stmt_set_tbname_tag.py | 2 +- tests/system-test/2-query/histogram.py | 2 +- tests/system-test/2-query/timezone.py | 24 +- tests/system-test/7-tmq/basic5.py | 23 +- tests/system-test/7-tmq/db.py | 2 +- tests/system-test/7-tmq/schema.py | 2 +- tests/system-test/7-tmq/subscribeDb.py | 2 +- tests/system-test/7-tmq/subscribeDb0.py | 2 +- tests/system-test/7-tmq/subscribeDb1.py | 2 +- tests/system-test/7-tmq/subscribeStb.py | 2 +- tests/system-test/7-tmq/subscribeStb0.py | 2 +- tests/system-test/7-tmq/subscribeStb1.py | 2 +- tests/system-test/7-tmq/subscribeStb2.py | 2 +- tests/system-test/7-tmq/subscribeStb3.py | 2 +- tests/system-test/7-tmq/subscribeStb4.py | 2 +- tests/system-test/7-tmq/tmqDnode.py | 2 +- tests/system-test/7-tmq/tmqModule.py | 2 +- tests/system-test/99-TDcase/TD-15517.py | 2 +- tests/system-test/99-TDcase/TD-15554.py | 2 +- tests/system-test/99-TDcase/TD-15557.py | 2 +- tests/system-test/99-TDcase/TD-15563.py | 2 +- tests/system-test/99-TDcase/TD-16025.py | 2 +- tests/system-test/test.py | 11 +- 34 files changed, 272 insertions(+), 297 deletions(-) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 7d96a89e85..bff49c2ca9 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -348,7 +348,7 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn int32_t outputMaxLen = (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; char* t = taosMemoryCalloc(1, outputMaxLen); - /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), outputMaxLen, &len); + /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len); varDataSetLen(t, len); colDataAppend(pOut->columnData, rowIndex, t, false); diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 5d6b4e7c2b..7e6e508817 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -28,92 +28,48 @@ #ifdef WINDOWS -#include -#include +#include #include +#include //#define TM_YEAR_BASE 1970 //origin -#define TM_YEAR_BASE 1900 //slguan +#define TM_YEAR_BASE 1900 // slguan /* -* We do not implement alternate representations. However, we always -* check whether a given modifier is allowed for a certain conversion. -*/ -#define ALT_E 0x01 -#define ALT_O 0x02 -#define LEGAL_ALT(x) { if (alt_format & ~(x)) return (0); } + * We do not implement alternate representations. However, we always + * check whether a given modifier is allowed for a certain conversion. + */ +#define ALT_E 0x01 +#define ALT_O 0x02 +#define LEGAL_ALT(x) \ + { \ + if (alt_format & ~(x)) return (0); \ + } +static int conv_num(const char **buf, int *dest, int llim, int ulim) { + int result = 0; -static int conv_num(const char **buf, int *dest, int llim, int ulim) -{ - int result = 0; + /* The limit also determines the number of valid digits. */ + int rulim = ulim; - /* The limit also determines the number of valid digits. */ - int rulim = ulim; + if (**buf < '0' || **buf > '9') return (0); - if (**buf < '0' || **buf > '9') - return (0); + do { + result *= 10; + result += *(*buf)++ - '0'; + rulim /= 10; + } while ((result * 10 <= ulim) && rulim && **buf >= '0' && **buf <= '9'); - do { - result *= 10; - result += *(*buf)++ - '0'; - rulim /= 10; - } while ((result * 10 <= ulim) && rulim && **buf >= '0' && **buf <= '9'); + if (result < llim || result > ulim) return (0); - if (result < llim || result > ulim) - return (0); - - *dest = result; - return (1); + *dest = result; + return (1); } -static const char *day[7] = { - "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", - "Friday", "Saturday" -}; -static const char *abday[7] = { - "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" -}; -static const char *mon[12] = { - "January", "February", "March", "April", "May", "June", "July", - "August", "September", "October", "November", "December" -}; -static const char *abmon[12] = { - "Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" -}; -static const char *am_pm[2] = { - "AM", "PM" -}; - -#define BILLION (1E9) - -static BOOL g_first_time = 1; -static LARGE_INTEGER g_counts_per_sec; - -int clock_gettime(int dummy, struct timespec *ct) -{ - LARGE_INTEGER count; - - if (g_first_time) - { - g_first_time = 0; - - if (0 == QueryPerformanceFrequency(&g_counts_per_sec)) - { - g_counts_per_sec.QuadPart = 0; - } - } - - if ((NULL == ct) || (g_counts_per_sec.QuadPart <= 0) || - (0 == QueryPerformanceCounter(&count))) - { - return -1; - } - - ct->tv_sec = count.QuadPart / g_counts_per_sec.QuadPart; - ct->tv_nsec = ((count.QuadPart % g_counts_per_sec.QuadPart) * BILLION) / g_counts_per_sec.QuadPart; - - return 0; -} +static const char *day[7] = {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}; +static const char *abday[7] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; +static const char *mon[12] = {"January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December"}; +static const char *abmon[12] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; +static const char *am_pm[2] = {"AM", "PM"}; #else #include @@ -121,301 +77,265 @@ int clock_gettime(int dummy, struct timespec *ct) char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { #ifdef WINDOWS - char c; - const char *bp; - size_t len = 0; - int alt_format, i, split_year = 0; + char c; + const char *bp; + size_t len = 0; + int alt_format, i, split_year = 0; - bp = buf; + bp = buf; - while ((c = *fmt) != '\0') { - /* Clear `alternate' modifier prior to new conversion. */ - alt_format = 0; + while ((c = *fmt) != '\0') { + /* Clear `alternate' modifier prior to new conversion. */ + alt_format = 0; - /* Eat up white-space. */ - if (isspace(c)) { - while (isspace(*bp)) - bp++; + /* Eat up white-space. */ + if (isspace(c)) { + while (isspace(*bp)) bp++; - fmt++; - continue; - } + fmt++; + continue; + } - if ((c = *fmt++) != '%') - goto literal; + if ((c = *fmt++) != '%') goto literal; - - again: switch (c = *fmt++) { - case '%': /* "%%" is converted to "%". */ - literal : - if (c != *bp++) - return (0); + again: + switch (c = *fmt++) { + case '%': /* "%%" is converted to "%". */ + literal: + if (c != *bp++) return (0); break; /* - * "Alternative" modifiers. Just set the appropriate flag - * and start over again. - */ - case 'E': /* "%E?" alternative conversion modifier. */ + * "Alternative" modifiers. Just set the appropriate flag + * and start over again. + */ + case 'E': /* "%E?" alternative conversion modifier. */ LEGAL_ALT(0); alt_format |= ALT_E; goto again; - case 'O': /* "%O?" alternative conversion modifier. */ + case 'O': /* "%O?" alternative conversion modifier. */ LEGAL_ALT(0); alt_format |= ALT_O; goto again; /* - * "Complex" conversion rules, implemented through recursion. - */ - case 'c': /* Date and time, using the locale's format. */ + * "Complex" conversion rules, implemented through recursion. + */ + case 'c': /* Date and time, using the locale's format. */ LEGAL_ALT(ALT_E); - if (!(bp = taosStrpTime(bp, "%x %X", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%x %X", tm))) return (0); break; - case 'D': /* The date as "%m/%d/%y". */ + case 'D': /* The date as "%m/%d/%y". */ LEGAL_ALT(0); - if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) return (0); break; - case 'R': /* The time as "%H:%M". */ + case 'R': /* The time as "%H:%M". */ LEGAL_ALT(0); - if (!(bp = taosStrpTime(bp, "%H:%M", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%H:%M", tm))) return (0); break; - case 'r': /* The time in 12-hour clock representation. */ + case 'r': /* The time in 12-hour clock representation. */ LEGAL_ALT(0); - if (!(bp = taosStrpTime(bp, "%I:%M:%S %p", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%I:%M:%S %p", tm))) return (0); break; - case 'T': /* The time as "%H:%M:%S". */ + case 'T': /* The time as "%H:%M:%S". */ LEGAL_ALT(0); - if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) return (0); break; - case 'X': /* The time, using the locale's format. */ + case 'X': /* The time, using the locale's format. */ LEGAL_ALT(ALT_E); - if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%H:%M:%S", tm))) return (0); break; - case 'x': /* The date, using the locale's format. */ + case 'x': /* The date, using the locale's format. */ LEGAL_ALT(ALT_E); - if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) - return (0); + if (!(bp = taosStrpTime(bp, "%m/%d/%y", tm))) return (0); break; /* - * "Elementary" conversion rules. - */ - case 'A': /* The day of week, using the locale's form. */ - case 'a': + * "Elementary" conversion rules. + */ + case 'A': /* The day of week, using the locale's form. */ + case 'a': LEGAL_ALT(0); for (i = 0; i < 7; i++) { - /* Full name. */ - len = strlen(day[i]); - if (strncmp(day[i], bp, len) == 0) - break; + /* Full name. */ + len = strlen(day[i]); + if (strncmp(day[i], bp, len) == 0) break; - /* Abbreviated name. */ - len = strlen(abday[i]); - if (strncmp(abday[i], bp, len) == 0) - break; + /* Abbreviated name. */ + len = strlen(abday[i]); + if (strncmp(abday[i], bp, len) == 0) break; } /* Nothing matched. */ - if (i == 7) - return (0); + if (i == 7) return (0); tm->tm_wday = i; bp += len; break; - case 'B': /* The month, using the locale's form. */ - case 'b': - case 'h': + case 'B': /* The month, using the locale's form. */ + case 'b': + case 'h': LEGAL_ALT(0); for (i = 0; i < 12; i++) { - /* Full name. */ - len = strlen(mon[i]); - if (strncmp(mon[i], bp, len) == 0) - break; + /* Full name. */ + len = strlen(mon[i]); + if (strncmp(mon[i], bp, len) == 0) break; - /* Abbreviated name. */ - len = strlen(abmon[i]); - if (strncmp(abmon[i], bp, len) == 0) - break; + /* Abbreviated name. */ + len = strlen(abmon[i]); + if (strncmp(abmon[i], bp, len) == 0) break; } /* Nothing matched. */ - if (i == 12) - return (0); + if (i == 12) return (0); tm->tm_mon = i; bp += len; break; - case 'C': /* The century number. */ + case 'C': /* The century number. */ LEGAL_ALT(ALT_E); - if (!(conv_num(&bp, &i, 0, 99))) - return (0); + if (!(conv_num(&bp, &i, 0, 99))) return (0); if (split_year) { - tm->tm_year = (tm->tm_year % 100) + (i * 100); - } - else { - tm->tm_year = i * 100; - split_year = 1; + tm->tm_year = (tm->tm_year % 100) + (i * 100); + } else { + tm->tm_year = i * 100; + split_year = 1; } break; - case 'd': /* The day of month. */ - case 'e': + case 'd': /* The day of month. */ + case 'e': LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_mday, 1, 31))) - return (0); + if (!(conv_num(&bp, &tm->tm_mday, 1, 31))) return (0); break; - case 'k': /* The hour (24-hour clock representation). */ + case 'k': /* The hour (24-hour clock representation). */ LEGAL_ALT(0); /* FALLTHROUGH */ - case 'H': + case 'H': LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_hour, 0, 23))) - return (0); + if (!(conv_num(&bp, &tm->tm_hour, 0, 23))) return (0); break; - case 'l': /* The hour (12-hour clock representation). */ + case 'l': /* The hour (12-hour clock representation). */ LEGAL_ALT(0); /* FALLTHROUGH */ - case 'I': + case 'I': LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_hour, 1, 12))) - return (0); - if (tm->tm_hour == 12) - tm->tm_hour = 0; + if (!(conv_num(&bp, &tm->tm_hour, 1, 12))) return (0); + if (tm->tm_hour == 12) tm->tm_hour = 0; break; - case 'j': /* The day of year. */ + case 'j': /* The day of year. */ LEGAL_ALT(0); - if (!(conv_num(&bp, &i, 1, 366))) - return (0); + if (!(conv_num(&bp, &i, 1, 366))) return (0); tm->tm_yday = i - 1; break; - case 'M': /* The minute. */ + case 'M': /* The minute. */ LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_min, 0, 59))) - return (0); + if (!(conv_num(&bp, &tm->tm_min, 0, 59))) return (0); break; - case 'm': /* The month. */ + case 'm': /* The month. */ LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &i, 1, 12))) - return (0); + if (!(conv_num(&bp, &i, 1, 12))) return (0); tm->tm_mon = i - 1; break; - case 'p': /* The locale's equivalent of AM/PM. */ + case 'p': /* The locale's equivalent of AM/PM. */ LEGAL_ALT(0); /* AM? */ if (strcmp(am_pm[0], bp) == 0) { - if (tm->tm_hour > 11) - return (0); + if (tm->tm_hour > 11) return (0); - bp += strlen(am_pm[0]); - break; + bp += strlen(am_pm[0]); + break; } /* PM? */ else if (strcmp(am_pm[1], bp) == 0) { - if (tm->tm_hour > 11) - return (0); + if (tm->tm_hour > 11) return (0); - tm->tm_hour += 12; - bp += strlen(am_pm[1]); - break; + tm->tm_hour += 12; + bp += strlen(am_pm[1]); + break; } /* Nothing matched. */ return (0); - case 'S': /* The seconds. */ + case 'S': /* The seconds. */ LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_sec, 0, 61))) - return (0); + if (!(conv_num(&bp, &tm->tm_sec, 0, 61))) return (0); break; - case 'U': /* The week of year, beginning on sunday. */ - case 'W': /* The week of year, beginning on monday. */ + case 'U': /* The week of year, beginning on sunday. */ + case 'W': /* The week of year, beginning on monday. */ LEGAL_ALT(ALT_O); /* - * XXX This is bogus, as we can not assume any valid - * information present in the tm structure at this - * point to calculate a real value, so just check the - * range for now. - */ - if (!(conv_num(&bp, &i, 0, 53))) - return (0); + * XXX This is bogus, as we can not assume any valid + * information present in the tm structure at this + * point to calculate a real value, so just check the + * range for now. + */ + if (!(conv_num(&bp, &i, 0, 53))) return (0); break; - case 'w': /* The day of week, beginning on sunday. */ + case 'w': /* The day of week, beginning on sunday. */ LEGAL_ALT(ALT_O); - if (!(conv_num(&bp, &tm->tm_wday, 0, 6))) - return (0); + if (!(conv_num(&bp, &tm->tm_wday, 0, 6))) return (0); break; - case 'Y': /* The year. */ + case 'Y': /* The year. */ LEGAL_ALT(ALT_E); - if (!(conv_num(&bp, &i, 0, 9999))) - return (0); + if (!(conv_num(&bp, &i, 0, 9999))) return (0); tm->tm_year = i - TM_YEAR_BASE; break; - case 'y': /* The year within 100 years of the epoch. */ + case 'y': /* The year within 100 years of the epoch. */ LEGAL_ALT(ALT_E | ALT_O); - if (!(conv_num(&bp, &i, 0, 99))) - return (0); + if (!(conv_num(&bp, &i, 0, 99))) return (0); if (split_year) { - tm->tm_year = ((tm->tm_year / 100) * 100) + i; - break; + tm->tm_year = ((tm->tm_year / 100) * 100) + i; + break; } split_year = 1; if (i <= 68) - tm->tm_year = i + 2000 - TM_YEAR_BASE; + tm->tm_year = i + 2000 - TM_YEAR_BASE; else - tm->tm_year = i + 1900 - TM_YEAR_BASE; + tm->tm_year = i + 1900 - TM_YEAR_BASE; break; /* - * Miscellaneous conversions. - */ - case 'n': /* Any kind of white-space. */ - case 't': + * Miscellaneous conversions. + */ + case 'n': /* Any kind of white-space. */ + case 't': LEGAL_ALT(0); - while (isspace(*bp)) - bp++; + while (isspace(*bp)) bp++; break; - - default: /* Unknown/unsupported conversion. */ + default: /* Unknown/unsupported conversion. */ return (0); } + } - - } - - /* LINTED functional specification */ - return ((char *)bp); + /* LINTED functional specification */ + return ((char *)bp); #else - return strptime(buf, fmt, tm); + return strptime(buf, fmt, tm); #endif } @@ -435,13 +355,9 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { #endif } -time_t taosTime(time_t *t) { - return time(t); -} +time_t taosTime(time_t *t) { return time(t); } -time_t taosMktime(struct tm *timep) { - return mktime(timep); -} +time_t taosMktime(struct tm *timep) { return mktime(timep); } struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { @@ -456,5 +372,36 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } +int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { +#ifdef WINDOWS + LARGE_INTEGER t; + FILETIME f; + static FILETIME ff; + static SYSTEMTIME ss; + static LARGE_INTEGER offset; -int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { return clock_gettime(clock_id, pTS); } \ No newline at end of file + ss.wYear = 1970; + ss.wMonth = 1; + ss.wDay = 1; + ss.wHour = 0; + ss.wMinute = 0; + ss.wSecond = 0; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + + GetSystemTimeAsFileTime(&f); + t.QuadPart = f.dwHighDateTime; + t.QuadPart <<= 32; + t.QuadPart |= f.dwLowDateTime; + + t.QuadPart -= offset.QuadPart; + pTS->tv_sec = t.QuadPart / 10000000; + pTS->tv_nsec = (t.QuadPart % 10000000)*100; + return (0); +#else + return clock_gettime(clock_id, pTS); +#endif +} \ No newline at end of file diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c2aed6555d..56989597ec 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -574,6 +574,9 @@ class TDDnodes: def stopAll(self): tdLog.info("stop all dnodes") + if (not self.dnodes[0].remoteIP == ""): + self.dnodes[0].remoteExec(self.dnodes[0].cfgDict, "for i in range(len(tdDnodes.dnodes)):\n tdDnodes.dnodes[i].running=1\ntdDnodes.stopAll()") + return for i in range(len(self.dnodes)): self.dnodes[i].stop() diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index f9f80bb910..f8898daf41 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -23,7 +23,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/0-others/udf_cluster.py b/tests/system-test/0-others/udf_cluster.py index de998e9087..c5c0c7b8f2 100644 --- a/tests/system-test/0-others/udf_cluster.py +++ b/tests/system-test/0-others/udf_cluster.py @@ -37,7 +37,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 0d24b09616..170d9b1421 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -9,8 +9,9 @@ from util.sql import * from util.cases import * from util.dnodes import * import subprocess -# import win32gui -# import threading +if (platform.system().lower() == 'windows'): + import win32gui +import threading class TDTestCase: @@ -535,17 +536,18 @@ class TDTestCase: return udf1_sqls ,udf2_sqls - # def checkRunTimeError(self): - # while 1: - # time.sleep(1) - # hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") - # if hwnd: - # os.system("TASKKILL /F /IM udfd.exe") + def checkRunTimeError(self): + if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""): + while 1: + time.sleep(1) + hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") + if hwnd: + os.system("TASKKILL /F /IM udfd.exe") def unexpected_create(self): - # if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""): - # checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True) - # checkErrorThread.start() + if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""): + checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True) + checkErrorThread.start() tdLog.info(" create function with out bufsize ") tdSql.query("drop function udf1 ") diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 3adc31cc39..4d59129b91 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -3,6 +3,7 @@ import taos import time import inspect import traceback +import socket from dataclasses import dataclass from util.log import * @@ -102,7 +103,7 @@ class TDconnect: def taos_connect( - host = "127.0.0.1", + host = socket.gethostname(), port = 6030, user = "root", passwd = "taosdata", diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index 8d2870fc2c..97220350e0 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -54,7 +54,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/1-insert/mutipythonnodebugtaosd.py b/tests/system-test/1-insert/mutipythonnodebugtaosd.py index 73d70b4348..3d6358f3ff 100644 --- a/tests/system-test/1-insert/mutipythonnodebugtaosd.py +++ b/tests/system-test/1-insert/mutipythonnodebugtaosd.py @@ -52,7 +52,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/1-insert/test_stmt_insert_query_ex.py b/tests/system-test/1-insert/test_stmt_insert_query_ex.py index c0836b79c8..6d49f6065a 100644 --- a/tests/system-test/1-insert/test_stmt_insert_query_ex.py +++ b/tests/system-test/1-insert/test_stmt_insert_query_ex.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/1-insert/test_stmt_muti_insert_query.py b/tests/system-test/1-insert/test_stmt_muti_insert_query.py index f838fd90f5..971be6c85f 100644 --- a/tests/system-test/1-insert/test_stmt_muti_insert_query.py +++ b/tests/system-test/1-insert/test_stmt_muti_insert_query.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py index fb9f455d4d..b540642847 100644 --- a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py +++ b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py @@ -49,7 +49,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index 2c203bdceb..c8952c4af5 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -400,7 +400,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index ff55ab31bf..b73ec2a94e 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -3,6 +3,8 @@ from util.log import * from util.sql import * from util.cases import * +import tzlocal +import platform import os @@ -15,16 +17,20 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method tdSql.prepare() # get system timezone - time_zone_arr = os.popen('timedatectl | grep zone').read( - ).strip().split(':') - if len(time_zone_arr) > 1: - time_zone = time_zone_arr[1].lstrip() - else: - # possibly in a docker container - time_zone_1 = os.popen('ls -l /etc/localtime|awk -F/ \'{print $(NF-1) "/" $NF}\'').read().strip() - time_zone_2 = os.popen('date "+(%Z, %z)"').read().strip() + if platform.system().lower() == 'windows': + time_zone_1 = tzlocal.get_localzone_name() + time_zone_2 = time.strftime('(UTC, %z)') time_zone = time_zone_1 + " " + time_zone_2 - print("expected time zone: " + time_zone) + else: + time_zone_arr = os.popen('timedatectl | grep zone').read().strip().split(':') + if len(time_zone_arr) > 1: + time_zone = time_zone_arr[1].lstrip() + else: + # possibly in a docker container + time_zone_1 = os.popen('ls -l /etc/localtime|awk -F/ \'{print $(NF-1) "/" $NF}\'').read().strip() + time_zone_2 = os.popen('date "+(%Z, %z)"').read().strip() + time_zone = time_zone_1 + " " + time_zone_2 + print("expected time zone: " + time_zone) tdLog.printNoPrefix("==========step1:create tables==========") tdSql.execute( diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 500e867121..d6ac4d4208 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -13,6 +13,12 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP #rpcDebugFlagVal = '143' #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal @@ -34,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -192,7 +198,10 @@ class TDTestCase: shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd += "> nul 2>&1 &" + else: + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -306,7 +315,10 @@ class TDTestCase: shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd += "> nul 2>&1 &" + else: + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) @@ -438,7 +450,10 @@ class TDTestCase: shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) - shellCmd += "> /dev/null 2>&1 &" + if (platform.system().lower() == 'windows'): + shellCmd += "> nul 2>&1 &" + else: + shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index e2c6cc2e77..70d02c4e29 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 51c8da2413..54b6881318 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 43b707e651..279518d283 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index ce273367c7..b0b8b06076 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index ca87f0dba5..9af78ce6c3 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 2b7f0d3d5f..9f308abd7c 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index a212cf7590..f7e56b4550 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 92347690d9..4098d151d1 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index d08adcdc83..45feb21019 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 58e36911c1..81105f5352 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index d06e144796..a6f1cab4a4 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/tmqDnode.py b/tests/system-test/7-tmq/tmqDnode.py index bb287134b1..235e9ef971 100644 --- a/tests/system-test/7-tmq/tmqDnode.py +++ b/tests/system-test/7-tmq/tmqDnode.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/7-tmq/tmqModule.py b/tests/system-test/7-tmq/tmqModule.py index ad5b4d70b3..086fde8f05 100644 --- a/tests/system-test/7-tmq/tmqModule.py +++ b/tests/system-test/7-tmq/tmqModule.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index b7cac43954..ebab6617c2 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index d7b2856b41..d97dd8c187 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -33,7 +33,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/99-TDcase/TD-15557.py b/tests/system-test/99-TDcase/TD-15557.py index e005985fe0..7a282e3176 100644 --- a/tests/system-test/99-TDcase/TD-15557.py +++ b/tests/system-test/99-TDcase/TD-15557.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index 5931360b90..ca182820d5 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -34,7 +34,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/99-TDcase/TD-16025.py b/tests/system-test/99-TDcase/TD-16025.py index 3a70eaf71b..6016b56192 100644 --- a/tests/system-test/99-TDcase/TD-16025.py +++ b/tests/system-test/99-TDcase/TD-16025.py @@ -41,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 8350f6752f..47e0cefb52 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -20,6 +20,7 @@ import time import base64 import json import platform +import socket from distutils.log import warn as printf from fabric2 import Connection sys.path.append("../pytest") @@ -149,7 +150,7 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') if masterIp == "": - host = '127.0.0.1' + host = socket.gethostname() else: try: config = eval(masterIp) @@ -170,8 +171,8 @@ if __name__ == "__main__": try: if key_word in open(fileName, encoding='UTF-8').read(): is_test_framework = 1 - except: - pass + except Exception as r: + print(r) updateCfgDictStr = '' if is_test_framework: moduleName = fileName.replace(".py", "").replace(os.sep, ".") @@ -181,8 +182,8 @@ if __name__ == "__main__": if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)): updateCfgDict = ucase.updatecfgDict updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode() - except : - pass + except Exception as r: + print(r) else: pass tdDnodes.deploy(1,updateCfgDict) From 4f7dc624330b2ab2cbd0bc0ebb4f65b656f2cb57 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 12:56:52 +0800 Subject: [PATCH 095/132] refactor(sync): add detail log flag --- source/libs/sync/src/syncSnapshot.c | 41 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a4dc89ad78..71b1a4490f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -105,8 +105,12 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot send begin seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, + msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -221,11 +225,16 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sTrace("snapshot send finish seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, + msgStr); } else { - sTrace("snapshot send sending seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); + sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d send msg:%s", host, port, pSender->seq, + pSender->ack, msgStr); } taosMemoryFree(msgStr); @@ -249,8 +258,12 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { syncSnapshotSend2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg); - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot send resend seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr); + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); + sTrace("sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, + msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -458,8 +471,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, msgStr); + sTrace("sync event snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -474,7 +487,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); sInfo( - "snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " + "sync event snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, raft log:%s", pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); taosMemoryFree(logSimpleStr); @@ -485,8 +498,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, msgStr); + sTrace("sync event snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -495,7 +508,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = false; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + sTrace("sync event snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); @@ -511,7 +524,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, + sTrace("sync event snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); From 8e65f1e426b49efdd84a64c452f47e8bed9cdaba Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 13:31:33 +0800 Subject: [PATCH 096/132] add hll partial/merge functions --- source/libs/function/inc/builtinsimpl.h | 2 ++ source/libs/function/src/builtins.c | 8 ++--- source/libs/function/src/builtinsimpl.c | 40 ++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 9de1580c73..614be17e08 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -126,7 +126,9 @@ int32_t getHistogramInfoSize(); bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t hllFunction(SqlFunctionCtx* pCtx); +int32_t hllFunctionMerge(SqlFunctionCtx* pCtx); int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getHLLInfoSize(); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 2158b8bca9..e2fc46aac0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1509,20 +1509,20 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "_hyperloglog_partial", .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateHLL, + .translateFunc = translateHLLPartial, .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, .processFunc = hllFunction, - .finalizeFunc = hllFinalize + .finalizeFunc = hllPartialFinalize }, { .name = "_hyperloglog_merge", .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateHLL, + .translateFunc = translateHLLMerge, .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, - .processFunc = hllFunction, + .processFunc = hllFunctionMerge, .finalizeFunc = hllFinalize }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 683d7e6342..5a4160f9a1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3411,7 +3411,6 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t resultBytes = getHistogramInfoSize(); char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); @@ -3557,6 +3556,27 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } +int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SHLLInfo* pInputInfo = (SHLLInfo *)varDataVal(data); + + for (int32_t k = 0; k < HLL_BUCKETS; ++k) { + if (pInfo->buckets[k] < pInputInfo->buckets[k]) { + pInputInfo->buckets[k] = pInfo->buckets[k]; + } + } + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + return TSDB_CODE_SUCCESS; +} + int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); @@ -3569,6 +3589,24 @@ int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getHLLInfoSize(); + char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStateInfo); return true; From bed14b6ee31c74f53449f69b91382bbf93c49a06 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 13:36:49 +0800 Subject: [PATCH 097/132] enable hll splitting --- source/libs/function/src/builtins.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e2fc46aac0..e858fb3159 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1503,7 +1503,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, .processFunc = hllFunction, - .finalizeFunc = hllFinalize + .finalizeFunc = hllFinalize, + .pPartialFunc = "_hyperloglog_partial", + .pMergeFunc = "_hyperloglog_merge" }, { .name = "_hyperloglog_partial", From 17f8ccc9012346a349bf8aaec40708f7641c1605 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 14:12:52 +0800 Subject: [PATCH 098/132] enh: record config index --- source/dnode/mnode/impl/src/mndSync.c | 11 +++++++++++ source/dnode/mnode/impl/src/mndTrans.c | 4 ++++ source/dnode/mnode/sdb/inc/sdb.h | 3 +++ source/dnode/mnode/sdb/src/sdb.c | 6 +++++- source/dnode/mnode/sdb/src/sdbFile.c | 24 ++++++++++++++++++++---- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 2004cd8d0a..df8dc15c4a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -50,6 +50,10 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM tsem_post(&pMgmt->syncSem); } else { if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) { + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } sdbWriteFile(pMnode->pSdb); } } @@ -59,11 +63,18 @@ int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { SMnode *pMnode = pFsm->data; pSnapshot->lastApplyIndex = sdbGetCommitIndex(pMnode->pSdb); pSnapshot->lastApplyTerm = sdbGetCommitTerm(pMnode->pSdb); + pSnapshot->lastConfigIndex = sdbGetCurConfig(pMnode->pSdb); return 0; } void mndRestoreFinish(struct SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; + + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } + if (!pMnode->deploy) { mInfo("mnode sync restore finished, and will handle outstanding transactions"); mndTransPullup(pMnode); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index aa4950bfba..1e98a3bbf9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1388,6 +1388,10 @@ void mndTransPullup(SMnode *pMnode) { mndReleaseTrans(pMnode, pTrans); } + SSnapshotMeta sMeta = {0}; + if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); + } sdbWriteFile(pMnode->pSdb); taosArrayDestroy(pArray); } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index dcea95df54..22f3faf620 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -171,6 +171,7 @@ typedef struct SSdb { int64_t lastCommitTerm; int64_t curVer; int64_t curTerm; + int64_t curConfig; int64_t tableVer[SDB_MAX]; int64_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; @@ -359,10 +360,12 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type); */ void sdbSetApplyIndex(SSdb *pSdb, int64_t index); void sdbSetApplyTerm(SSdb *pSdb, int64_t term); +void sdbSetCurConfig(SSdb *pSdb, int64_t config); int64_t sdbGetApplyIndex(SSdb *pSdb); int64_t sdbGetApplyTerm(SSdb *pSdb); int64_t sdbGetCommitIndex(SSdb *pSdb); int64_t sdbGetCommitTerm(SSdb *pSdb); +int64_t sdbGetCurConfig(SSdb *pSdb); SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index ccd5018aca..170bf3bc90 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -161,10 +161,14 @@ void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; } void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; } +void sdbSetCurConfig(SSdb *pSdb, int64_t config) { pSdb->curConfig = config; } + int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; } int64_t sdbGetApplyTerm(SSdb *pSdb) { return pSdb->curTerm; } int64_t sdbGetCommitIndex(SSdb *pSdb) { return pSdb->lastCommitVer; } -int64_t sdbGetCommitTerm(SSdb *pSdb) { return pSdb->lastCommitTerm; } \ No newline at end of file +int64_t sdbGetCommitTerm(SSdb *pSdb) { return pSdb->lastCommitTerm; } + +int64_t sdbGetCurConfig(SSdb *pSdb) { return pSdb->curConfig; } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index c0b27c33e8..bacf4e09bf 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -110,6 +110,16 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { return -1; } + ret = taosReadFile(pFile, &pSdb->curConfig, sizeof(int64_t)); + if (ret < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (ret != sizeof(int64_t)) { + terrno = TSDB_CODE_FILE_CORRUPTED; + return -1; + } + for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = 0; ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); @@ -173,6 +183,11 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { return -1; } + if (taosWriteFile(pFile, &pSdb->curConfig, sizeof(int64_t)) != sizeof(int64_t)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = 0; if (i < SDB_MAX) { @@ -288,8 +303,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { pSdb->lastCommitVer = pSdb->curVer; pSdb->lastCommitTerm = pSdb->curTerm; memcpy(pSdb->tableVer, tableVer, sizeof(tableVer)); - mDebug("read sdb file:%s successfully, ver:%" PRId64 " term:%" PRId64, file, pSdb->lastCommitVer, - pSdb->lastCommitTerm); + mDebug("read sdb file:%s successfully, index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, pSdb->lastCommitVer, + pSdb->lastCommitTerm, pSdb->curConfig); _OVER: taosCloseFile(&pFile); @@ -498,6 +513,7 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { taosThreadMutexLock(&pSdb->filelock); int64_t commitIndex = pSdb->lastCommitVer; int64_t commitTerm = pSdb->lastCommitTerm; + int64_t curConfig = pSdb->curConfig; if (taosCopyFile(datafile, pIter->name) < 0) { taosThreadMutexUnlock(&pSdb->filelock); terrno = TAOS_SYSTEM_ERROR(errno); @@ -516,8 +532,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mInfo("sdbiter:%p, is created to read snapshot, index:%" PRId64 " term:%" PRId64 " file:%s", pIter, commitIndex, - commitTerm, pIter->name); + mInfo("sdbiter:%p, is created to read snapshot, index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pIter, + commitIndex, commitTerm, curConfig, pIter->name); return 0; } From ee323021016eebf745a09f3502c5ffee28ce3cb1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 14:14:45 +0800 Subject: [PATCH 099/132] refactor(sync): add detail log flag --- source/libs/sync/src/syncMain.c | 10 ++++++-- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/sync/src/syncSnapshot.c | 40 +++++++++++++++++++---------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fedbb5c1b4..0fa43db8d9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = false; +bool gRaftDetailLog = true; static int32_t tsNodeRefId = -1; @@ -826,7 +826,13 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - int32_t electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); + int32_t electMS; + + if (pSyncNode->pRaftCfg->isStandBy) { + electMS = TIMER_MAX_MS; + } else { + electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); + } ret = syncNodeRestartElectTimer(pSyncNode, electMS); return ret; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 466863c0db..49509ae979 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -184,8 +184,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, const char* linuxErrMsg = strerror(errno); sError("raftLogGetEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); - walCloseReadHandle(pWalHandle); ASSERT(0); + walCloseReadHandle(pWalHandle); return code; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 71b1a4490f..0ceea49112 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -230,7 +230,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { uint16_t port; syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { - sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, + sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", + host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr); } else { sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d send msg:%s", host, port, pSender->seq, @@ -470,9 +471,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->ack = pMsg->seq; needRsp = true; - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, msgStr); + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sTrace("sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -486,10 +490,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); SSnapshot snapshot; pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sInfo( - "sync event snapshot recv finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " + "sync event snapshot recv from %s:%d finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, raft log:%s", - pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); + host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr); taosMemoryFree(logSimpleStr); pReceiver->pWriter = NULL; @@ -498,8 +505,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -507,9 +514,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { snapshotReceiverStop(pReceiver, false); needRsp = false; + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, msgStr); + sTrace("sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, + port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); @@ -523,9 +534,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { } needRsp = true; - char *msgStr = syncSnapshotSend2Str(pMsg); - sTrace("sync event snapshot recv receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, msgStr); + char *msgStr = syncSnapshotSend2Str(pMsg); + char host[128]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sTrace("sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, + port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr); taosMemoryFree(msgStr); } else { From d80d55c0f71efcf260b707938b3c66b66dd8162b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 14:14:55 +0800 Subject: [PATCH 100/132] test: case for mnode --- tests/script/tsim/mnode/basic3.sim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index bc70cd7a85..ecd663a2d3 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,8 +39,11 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 +system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 +return + $x = 0 step2: $x = $x + 1 From 7ae6d8f77d7e585ae197a72fb53780b24df0e39d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 14:18:57 +0800 Subject: [PATCH 101/132] refactor(sync): set gRaftDetailLog true --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0fa43db8d9..2a5d263e4c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = true; +bool gRaftDetailLog = false; static int32_t tsNodeRefId = -1; From 20b1ee6160f4483894b57e23726cc2412ca7e245 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 14:21:27 +0800 Subject: [PATCH 102/132] fix bugs --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5a4160f9a1..6e969749c6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3569,7 +3569,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) { for (int32_t k = 0; k < HLL_BUCKETS; ++k) { if (pInfo->buckets[k] < pInputInfo->buckets[k]) { - pInputInfo->buckets[k] = pInfo->buckets[k]; + pInfo->buckets[k] = pInputInfo->buckets[k]; } } From 04f1f71095a97ade5901cb76c1bf196740f24732 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 9 Jun 2022 14:27:02 +0800 Subject: [PATCH 103/132] fix:error in schemaless --- source/client/src/clientSml.c | 7 +++++-- source/client/test/smlTest.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 995722a49f..fe21d48a65 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1548,7 +1548,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV } size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && type->valuestring[0] == 's') { + if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { //seconds timeDouble = timeDouble * 1e9; if(smlDoubleToInt64OverFlow(timeDouble)){ @@ -1556,9 +1556,10 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV return TSDB_CODE_TSC_INVALID_TIME_STAMP; } *tsVal = timeDouble; - } else if (typeLen == 2 && type->valuestring[1] == 's') { + } else if (typeLen == 2 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { switch (type->valuestring[0]) { case 'm': + case 'M': //milliseconds timeDouble = timeDouble * 1e6; if(smlDoubleToInt64OverFlow(timeDouble)){ @@ -1568,6 +1569,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV *tsVal = timeDouble; break; case 'u': + case 'U': //microseconds timeDouble = timeDouble * 1e3; if(smlDoubleToInt64OverFlow(timeDouble)){ @@ -1577,6 +1579,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV *tsVal = timeDouble; break; case 'n': + case 'N': //nanoseconds *tsVal = timeDouble; break; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 46c451f164..8c382339b1 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1329,3 +1329,37 @@ TEST(testCase, sml_oom_Test) { ASSERT_EQ(taos_errno(res), 0); taos_free_result(pRes); } + +TEST(testCase, sml_16368_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use d16368"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(info, nullptr); + + const char *sql[] = { + "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, \"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": \"binary\"}}},\n" + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, \"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}},\n" + "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": {\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}},\n" + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": {\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": \"nchar\"}}},\n" + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": {\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": {\"value\": \"ste2\", \"type\": \"nchar\"}}},\n" + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]" + }; + int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); + ASSERT_EQ(ret, 0); + + destroyRequest(request); + smlDestroyInfo(info); +} From 08a19b7b28f86228f9b49dbe635395058f2998ad Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 9 Jun 2022 15:00:48 +0800 Subject: [PATCH 104/132] enh: set 0 in flen part for tsrow --- source/common/src/tdataformat.c | 24 ++++++-- source/common/test/dataformatTest.cpp | 81 ++++++++++++++------------- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 84a2cf0544..1ddb606ccf 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -244,7 +244,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S } } - // ASSERT(flags); // only 1 column(ts) + ASSERT(flags); // decide uint32_t nData = 0; @@ -268,8 +268,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; break; default: - break; // only ts column - // ASSERT(0); + break; + ASSERT(0); } uint8_t tflags = 0; @@ -374,7 +374,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S ptv = pf + pTSchema->flen; break; default: - // ASSERT(0); + ASSERT(0); break; } } else { @@ -421,12 +421,26 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S _set_none: if ((flags & 0xf0) == 0) { setBitMap(pb, 0, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } } continue; _set_null: if ((flags & 0xf0) == 0) { setBitMap(pb, 1, iColumn - 1, flags); + if (flags & TSROW_HAS_VAL) { // set 0 + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = 0; + } else { + tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type); + } + } } else { SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); pTSKVRow->nCols++; @@ -497,7 +511,7 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal SValue value; ASSERT(iCol < pTSchema->numOfCols); - // ASSERT(flags); // only 1 ts column + ASSERT(flags); ASSERT(pRow->sver == pTSchema->version); if (iCol == 0) { diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 81e91da0d6..020b1e0deb 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -52,61 +52,61 @@ STSchema *genSTSchema(int16_t nCols) { switch (i) { case 0: { - pSchema[0].type = TSDB_DATA_TYPE_TIMESTAMP; - pSchema[0].bytes = TYPE_BYTES[pSchema[0].type]; + pSchema[i].type = TSDB_DATA_TYPE_TIMESTAMP; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 1: { - pSchema[1].type = TSDB_DATA_TYPE_INT; - pSchema[1].bytes = TYPE_BYTES[pSchema[1].type]; + pSchema[i].type = TSDB_DATA_TYPE_INT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; ; } break; case 2: { - pSchema[2].type = TSDB_DATA_TYPE_BIGINT; - pSchema[2].bytes = TYPE_BYTES[pSchema[2].type]; + pSchema[i].type = TSDB_DATA_TYPE_BIGINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 3: { - pSchema[3].type = TSDB_DATA_TYPE_FLOAT; - pSchema[3].bytes = TYPE_BYTES[pSchema[3].type]; + pSchema[i].type = TSDB_DATA_TYPE_FLOAT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 4: { - pSchema[4].type = TSDB_DATA_TYPE_DOUBLE; - pSchema[4].bytes = TYPE_BYTES[pSchema[4].type]; + pSchema[i].type = TSDB_DATA_TYPE_DOUBLE; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 5: { - pSchema[5].type = TSDB_DATA_TYPE_BINARY; - pSchema[5].bytes = 12; + pSchema[i].type = TSDB_DATA_TYPE_BINARY; + pSchema[i].bytes = 12; } break; case 6: { - pSchema[6].type = TSDB_DATA_TYPE_NCHAR; - pSchema[6].bytes = 42; + pSchema[i].type = TSDB_DATA_TYPE_NCHAR; + pSchema[i].bytes = 42; } break; case 7: { - pSchema[7].type = TSDB_DATA_TYPE_TINYINT; - pSchema[7].bytes = TYPE_BYTES[pSchema[7].type]; + pSchema[i].type = TSDB_DATA_TYPE_TINYINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 8: { - pSchema[8].type = TSDB_DATA_TYPE_SMALLINT; - pSchema[8].bytes = TYPE_BYTES[pSchema[8].type]; + pSchema[i].type = TSDB_DATA_TYPE_SMALLINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 9: { - pSchema[9].type = TSDB_DATA_TYPE_BOOL; - pSchema[9].bytes = TYPE_BYTES[pSchema[9].type]; + pSchema[i].type = TSDB_DATA_TYPE_BOOL; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 10: { - pSchema[10].type = TSDB_DATA_TYPE_UTINYINT; - pSchema[10].bytes = TYPE_BYTES[pSchema[10].type]; + pSchema[i].type = TSDB_DATA_TYPE_UTINYINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 11: { - pSchema[11].type = TSDB_DATA_TYPE_USMALLINT; - pSchema[11].bytes = TYPE_BYTES[pSchema[11].type]; + pSchema[i].type = TSDB_DATA_TYPE_USMALLINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 12: { - pSchema[12].type = TSDB_DATA_TYPE_UINT; - pSchema[12].bytes = TYPE_BYTES[pSchema[12].type]; + pSchema[i].type = TSDB_DATA_TYPE_UINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; case 13: { - pSchema[13].type = TSDB_DATA_TYPE_UBIGINT; - pSchema[13].bytes = TYPE_BYTES[pSchema[13].type]; + pSchema[i].type = TSDB_DATA_TYPE_UBIGINT; + pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; } break; default: @@ -146,9 +146,9 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { case 0: sscanf(data[i], "%" PRIi64, &colVal.value.ts); break; - case 1: { + case 1: sscanf(data[i], "%" PRIi32, &colVal.value.i32); - } break; + break; case 2: sscanf(data[i], "%" PRIi64, &colVal.value.i64); break; @@ -404,9 +404,10 @@ static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) { } TEST(testCase, AllNormTest) { - int16_t nCols = 1; - STSRow2 *row = nullptr; - SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); + int16_t nCols = 14; + STSRowBuilder rb = tsRowBuilderInit(); + STSRow2 *row = nullptr; + SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); STSchema *pTSchema = genSTSchema(nCols); @@ -414,15 +415,16 @@ TEST(testCase, AllNormTest) { // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, // c9 bool - char *data[10] = {"1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "1"}; + char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}; genTestData((const char **)&data, nCols, &pArray); - tTSRowNew(NULL, pArray, pTSchema, &row); + tTSRowNew(&rb, pArray, pTSchema, &row); debugPrintTSRow(row, pTSchema, __func__, __LINE__); checkTSRow((const char **)&data, row, pTSchema); + tsRowBuilderClear(&rb); taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } @@ -443,12 +445,12 @@ TEST(testCase, NoneTest) { const char *data[nRows][nCols] = { {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220002", "no", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "nu"}, - {"1653694220003", "nu", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, + {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, @@ -465,13 +467,12 @@ TEST(testCase, NoneTest) { {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", "no"}}; - for (int r = 0; r < nRows; ++r) { genTestData((const char **)&data[r], nCols, &pArray); tTSRowNew(NULL, pArray, pTSchema, &row); debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print checkTSRow((const char **)&data[r], row, pTSchema); // check - taosMemoryFreeClear(row); + tTSRowFree(row); taosArrayClear(pArray); } From fb3b66413e1340d7a7ec829f210a9b53ebe44ab2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 9 Jun 2022 15:13:58 +0800 Subject: [PATCH 105/132] enh: code optimization for trow --- source/common/test/dataformatTest.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 020b1e0deb..4be0711546 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -274,9 +274,6 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_MEDIUMBLOB: printf("MedBLOB "); break; - // case TSDB_DATA_TYPE_BINARY: - // printf("BINARY "); - // break; case TSDB_DATA_TYPE_MAX: printf("UNDEF "); break; From eab330c092b8056b22a2dbf8038a802ba46b90e5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Jun 2022 15:14:11 +0800 Subject: [PATCH 106/132] enh: refactor index code --- source/libs/index/inc/indexCache.h | 22 ++++---- source/libs/index/inc/indexInt.h | 2 +- source/libs/index/inc/indexTfile.h | 8 +-- source/libs/index/src/index.c | 30 +++++------ source/libs/index/src/indexCache.c | 80 ++++++++++++++-------------- source/libs/index/src/indexTfile.c | 14 ++--- source/libs/index/test/indexTests.cc | 10 ++-- 7 files changed, 83 insertions(+), 83 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 6c95eb987b..8b5885d58b 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -62,25 +62,25 @@ typedef struct CacheTerm { } CacheTerm; // -IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); +IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); -void indexCacheForceToMerge(void* cache); -void indexCacheDestroy(void* cache); -void indexCacheBroadcast(void* cache); -void indexCacheWait(void* cache); +void idxCacheForceToMerge(void* cache); +void idxCacheDestroy(void* cache); +void idxCacheBroadcast(void* cache); +void idxCacheWait(void* cache); -Iterate* indexCacheIteratorCreate(IndexCache* cache); +Iterate* idxCacheIteratorCreate(IndexCache* cache); void idxCacheIteratorDestroy(Iterate* iiter); -int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid); +int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid); // int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s); +int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s); -void indexCacheRef(IndexCache* cache); -void indexCacheUnRef(IndexCache* cache); +void idxCacheRef(IndexCache* cache); +void idxCacheUnRef(IndexCache* cache); -void indexCacheDebug(IndexCache* cache); +void idxCacheDebug(IndexCache* cache); void idxCacheDestroyImm(IndexCache* cache); #ifdef __cplusplus diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 47f7260d3a..b6bfb25c24 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -138,7 +138,7 @@ int32_t indexRemoveRef(int64_t ref); void indexAcquireRef(int64_t ref); void indexReleaseRef(int64_t ref); -int32_t indexSerialCacheKey(ICacheKey* key, char* buf); +int32_t idxSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); diff --git a/source/libs/index/inc/indexTfile.h b/source/libs/index/inc/indexTfile.h index ca55aa93da..6cfea5bc0b 100644 --- a/source/libs/index/inc/indexTfile.h +++ b/source/libs/index/inc/indexTfile.h @@ -117,10 +117,10 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // -IndexTFile* indexTFileCreate(const char* path); -void indexTFileDestroy(IndexTFile* tfile); -int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); -int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr); +IndexTFile* idxTFileCreate(const char* path); +void idxTFileDestroy(IndexTFile* tfile); +int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); +int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr); Iterate* tfileIteratorCreate(TFileReader* reader); void tfileIteratorDestroy(Iterate* iterator); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9b8bee5623..bbcef4c3e1 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -106,8 +106,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { return -1; } - // sIdx->cache = (void*)indexCacheCreate(sIdx); - sIdx->tindex = indexTFileCreate(path); + // sIdx->cache = (void*)idxCacheCreate(sIdx); + sIdx->tindex = idxTFileCreate(path); if (sIdx->tindex == NULL) { goto END; } @@ -136,7 +136,7 @@ void indexDestroy(void* handle) { SIndex* sIdx = handle; taosThreadMutexDestroy(&sIdx->mtx); tsem_destroy(&sIdx->sem); - indexTFileDestroy(sIdx->tindex); + idxTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); return; @@ -147,12 +147,12 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - indexCacheForceToMerge((void*)(*pCache)); + idxCacheForceToMerge((void*)(*pCache)); indexInfo("%s wait to merge", (*pCache)->colName); indexWait((void*)(sIdx)); indexInfo("%s finish to wait", (*pCache)->colName); iter = taosHashIterate(sIdx->colObj, iter); - indexCacheUnRef(*pCache); + idxCacheUnRef(*pCache); } taosHashCleanup(sIdx->colObj); sIdx->colObj = NULL; @@ -186,11 +186,11 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { - IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType); + IndexCache* pCache = idxCacheCreate(index, p->suid, p->colName, p->colType); taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } @@ -201,12 +201,12 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); - int ret = indexCachePut(*cache, p, uid); + int ret = idxCachePut(*cache, p, uid); if (ret != 0) { return ret; } @@ -331,7 +331,7 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); @@ -345,14 +345,14 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) int64_t st = taosGetTimestampUs(); SIdxTRslt* tr = idxTRsltCreate(); - if (0 == indexCacheSearch(cache, query, tr, &s)) { + if (0 == idxCacheSearch(cache, query, tr, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by", term->colName); // coloum already drop by other oper, no need to query tindex return 0; } else { st = taosGetTimestampUs(); - if (0 != indexTFileSearch(sIdx->tindex, query, tr)) { + if (0 != idxTFileSearch(sIdx->tindex, query, tr)) { indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); goto END; } @@ -472,7 +472,7 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { indexWarn("empty tfile reader found"); } // handle flush - Iterate* cacheIter = indexCacheIteratorCreate(pCache); + Iterate* cacheIter = idxCacheIteratorCreate(pCache); if (cacheIter == NULL) { indexError("%p immtable is empty, ignore merge opera", pCache); idxCacheDestroyImm(pCache); @@ -532,7 +532,7 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { tfileIteratorDestroy(tfileIter); tfileReaderUnRef(pReader); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); int64_t cost = taosGetTimestampUs() - st; if (ret != 0) { @@ -620,7 +620,7 @@ END: return -1; } -int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { +int32_t idxSerialCacheKey(ICacheKey* key, char* buf) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); char* p = buf; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 20cd9c8b4c..1ddb5604c8 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -68,7 +68,7 @@ static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRsl cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, cacheSearchRange_JSON}}; -static void doMergeWork(SSchedMsg* msg); +static void idxDoMergeWork(SSchedMsg* msg); static bool idxCacheIteratorNext(Iterate* itera); static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { @@ -331,9 +331,9 @@ static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTRslt* tr, ST // impl later return 0; } -static IterateValue* indexCacheIteratorGetValue(Iterate* iter); +static IterateValue* idxCacheIteratorGetValue(Iterate* iter); -IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { +IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); @@ -352,13 +352,13 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in taosThreadMutexInit(&cache->mtx, NULL); taosThreadCondInit(&cache->finished, NULL); - indexCacheRef(cache); + idxCacheRef(cache); if (idx != NULL) { indexAcquireRef(idx->refId); } return cache; } -void indexCacheDebug(IndexCache* cache) { +void idxCacheDebug(IndexCache* cache) { MemTable* tbl = NULL; taosThreadMutexLock(&cache->mtx); @@ -405,7 +405,7 @@ void indexCacheDebug(IndexCache* cache) { } } -void indexCacheDestroySkiplist(SSkipList* slt) { +void idxCacheDestroySkiplist(SSkipList* slt) { SSkipListIterator* iter = tSkipListCreateIter(slt); while (iter != NULL && tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); @@ -418,11 +418,11 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroyIter(iter); tSkipListDestroy(slt); } -void indexCacheBroadcast(void* cache) { +void idxCacheBroadcast(void* cache) { IndexCache* pCache = cache; taosThreadCondBroadcast(&pCache->finished); } -void indexCacheWait(void* cache) { +void idxCacheWait(void* cache) { IndexCache* pCache = cache; taosThreadCondWait(&pCache->finished, &pCache->mtx); } @@ -435,14 +435,14 @@ void idxCacheDestroyImm(IndexCache* cache) { tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - indexCacheBroadcast(cache); + idxCacheBroadcast(cache); taosThreadMutexUnlock(&cache->mtx); idxMemUnRef(tbl); idxMemUnRef(tbl); } -void indexCacheDestroy(void* cache) { +void idxCacheDestroy(void* cache) { IndexCache* pCache = cache; if (pCache == NULL) { return; @@ -460,7 +460,7 @@ void indexCacheDestroy(void* cache) { taosMemoryFree(pCache); } -Iterate* indexCacheIteratorCreate(IndexCache* cache) { +Iterate* idxCacheIteratorCreate(IndexCache* cache) { if (cache->imm == NULL) { return NULL; } @@ -477,7 +477,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->val.colVal = NULL; iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; iiter->next = idxCacheIteratorNext; - iiter->getValue = indexCacheIteratorGetValue; + iiter->getValue = idxCacheIteratorGetValue; taosThreadMutexUnlock(&cache->mtx); @@ -492,9 +492,9 @@ void idxCacheIteratorDestroy(Iterate* iter) { taosMemoryFree(iter); } -int indexCacheSchedToMerge(IndexCache* pCache, bool notify) { +int idxCacheSchedToMerge(IndexCache* pCache, bool notify) { SSchedMsg schedMsg = {0}; - schedMsg.fp = doMergeWork; + schedMsg.fp = idxDoMergeWork; schedMsg.ahandle = pCache; if (notify) { schedMsg.thandle = taosMemoryMalloc(1); @@ -505,17 +505,17 @@ int indexCacheSchedToMerge(IndexCache* pCache, bool notify) { return 0; } -static void indexCacheMakeRoomForWrite(IndexCache* cache) { +static void idxCacheMakeRoomForWrite(IndexCache* cache) { while (true) { if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - indexCacheWait(cache); + idxCacheWait(cache); } else { bool quit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; - indexCacheRef(cache); + idxCacheRef(cache); cache->imm = cache->mem; cache->mem = idxInternalCacheCreate(cache->type); cache->mem->pCache = cache; @@ -525,18 +525,18 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } // sched to merge // unref cache in bgwork - indexCacheSchedToMerge(cache, quit); + idxCacheSchedToMerge(cache, quit); } } } -int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { +int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); IndexCache* pCache = cache; - indexCacheRef(pCache); + idxCacheRef(pCache); // encode data CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); if (cache == NULL) { @@ -559,7 +559,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexLock(&pCache->mtx); pCache->occupiedMem += estimate; - indexCacheMakeRoomForWrite(pCache); + idxCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; idxMemRef(tbl); tSkipListPut(tbl->mem, (char*)ct); @@ -567,29 +567,29 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexUnlock(&pCache->mtx); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); return 0; // encode end } -void indexCacheForceToMerge(void* cache) { +void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; - indexCacheRef(pCache); + idxCacheRef(pCache); taosThreadMutexLock(&pCache->mtx); indexInfo("%p is forced to merge into tfile", pCache); pCache->occupiedMem += MEM_SIGNAL_QUIT; - indexCacheMakeRoomForWrite(pCache); + idxCacheMakeRoomForWrite(pCache); taosThreadMutexUnlock(&pCache->mtx); - indexCacheUnRef(pCache); + idxCacheUnRef(pCache); return; } -int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { +int idxCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache* pCache = cache; return 0; } -static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) { +static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) { if (mem == NULL) { return 0; } @@ -603,7 +603,7 @@ static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* t return cacheSearch[0][qtype](mem, term, tr, s); } } -int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { +int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); if (cache == NULL) { return 0; @@ -618,10 +618,10 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe idxMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); - int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0; + int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0; if (ret == 0 && *s != kTypeDeletion) { // continue search in imm - ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0; + ret = (imm && imm->mem) ? idxQueryMem(imm, query, result, s) : 0; } idxMemUnRef(mem); @@ -631,20 +631,20 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe return ret; } -void indexCacheRef(IndexCache* cache) { +void idxCacheRef(IndexCache* cache) { if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } -void indexCacheUnRef(IndexCache* cache) { +void idxCacheUnRef(IndexCache* cache) { if (cache == NULL) { return; } int ref = T_REF_DEC(cache); if (ref == 0) { - indexCacheDestroy(cache); + idxCacheDestroy(cache); } } @@ -662,7 +662,7 @@ void idxMemUnRef(MemTable* tbl) { int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; - indexCacheDestroySkiplist(slt); + idxCacheDestroySkiplist(slt); taosMemoryFree(tbl); } } @@ -693,15 +693,15 @@ static int32_t idxCacheTermCompare(const void* l, const void* r) { return cmp; } -static int indexFindCh(char* a, char c) { +static int idxFindCh(char* a, char c) { char* p = a; while (*p != 0 && *p++ != c) { } return p - a; } static int idxCacheJsonTermCompareImpl(char* a, char* b) { - // int alen = indexFindCh(a, '&'); - // int blen = indexFindCh(b, '&'); + // int alen = idxFindCh(a, '&'); + // int blen = idxFindCh(b, '&'); // int cmp = strncmp(a, b, MIN(alen, blen)); // if (cmp == 0) { @@ -742,7 +742,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) { return tbl; } -static void doMergeWork(SSchedMsg* msg) { +static void idxDoMergeWork(SSchedMsg* msg) { IndexCache* pCache = msg->ahandle; SIndex* sidx = (SIndex*)pCache->index; @@ -771,7 +771,7 @@ static bool idxCacheIteratorNext(Iterate* itera) { return next; } -static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { +static IterateValue* idxCacheIteratorGetValue(Iterate* iter) { // opt later return &iter->val; } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b64db1dde4..619f89cc31 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -118,7 +118,7 @@ TFileCache* tfileCacheCreate(const char* path) { ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = (int32_t)strlen(header->colName)}; char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(&key, buf); + int32_t sz = idxSerialCacheKey(&key, buf); assert(sz < sizeof(buf)); taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); @@ -149,7 +149,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(key, buf); + int32_t sz = idxSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); if (reader == NULL || *reader == NULL) { @@ -161,7 +161,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { } void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { char buf[128] = {0}; - int32_t sz = indexSerialCacheKey(key, buf); + int32_t sz = idxSerialCacheKey(key, buf); // remove last version index reader TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL && *p != NULL) { @@ -620,7 +620,7 @@ void tfileWriterDestroy(TFileWriter* tw) { taosMemoryFree(tw); } -IndexTFile* indexTFileCreate(const char* path) { +IndexTFile* idxTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); if (cache == NULL) { return NULL; @@ -635,7 +635,7 @@ IndexTFile* indexTFileCreate(const char* path) { tfile->cache = cache; return tfile; } -void indexTFileDestroy(IndexTFile* tfile) { +void idxTFileDestroy(IndexTFile* tfile) { if (tfile == NULL) { return; } @@ -644,7 +644,7 @@ void indexTFileDestroy(IndexTFile* tfile) { taosMemoryFree(tfile); } -int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { +int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { int ret = -1; if (tfile == NULL) { return ret; @@ -667,7 +667,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) { return tfileReaderSearch(reader, query, result); } -int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { +int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = // term->nColName, .version = 1}; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 90dea3a377..e18297cd25 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -521,10 +521,10 @@ class CacheObj { public: CacheObj() { // TODO - cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); + cache = idxCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); } int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { - int ret = indexCachePut(cache, term, uid); + int ret = idxCachePut(cache, term, uid); if (ret != 0) { // std::cout << "failed to put into cache: " << ret << std::endl; @@ -533,12 +533,12 @@ class CacheObj { } void Debug() { // - indexCacheDebug(cache); + idxCacheDebug(cache); } int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { SIdxTRslt* tr = idxTRsltCreate(); - int ret = indexCacheSearch(cache, query, tr, s); + int ret = idxCacheSearch(cache, query, tr, s); idxTRsltMergeTo(tr, result); idxTRsltDestroy(tr); @@ -549,7 +549,7 @@ class CacheObj { } ~CacheObj() { // TODO - indexCacheDestroy(cache); + idxCacheDestroy(cache); } private: From 001061fcd2fac7595c096da7e5a1db3ecf9a46b0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 9 Jun 2022 15:17:10 +0800 Subject: [PATCH 107/132] add get tb index async api --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 4 +- include/libs/nodes/plannodes.h | 1 + source/common/src/tmsg.c | 9 ++ source/libs/catalog/inc/catalogInt.h | 8 +- source/libs/catalog/src/catalog.c | 6 +- source/libs/catalog/src/ctgAsync.c | 101 +++++++++++-- source/libs/catalog/src/ctgRemote.c | 4 +- source/libs/catalog/src/ctgUtil.c | 18 +++ source/libs/command/inc/commandInt.h | 6 +- source/libs/command/src/explain.c | 156 ++++++++++++++++++++- source/libs/planner/src/planPhysiCreater.c | 1 + 12 files changed, 295 insertions(+), 20 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5453c8d0ea..b3e1022dc4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2510,6 +2510,7 @@ typedef struct { int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp); +void tFreeSTableIndexInfo(void *pInfo); typedef struct { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 4d074db08d..b0fac72172 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -67,6 +67,7 @@ typedef struct SCatalogReq { SArray *pUdf; // element is udf name SArray *pIndex; // element is index name SArray *pUser; // element is SUserAuthInfo + SArray *pTableIndex; // element is SNAME bool qNodeRequired; // valid qnode bool forceUpdate; } SCatalogReq; @@ -82,6 +83,7 @@ typedef struct SMetaData { SArray *pDbInfo; // pRes = SDbInfo* SArray *pTableMeta; // pRes = STableMeta* SArray *pTableHash; // pRes = SVgroupInfo* + SArray *pTableIndex; // pRes = SArray* SArray *pUdfList; // pRes = SFuncInfo* SArray *pIndex; // pRes = SIndexInfo* SArray *pUser; // pRes = bool* @@ -277,7 +279,7 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes); +int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 00380724b3..90cbb7a2e6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -304,6 +304,7 @@ typedef struct SDownstreamSourceNode { typedef struct SExchangePhysiNode { SPhysiNode node; int32_t srcGroupId; // group id of datasource suplans + bool singleChannel; SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b9b4556255..c34e35733f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2491,6 +2491,15 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR return 0; } +void tFreeSTableIndexInfo(void* info) { + if (NULL == info) { + return; + } + + STableIndexInfo *pInfo = (STableIndexInfo*)info; + + taosMemoryFree(pInfo->expr); +} int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { SEncoder encoder = {0}; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 3de9ba5f0e..b581c15e5d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -67,6 +67,7 @@ typedef enum { CTG_TASK_GET_DB_INFO, CTG_TASK_GET_TB_META, CTG_TASK_GET_TB_HASH, + CTG_TASK_GET_TB_INDEX, CTG_TASK_GET_INDEX, CTG_TASK_GET_UDF, CTG_TASK_GET_USER, @@ -93,6 +94,10 @@ typedef struct SCtgTbMetaCtx { int32_t flag; } SCtgTbMetaCtx; +typedef struct SCtgTbIndexCtx { + SName* pName; +} SCtgTbIndexCtx; + typedef struct SCtgDbVgCtx { char dbFName[TSDB_DB_FNAME_LEN]; } SCtgDbVgCtx; @@ -189,6 +194,7 @@ typedef struct SCtgJob { int32_t indexNum; int32_t userNum; int32_t dbInfoNum; + int32_t tbIndexNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -490,7 +496,7 @@ int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutpu int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask); -int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, const char *tbFName, SArray** out, SCtgTask* pTask); +int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, SName* name, SArray** out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0f6a79c14c..502949f191 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1136,14 +1136,14 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL)); } -int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes) { +int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == tbFName || NULL == pRes) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pRes) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), tbFName, pRes, NULL)); + CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), (SName*)pTableName, pRes, NULL)); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index bc2c8dad7a..2574528d15 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } @@ -232,6 +232,35 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_INDEX; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbIndexCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbIndexCtx* ctx = task.taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(task.taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) { int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum; if (dbNum > 0) { @@ -329,8 +358,9 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex); int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); + int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); - *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum; + *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum; if (*taskNum <= 0) { ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); return TSDB_CODE_SUCCESS; @@ -360,6 +390,7 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pJob->indexNum = indexNum; pJob->userNum = userNum; pJob->dbInfoNum = dbInfoNum; + pJob->tbIndexNum = tbIndexNum; pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask)); @@ -398,6 +429,11 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name)); } + for (int32_t i = 0; i < tbIndexNum; ++i) { + SName* name = taosArrayGet(pReq->pTableIndex, i); + CTG_ERR_JRET(ctgInitGetTbIndexTask(pJob, taskIdx++, name)); + } + for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName)); @@ -479,6 +515,21 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableIndex) { + pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + if (NULL == pJob->jobRes.pTableIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; + taosArrayPush(pJob->jobRes.pTableHash, &res); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgDumpIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { @@ -817,6 +868,20 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + + int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1056,13 +1121,24 @@ _return: CTG_RET(code); } +int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + void *pTrans = pTask->pJob->pTrans; + const SEpSet* pMgmtEps = &pTask->pJob->pMgmtEps; + SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + + CTG_ERR_RET(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), pCtx->pName, NULL, pTask)); + return TSDB_CODE_SUCCESS; +} + + int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; void *pTrans = pTask->pJob->pTrans; const SEpSet* pMgmtEps = &pTask->pJob->pMgmtEps; CTG_ERR_RET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), NULL, pTask)); - return TSDB_CODE_SUCCESS; } @@ -1168,15 +1244,16 @@ int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) { } SCtgAsyncFps gCtgAsyncFps[] = { - {ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes}, - {ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes}, - {ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes}, - {ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes}, - {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes}, - {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes}, - {ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes}, - {ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes}, - {ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes}, + {ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes}, + {ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes}, + {ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes}, + {ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes}, + {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes}, + {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes}, + {ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes}, + {ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes}, + {ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes}, + {ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 94c59ddc53..4c38a27a54 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -427,11 +427,13 @@ int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo * return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, const char *tbFName, SArray** out, SCtgTask* pTask) { +int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, SName *name, SArray** out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index b5160fe8de..98f6d6c88a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -59,6 +59,9 @@ void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableHash); pData->pTableHash = NULL; + + taosArrayDestroy(pData->pTableIndex); + pData->pTableIndex = NULL; taosArrayDestroy(pData->pUdfList); pData->pUdfList = NULL; @@ -248,6 +251,14 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { taosMemoryFreeClear(pCtx->out); break; } + case TDMT_MND_GET_TABLE_INDEX: { + SArray** pOut = (SArray**)pCtx->out; + if (pOut) { + taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCtx->out); + } + break; + } case TDMT_MND_RETRIEVE_FUNC: { SFuncInfo* pOut = (SFuncInfo*)pCtx->out; taosMemoryFree(pOut->pCode); @@ -344,6 +355,13 @@ void ctgFreeTask(SCtgTask* pTask) { taosMemoryFreeClear(pTask->res); break; } + case CTG_TASK_GET_TB_INDEX: { + SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(pTask->taskCtx); + taosArrayDestroyEx(pTask->res, tFreeSTableIndexInfo); + break; + } case CTG_TASK_GET_INDEX: { taosMemoryFreeClear(pTask->taskCtx); taosMemoryFreeClear(pTask->res); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index c8412afdc7..06e9af7569 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -32,15 +32,17 @@ extern "C" { #define EXPLAIN_PROJECTION_FORMAT "Projection" #define EXPLAIN_JOIN_FORMAT "%s" #define EXPLAIN_AGG_FORMAT "Aggragate" +#define EXPLAIN_INDEF_ROWS_FORMAT "Indefinite Rows Function" #define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1" #define EXPLAIN_SORT_FORMAT "Sort" #define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s" +#define EXPLAIN_FILL_FORMAT "Fill" #define EXPLAIN_SESSION_FORMAT "Session" #define EXPLAIN_STATE_WINDOW_FORMAT "StateWindow on Column %s" #define EXPLAIN_PARITION_FORMAT "Partition on Column %s" #define EXPLAIN_ORDER_FORMAT "Order: %s" #define EXPLAIN_FILTER_FORMAT "Filter: " -#define EXPLAIN_FILL_FORMAT "Fill: %s" +#define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: " #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: " #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]" #define EXPLAIN_OUTPUT_FORMAT "Output: " @@ -66,6 +68,8 @@ extern "C" { #define EXPLAIN_WIDTH_FORMAT "width=%d" #define EXPLAIN_FUNCTIONS_FORMAT "functions=%d" #define EXPLAIN_EXECINFO_FORMAT "cost=%.3f..%.3f rows=%" PRIu64 +#define EXPLAIN_MODE_FORMAT "mode=%s" +#define EXPLAIN_STRING_TYPE_FORMAT "%s" typedef struct SExplainGroup { int32_t nodeNum; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index dc9215b527..b4308c37c2 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -179,6 +179,21 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = mergePhysiNode->node.pChildren; break; } + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: { + SIndefRowsFuncPhysiNode *indefPhysiNode = (SIndefRowsFuncPhysiNode *)pNode; + pPhysiChildren = indefPhysiNode->node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: { + SMergeIntervalPhysiNode *intPhysiNode = (SMergeIntervalPhysiNode *)pNode; + pPhysiChildren = intPhysiNode->window.node.pChildren; + break; + } + case QUERY_NODE_PHYSICAL_PLAN_FILL: { + SFillPhysiNode *fillPhysiNode = (SFillPhysiNode *)pNode; + pPhysiChildren = fillPhysiNode->node.pChildren; + break; + } default: qError("not supported physical node type %d", pNode->type); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -212,12 +227,15 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); +/* if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); return TSDB_CODE_QRY_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); +*/ + taosArrayPush(*pExecInfo, rsp->subplanInfo); } ++group->physiPlanExecIdx; @@ -599,6 +617,42 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: { + SIndefRowsFuncPhysiNode *pIndefNode = (SIndefRowsFuncPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_INDEF_ROWS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + if (pIndefNode->pVectorFuncs) { + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIndefNode->pVectorFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIndefNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pIndefNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIndefNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pIndefNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIndefNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); @@ -607,7 +661,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum); + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->singleChannel ? 1 : group->nodeNum); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -750,6 +804,106 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: { + SMergeIntervalPhysiNode *pIntNode = (SMergeIntervalPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + uint8_t precision = getIntervalPrecision(pIntNode); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + pIntNode->slidingUnit); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pIntNode->window.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_FILL: { + SFillPhysiNode *pFillNode = (SFillPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_FILL_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_MODE_FORMAT, nodesGetFillModeString(pFillNode->mode)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pFillNode->node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pFillNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pFillNode->node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + if (pFillNode->pValues) { + SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues; + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT); + SNode* tNode = NULL; + int32_t i = 0; + FOREACH(tNode, pValues->pNodeList) { + if (i) { + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + SValueNode* tValue = (SValueNode*)tNode; + char *value = nodesGetStrValueFromNode(tValue); + EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value); + taosMemoryFree(value); + ++i; + } + + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, + pFillNode->timeRange.ekey); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pFillNode->node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pFillNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 52add839b4..6391772abe 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1186,6 +1186,7 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pMerge->srcGroupId; + pExchange->singleChannel = true; pExchange->node.pParent = (SPhysiNode*)pMerge; pExchange->node.pOutputDataBlockDesc = nodesCloneNode(pMerge->node.pOutputDataBlockDesc); if (NULL == pExchange->node.pOutputDataBlockDesc) { From 07cd4a9dfeac2f6c132902175817461007a73c8e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 9 Jun 2022 15:36:02 +0800 Subject: [PATCH 108/132] fix(os): add win32 taosd case --- tests/system-test/2-query/timezone.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index b73ec2a94e..20ee58feac 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -3,9 +3,10 @@ from util.log import * from util.sql import * from util.cases import * -import tzlocal import platform import os +if platform.system().lower() == 'windows': + import tzlocal class TDTestCase: From fb26a97fae97352063e4e764e72dca67182f2556 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 9 Jun 2022 15:45:31 +0800 Subject: [PATCH 109/132] fix:error in schemaless --- source/client/src/clientSml.c | 2 +- source/client/test/smlTest.cpp | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index fe21d48a65..93e193f0ce 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1556,7 +1556,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV return TSDB_CODE_TSC_INVALID_TIME_STAMP; } *tsVal = timeDouble; - } else if (typeLen == 2 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { + } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { case 'm': case 'M': diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 8c382339b1..8ec411fbff 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1340,12 +1340,6 @@ TEST(testCase, sml_16368_Test) { pRes = taos_query(taos, "use d16368"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); - ASSERT_NE(request, nullptr); - - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - const char *sql[] = { "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, \"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": \"binary\"}}},\n" "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" @@ -1357,9 +1351,7 @@ TEST(testCase, sml_16368_Test) { "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]" }; - int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); - ASSERT_EQ(ret, 0); - - destroyRequest(request); - smlDestroyInfo(info); + pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); } From daa2e868877fa2198a6100ea9eb58ef1dcb76f9b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Jun 2022 15:53:34 +0800 Subject: [PATCH 110/132] enh: refactor index code --- source/libs/index/inc/indexComm.h | 4 ++-- source/libs/index/src/index.c | 8 ++++---- source/libs/index/src/indexCache.c | 4 ++-- source/libs/index/src/indexComm.c | 16 ++++++++-------- source/libs/index/src/indexFilter.c | 2 +- source/libs/index/src/indexTfile.c | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index bccba98116..09fd2f3555 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -34,11 +34,11 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -__compar_fn_t indexGetCompar(int8_t type); +__compar_fn_t idxGetCompar(int8_t type); TExeCond tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType); TExeCond tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b); -_cache_range_compare indexGetCompare(RangeType ty); +_cache_range_compare idxGetCompare(RangeType ty); int32_t idxConvertData(void* src, int8_t type, void** dst); int32_t idxConvertDataToStr(void* src, int8_t type, void** dst); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index bbcef4c3e1..1240757107 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -465,8 +465,8 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { IndexCache* pCache = (IndexCache*)cache; - while (quit && atomic_load_32(&pCache->merging) == 1) { - } + while (quit && atomic_load_32(&pCache->merging) == 1) + ; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); if (pReader == NULL) { indexWarn("empty tfile reader found"); @@ -561,7 +561,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { value->colVal = NULL; } -static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) { +static int64_t idxGetAvailableVer(SIndex* sIdx, IndexCache* cache) { ICacheKey key = {.suid = cache->suid, .colName = cache->colName, .nColName = strlen(cache->colName)}; int64_t ver = CACHE_VERSION(cache); @@ -579,7 +579,7 @@ static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) { return ver; } static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { - int64_t version = indexGetAvaialbleVer(sIdx, cache); + int64_t version = idxGetAvailableVer(sIdx, cache); indexInfo("file name version: %" PRId64 "", version); uint8_t colType = cache->type; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 1ddb5604c8..2158b9ca21 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -127,7 +127,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* MemTable* mem = cache; IndexCache* pCache = mem->pCache; - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); pCt->colVal = term->colVal; @@ -257,7 +257,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR if (cache == NULL) { return 0; } - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index be9243b8dd..cf327177a7 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -75,35 +75,35 @@ char* idxInt2str(int64_t val, char* dst, int radix) { ; return dst - 1; } -__compar_fn_t indexGetCompar(int8_t type) { +__compar_fn_t idxGetCompar(int8_t type) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); } static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_THAN, a, b, type); } static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_THAN, a, b, type); } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } static TExeCond tCompareContains(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } static TExeCond tCompareEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = indexGetCompar(type); + __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { @@ -205,7 +205,7 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = { tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains, tCompareEqual}; -_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } +_cache_range_compare idxGetCompare(RangeType ty) { return rangeCompare[ty]; } char* idxPackJsonData(SIndexTerm* itm) { /* diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index e4af4a7a3f..bd78ec574a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -318,7 +318,7 @@ int sifLessThan(void *a, void *b, int16_t dtype) { } int sifEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); - //__compar_fn_t func = indexGetCompar(dtype); + //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 619f89cc31..34ebd64855 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -305,7 +305,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, int ret = 0; char* p = tem->colVal; int skip = 0; - _cache_range_compare cmpFn = indexGetCompare(type); + _cache_range_compare cmpFn = idxGetCompare(type); SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); @@ -431,7 +431,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt p = idxPackJsonDataPrefix(tem, &skip); } - _cache_range_compare cmpFn = indexGetCompare(ctype); + _cache_range_compare cmpFn = idxGetCompare(ctype); SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); From bd4d5eca0bf83d21c4649e8e7dc2a8b67d59f8c3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 15:55:04 +0800 Subject: [PATCH 111/132] comment out monitor test for windows --- tests/system-test/fulltest.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index 6ef66c58c2..89caab49d6 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -3,7 +3,7 @@ python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py -python3 .\test.py -f 0-others\taosdMonitor.py +@REM python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py python3 .\test.py -f 0-others\udf_restart_taosd.py @@ -100,4 +100,4 @@ python3 .\test.py -f 0-others\udf_restart_taosd.py @REM python3 .\test.py -f 7-tmq\subscribeStb2.py @REM python3 .\test.py -f 7-tmq\subscribeStb3.py @REM python3 .\test.py -f 7-tmq\subscribeStb4.py -@REM python3 .\test.py -f 7-tmq\db.py \ No newline at end of file +@REM python3 .\test.py -f 7-tmq\db.py From f2b98f773542b9f360d26d8ec05828fe0d151800 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 9 Jun 2022 15:57:53 +0800 Subject: [PATCH 112/132] docs: refine Grafana plugin installation --- docs-cn/20-third-party/01-grafana.mdx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs-cn/20-third-party/01-grafana.mdx b/docs-cn/20-third-party/01-grafana.mdx index 2daaebc32d..31cd434080 100644 --- a/docs-cn/20-third-party/01-grafana.mdx +++ b/docs-cn/20-third-party/01-grafana.mdx @@ -3,6 +3,9 @@ sidebar_label: Grafana title: Grafana --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/) 快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine 中数据表的内容可以在仪表盘(DashBoard)上进行可视化展现。关于 TDengine 插件的使用您可以在[GitHub](https://github.com/taosdata/grafanaplugin/blob/master/README.md)中了解更多。 ## 前置条件 @@ -23,9 +26,12 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/ ## 配置 Grafana -### 安装方式一：安装脚本 +### 安装 Grafana Plugin 并配置数据源 -将集群信息设置为环境变量（也可以使用 [`.env`(dotenv) 文件](https://hexdocs.pm/dotenvy/dotenv-file-format.html) ）： + + + +将集群信息设置为环境变量；也可以使用 `.env` 文件，请参考 [dotenv](https://hexdocs.pm/dotenvy/dotenv-file-format.html)： ```sh export TDENGINE_API=http://tdengine.local:6041 @@ -42,7 +48,8 @@ bash -c "$(curl -fsSL https://raw.githubusercontent.com/taosdata/grafanaplugin/m 该脚本将自动安装 Grafana 插件并配置数据源。安装完毕后，需要重启 Grafana 服务后生效。 -### 安装方式二：手动安装 TDengine 数据源插件 + + 使用 [`grafana-cli` 命令行工具](https://grafana.com/docs/grafana/latest/administration/cli/) 进行插件[安装](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation)。 @@ -94,6 +101,9 @@ GF_INSTALL_PLUGINS=tdengine-datasource ![TDengine Database Grafana plugin add data source](./add_datasource4.webp) + + + ### 创建 Dashboard 回到主界面创建 Dashboard，点击 Add Query 进入面板查询页面： From efad6dc174e0b2fae087677e6b0b3212b169eff3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 9 Jun 2022 16:14:14 +0800 Subject: [PATCH 113/132] docs: refine Grafana plugin installation --- docs-en/20-third-party/01-grafana.mdx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs-en/20-third-party/01-grafana.mdx b/docs-en/20-third-party/01-grafana.mdx index 1e1b49703e..033ef3b206 100644 --- a/docs-en/20-third-party/01-grafana.mdx +++ b/docs-en/20-third-party/01-grafana.mdx @@ -3,6 +3,9 @@ sidebar_label: Grafana title: Grafana --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + TDengine can be quickly integrated with the open-source data visualization system [Grafana](https://www.grafana.com/) to build a data monitoring and alerting system. The whole process does not require any code development. And you can visualize the contents of the data tables in TDengine on a dashboard. You can learn more about using the TDengine plugin on [GitHub](https://github.com/taosdata/grafanaplugin/blob/master/README.md). @@ -25,7 +28,10 @@ TDengine currently supports Grafana versions 7.5 and above. Users can go to the ## Configuring Grafana -### Option 1: Install with `install.sh` +### Install Grafana Plugin and Configure Data Source + + + Set the url and authorization environment variables by `export` or a [`.env`(dotenv) file](https://hexdocs.pm/dotenvy/dotenv-file-format.html): @@ -46,7 +52,8 @@ With this script, TDengine data source plugin and the Grafana data source will b And then, restart Grafana service and open Grafana in web-browser, usually . -### Option 2: Install Plugin Manually + + Follow the installation steps in [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) with the [``grafana-cli`` command-line tool](https://grafana.com/docs/grafana/latest/administration/cli/) for plugin installation. @@ -98,6 +105,9 @@ Click `Save & Test` to test. You should see a success message if the test worked ![TDengine Database TDinsight plugin add database 4](./grafana/add_datasource4.webp) + + + ### Create Dashboard Go back to the main interface to create a dashboard and click Add Query to enter the panel query page: From 8997f2cdc1507e2594f126e9e0cca2a6450c7084 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 9 Jun 2022 16:26:30 +0800 Subject: [PATCH 114/132] fix: solve a non-standard explicit type conversion --- source/common/test/dataformatTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 4be0711546..a52bb6b516 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -402,7 +402,7 @@ static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) { TEST(testCase, AllNormTest) { int16_t nCols = 14; - STSRowBuilder rb = tsRowBuilderInit(); + STSRowBuilder rb = {0}; STSRow2 *row = nullptr; SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); From 9a61a567c5a255d46ad49e4970ffc4c997119651 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 16:32:37 +0800 Subject: [PATCH 115/132] refactor(sync): set gRaftDetailLog false --- source/libs/sync/src/syncAppendEntriesReply.c | 2 +- source/libs/sync/src/syncSnapshot.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 150b6f2410..5a543e1605 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -179,7 +179,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries snapshotSenderStart(pSender); char* s = snapshotSender2Str(pSender); - sInfo("snapshot send start sender first time, sender:%s", s); + sInfo("sync event snapshot send start sender first time, sender:%s", s); taosMemoryFree(s); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 0ceea49112..a68312d07f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -109,8 +109,8 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { char host[128]; uint16_t port; syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port); - sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack, - msgStr); + sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", host, + port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr); taosMemoryFree(msgStr); syncSnapshotSendDestroy(pMsg); @@ -234,8 +234,9 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr); } else { - sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d send msg:%s", host, port, pSender->seq, - pSender->ack, msgStr); + sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", + host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + msgStr); } taosMemoryFree(msgStr); From b23fa19bc4954ce79352ab4d4e6fd1cb1eaa43fc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 9 Jun 2022 16:50:11 +0800 Subject: [PATCH 116/132] feat(stream): sink table can be other db --- source/common/src/tdatablock.c | 11 +++---- source/dnode/mnode/impl/src/mndScheduler.c | 8 ++--- source/libs/wal/src/walWrite.c | 4 ++- source/os/src/osFile.c | 35 ++++++++++------------ 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a3a94c50f2..509e025363 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1751,7 +1751,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); - if (!pTag) { + if (pTag == NULL) { tdDestroySVCreateTbReq(&createTbReq); taosArrayDestroy(tagArray); return NULL; @@ -1762,9 +1762,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); tdDestroySVCreateTbReq(&createTbReq); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); taosArrayDestroy(tagArray); return NULL; } @@ -1803,8 +1801,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo int32_t schemaLen = 0; if (createTb) { SVCreateTbReq createTbReq = {0}; - char* cname = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - snprintf(cname, TSDB_TABLE_FNAME_LEN, "%s:%ld", stbFullName, pDataBlock->info.groupId); + char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); createTbReq.name = cname; createTbReq.flags = 0; createTbReq.type = TSDB_CHILD_TABLE; @@ -1818,7 +1815,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayPush(tagArray, &tagVal); STag* pTag = NULL; tTagNew(tagArray, 1, false, &pTag); - if (!pTag) { + if (pTag == NULL) { tdDestroySVCreateTbReq(&createTbReq); taosArrayDestroy(tagArray); taosMemoryFreeClear(ret); @@ -1944,7 +1941,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { blockDataEnsureCapacity(pBlock, numOfRows); - pBlock->info.rows = numOfRows; const char* pStart = pData; @@ -2018,6 +2014,7 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t pStart += colLen[i]; } + pBlock->info.rows = numOfRows; ASSERT(pStart - pData == dataLen); return pStart; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3d5dab1894..bac76a19d3 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -186,7 +186,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; - if (pVgroup->dbUid != pStream->dbUid) { + if (strcmp(pVgroup->dbName, pStream->targetDb) != 0) { sdbRelease(pSdb, pVgroup); continue; } @@ -286,7 +286,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); bool hasExtraSink = false; - if (totLevel == 2 || strcmp(pStream->sourceDb, pStream->targetDb) != 0) { + bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; + if (totLevel == 2 || externalTargetDB) { SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); // add extra sink @@ -405,7 +406,6 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { if (pStream->fixedSinkVgId == 0) { pTask->dispatchType = TASK_DISPATCH__SHUFFLE; - /*pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;*/ pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); ASSERT(pDb); @@ -426,10 +426,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); - /*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/ if (pLastLevelTask->nodeId == pVgInfo->vgId) { pVgInfo->taskId = pLastLevelTask->taskId; - /*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/ break; } } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a9a8f8a1f4..793ab8b2fb 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -253,7 +253,9 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); + /*int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_CUR);*/ + /*wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);*/ + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 32fed66cd4..05b7498cc0 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -174,9 +174,9 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { struct stat fileStat; #ifdef WINDOWS - int32_t code = _stat(path, &fileStat); + int32_t code = _stat(path, &fileStat); #else - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); #endif if (code < 0) { return code; @@ -201,7 +201,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #ifdef WINDOWS BY_HANDLE_FILE_INFORMATION bhfi; - HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); + HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); if (GetFileInformationByHandle(handle, &bhfi) == FALSE) { printf("taosFStatFile get file info fail."); return -1; @@ -216,7 +216,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { } #else - + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { @@ -238,7 +238,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } -TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { @@ -316,12 +316,12 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { (*ppFile)->fp = NULL; } if ((*ppFile)->fd >= 0) { - #ifdef WINDOWS +#ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle((*ppFile)->fd); !FlushFileBuffers(h); - #else +#else fsync((*ppFile)->fd); - #endif +#endif close((*ppFile)->fd); (*ppFile)->fd = -1; } @@ -345,11 +345,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { char *tbuf = (char *)buf; while (leftbytes > 0) { - #ifdef WINDOWS +#ifdef WINDOWS readbytes = _read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); - #else +#else readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); - #endif +#endif if (readbytes < 0) { if (errno == EINTR) { continue; @@ -433,9 +433,6 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { - if (pFile == NULL) { - return 0; - } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif @@ -459,9 +456,9 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { struct stat fileStat; #ifdef WINDOWS - int32_t code = _fstat(pFile->fd, &fileStat); + int32_t code = _fstat(pFile->fd, &fileStat); #else - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); #endif if (code < 0) { return code; @@ -565,12 +562,12 @@ int32_t taosFsyncFile(TdFilePtr pFile) { if (pFile->fp != NULL) return fflush(pFile->fp); if (pFile->fd >= 0) { - #ifdef WINDOWS +#ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); return !FlushFileBuffers(h); - #else +#else return fsync(pFile->fd); - #endif +#endif } return 0; } From 6ba0f160b7c01e3aca82a71098fe6ad0d1f25081 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 17:10:31 +0800 Subject: [PATCH 117/132] fix(sync): compatibility for both mnode(use snapshot) and vnode(no snapshot) --- tests/script/tsim/mnode/basic3.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index ecd663a2d3..dec036faaf 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -39,10 +39,10 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql create mnode on dnode 3 +return system sh/exec.sh -n dnode1 -s stop -x SIGKILL sql_error create mnode on dnode 4 -return $x = 0 step2: @@ -150,4 +150,4 @@ endi system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode3 -s stop -system sh/exec.sh -n dnode4 -s stop \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop From 3148df2107512f3ea5af772d12f16d7fe2f13b2e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 9 Jun 2022 17:16:06 +0800 Subject: [PATCH 118/132] fix(sync): compatibility for both mnode(use snapshot) and vnode(no snapshot) --- source/libs/sync/src/syncMain.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bdf89a8914..9516df64da 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1215,13 +1215,14 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) { // update sender private term SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); - ASSERT(pMySender != NULL); - for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { - if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { - pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + if (pMySender != NULL) { + for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) { + if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) { + pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm; + } } + (pMySender->privateTerm) += 100; } - (pMySender->privateTerm) += 100; // stop elect timer syncNodeStopElectTimer(pSyncNode); From 32591162b48bc7c9b8fc4d6ed848ccb641959474 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 17:17:10 +0800 Subject: [PATCH 119/132] fix(query): add to_iso8601 function timezone validation TD-16372 --- source/libs/function/src/builtins.c | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index eca4587c03..b628128670 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1020,6 +1020,22 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { * */ +static bool validateHourRange(int8_t hour) { + if (hour < 0 || hour > 12) { + return false; + } + + return true; +} + +static bool validateMinuteRange(int8_t hour, int8_t minute) { + if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5))) { + return true; + } + + return false; +} + static bool validateTimezoneFormat(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; @@ -1028,6 +1044,8 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { char* tz = varDataVal(pVal->datum.p); int32_t len = varDataLen(pVal->datum.p); + char buf[3] = {0}; + int8_t hour = -1, minute = -1; if (len == 0) { return false; } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) { @@ -1040,6 +1058,20 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { if (!isdigit(tz[i])) { return false; } + + if (i == 2) { + memcpy(buf, &tz[i - 1], 2); + hour = taosStr2Int8(buf, NULL, 10); + if (!validateHourRange(hour)) { + return false; + } + } else if (i == 4) { + memcpy(buf, &tz[i - 1], 2); + minute = taosStr2Int8(buf, NULL, 10); + if (!validateMinuteRange(hour, minute)) { + return false; + } + } } break; } @@ -1051,9 +1083,24 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } continue; } + if (!isdigit(tz[i])) { return false; } + + if (i == 2) { + memcpy(buf, &tz[i - 1], 2); + hour = taosStr2Int8(buf, NULL, 10); + if (!validateHourRange(hour)) { + return false; + } + } else if (i == 5) { + memcpy(buf, &tz[i - 1], 2); + minute = taosStr2Int8(buf, NULL, 10); + if (!validateMinuteRange(hour, minute)) { + return false; + } + } } break; } From f057cd1943d766e285f5693347e9cdf752b3ba97 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 9 Jun 2022 17:22:42 +0800 Subject: [PATCH 120/132] enh: refactor idx module --- source/libs/index/inc/indexInt.h | 22 +++++++++++----------- source/libs/index/src/index.c | 30 +++++++++++++++--------------- source/libs/index/src/indexCache.c | 20 ++++++++++---------- source/libs/index/src/indexComm.c | 6 +++--- source/libs/index/src/indexJson.c | 4 ++-- source/libs/index/src/indexTfile.c | 10 +++++----- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index b6bfb25c24..906cbb6a20 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -133,24 +133,24 @@ typedef struct TFileCacheKey { } ICacheKey; int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit); -int64_t indexAddRef(void* p); -int32_t indexRemoveRef(int64_t ref); -void indexAcquireRef(int64_t ref); -void indexReleaseRef(int64_t ref); +int64_t idxAddRef(void* p); +int32_t idxRemoveRef(int64_t ref); +void idxAcquireRef(int64_t ref); +void idxReleaseRef(int64_t ref); int32_t idxSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) +#define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F) -#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ - do { \ - uint8_t oldTy = ty; \ - ty = (ty >> 4) | exTy; \ - ty = (ty << 4) | oldTy; \ +#define IDX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ + do { \ + uint8_t oldTy = ty; \ + ty = (ty >> 4) | exTy; \ + ty = (ty << 4) | oldTy; \ } while (0) #ifdef __cplusplus diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 1240757107..04d7e04b30 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -90,7 +90,7 @@ static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateV // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); -static void indexPost(void* idx) { +static void idxPost(void* idx) { SIndex* pIdx = idx; tsem_post(&pIdx->sem); } @@ -118,8 +118,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadMutexInit(&sIdx->mtx, NULL); tsem_init(&sIdx->sem, 0, 0); - sIdx->refId = indexAddRef(sIdx); - indexAcquireRef(sIdx->refId); + sIdx->refId = idxAddRef(sIdx); + idxAcquireRef(sIdx->refId); *index = sIdx; return 0; @@ -157,23 +157,23 @@ void indexClose(SIndex* sIdx) { taosHashCleanup(sIdx->colObj); sIdx->colObj = NULL; } - indexReleaseRef(sIdx->refId); - indexRemoveRef(sIdx->refId); + idxReleaseRef(sIdx->refId); + idxRemoveRef(sIdx->refId); } -int64_t indexAddRef(void* p) { +int64_t idxAddRef(void* p) { // impl return taosAddRef(indexRefMgt, p); } -int32_t indexRemoveRef(int64_t ref) { +int32_t idxRemoveRef(int64_t ref) { // impl later return taosRemoveRef(indexRefMgt, ref); } -void indexAcquireRef(int64_t ref) { +void idxAcquireRef(int64_t ref) { // impl taosAcquireRef(indexRefMgt, ref); } -void indexReleaseRef(int64_t ref) { +void idxReleaseRef(int64_t ref) { // impl taosReleaseRef(indexRefMgt, ref); } @@ -289,7 +289,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->nColName = nColName; char* buf = NULL; - int32_t len = idxConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf); + int32_t len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf); assert(len != -1); tm->colVal = buf; @@ -479,9 +479,9 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { tfileReaderUnRef(pReader); atomic_store_32(&pCache->merging, 0); if (quit) { - indexPost(sIdx); + idxPost(sIdx); } - indexReleaseRef(sIdx->refId); + idxReleaseRef(sIdx->refId); return 0; } @@ -542,9 +542,9 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { } atomic_store_32(&pCache->merging, 0); if (quit) { - indexPost(sIdx); + idxPost(sIdx); } - indexReleaseRef(sIdx->refId); + idxReleaseRef(sIdx->refId); return ret; } @@ -621,7 +621,7 @@ END: } int32_t idxSerialCacheKey(ICacheKey* key, char* buf) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); char* p = buf; char tbuf[65] = {0}; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 2158b9ca21..040e8ed830 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -187,7 +187,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr pCt->version = atomic_load_64(&pCache->version); char* exBuf = NULL; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { exBuf = idxPackJsonData(term); pCt->colVal = exBuf; } @@ -266,7 +266,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR pCt->colVal = term->colVal; pCt->version = atomic_load_64(&pCache->version); - int8_t dType = INDEX_TYPE_GET_TYPE(term->colType); + int8_t dType = IDX_TYPE_GET_TYPE(term->colType); int skip = 0; char* exBuf = NULL; if (type == CONTAINS) { @@ -342,7 +342,7 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 cache->mem = idxInternalCacheCreate(type); cache->mem->pCache = cache; - cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); + cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; @@ -354,7 +354,7 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 idxCacheRef(cache); if (idx != NULL) { - indexAcquireRef(idx->refId); + idxAcquireRef(idx->refId); } return cache; } @@ -455,7 +455,7 @@ void idxCacheDestroy(void* cache) { taosThreadMutexDestroy(&pCache->mtx); taosThreadCondDestroy(&pCache->finished); if (pCache->index != NULL) { - indexReleaseRef(((SIndex*)pCache->index)->refId); + idxReleaseRef(((SIndex*)pCache->index)->refId); } taosMemoryFree(pCache); } @@ -500,7 +500,7 @@ int idxCacheSchedToMerge(IndexCache* pCache, bool notify) { schedMsg.thandle = taosMemoryMalloc(1); } schedMsg.msg = NULL; - indexAcquireRef(pCache->index->refId); + idxAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); return 0; } @@ -533,7 +533,7 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (cache == NULL) { return -1; } - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); IndexCache* pCache = cache; idxCacheRef(pCache); @@ -597,7 +597,7 @@ static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { return cacheSearch[1][qtype](mem, term, tr, s); } else { return cacheSearch[0][qtype](mem, term, tr, s); @@ -730,9 +730,9 @@ static int32_t idxCacheJsonTermCompare(const void* l, const void* r) { return cmp; } static MemTable* idxInternalCacheCreate(int8_t type) { - int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; + int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; int32_t (*cmpFn)(const void* l, const void* r) = - INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; + IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); idxMemRef(tbl); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index cf327177a7..99b49f97bd 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -212,7 +212,7 @@ char* idxPackJsonData(SIndexTerm* itm) { * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); @@ -240,7 +240,7 @@ char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); @@ -267,7 +267,7 @@ char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) { * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ - uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c index 88b3d907bb..8ce625dfb9 100644 --- a/source/libs/index/src/indexJson.c +++ b/source/libs/index/src/indexJson.c @@ -30,7 +30,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { } else { p->colType = TSDB_DATA_TYPE_DOUBLE; } - INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } // handle put return indexPut(index, terms, uid); @@ -48,7 +48,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res } else { p->colType = TSDB_DATA_TYPE_DOUBLE; } - INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); + IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON); } // handle search return indexSearch(index, tq, result); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 34ebd64855..d632540ee1 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -281,7 +281,7 @@ static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { return 0; } static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; char* p = tem->colVal; @@ -457,7 +457,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt } else if (0 != strncmp(ch, p, skip)) { continue; } - cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); + cond = cmpFn(ch + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType)); } if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); @@ -476,7 +476,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; int ret = 0; - if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { ret = tfSearch[1][qtype](reader, term, tr); } else { ret = tfSearch[0][qtype](reader, term, tr); @@ -536,7 +536,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { __compar_fn_t fn; int8_t colType = tw->header.colType; - colType = INDEX_TYPE_GET_TYPE(colType); + colType = IDX_TYPE_GET_TYPE(colType); if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { fn = tfileStrCompare; } else { @@ -845,7 +845,7 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { TFileHeader* header = &write->header; uint8_t colType = header->colType; - colType = INDEX_TYPE_GET_TYPE(colType); + colType = IDX_TYPE_GET_TYPE(colType); FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); if (fstBuilderInsert(write->fb, key, tval->offset)) { fstSliceDestroy(&key); From cbfa116d069c10a42360bcd6504e4080d2d236be Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 17:17:10 +0800 Subject: [PATCH 121/132] fix(query): add to_iso8601 function timezone validation TD-16372 --- source/libs/function/src/builtins.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b628128670..04b171b9af 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1028,8 +1028,8 @@ static bool validateHourRange(int8_t hour) { return true; } -static bool validateMinuteRange(int8_t hour, int8_t minute) { - if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5))) { +static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { + if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '-')) { return true; } @@ -1068,7 +1068,7 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } else if (i == 4) { memcpy(buf, &tz[i - 1], 2); minute = taosStr2Int8(buf, NULL, 10); - if (!validateMinuteRange(hour, minute)) { + if (!validateMinuteRange(hour, minute, tz[0])) { return false; } } @@ -1097,7 +1097,7 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } else if (i == 5) { memcpy(buf, &tz[i - 1], 2); minute = taosStr2Int8(buf, NULL, 10); - if (!validateMinuteRange(hour, minute)) { + if (!validateMinuteRange(hour, minute, tz[0])) { return false; } } From 4da408afe6466795a844827c5b0ebc0a8c30002d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 17:34:56 +0800 Subject: [PATCH 122/132] enh: call walEndSnapshot while sdb writefile --- source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/dnode/mnode/impl/src/mndMain.c | 34 +++++++++++++++++-- source/dnode/mnode/impl/src/mndSync.c | 46 ++++++++++---------------- source/dnode/mnode/sdb/inc/sdb.h | 3 ++ source/dnode/mnode/sdb/src/sdb.c | 2 ++ source/dnode/mnode/sdb/src/sdbFile.c | 13 +++++++- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 7 files changed, 69 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 2bee0599f5..4869a19856 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -76,7 +76,6 @@ typedef struct { } STelemMgmt; typedef struct { - SWal *pWal; sem_t syncSem; int64_t sync; bool standby; @@ -109,6 +108,7 @@ typedef struct SMnode { SQHandle *pQuery; SHashObj *infosMeta; SHashObj *perfsMeta; + SWal *pWal; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8cb0da0a59..813e4c30b5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -139,10 +139,40 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) { return 0; } +static int32_t mndInitWal(SMnode *pMnode) { + char path[PATH_MAX + 20] = {0}; + snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); + SWalCfg cfg = { + .vgId = 1, + .fsyncPeriod = 0, + .rollPeriod = -1, + .segSize = -1, + .retentionPeriod = -1, + .retentionSize = -1, + .level = TAOS_WAL_FSYNC, + }; + + pMnode->pWal = walOpen(path, &cfg); + if (pMnode->pWal == NULL) { + mError("failed to open wal since %s", terrstr()); + return -1; + } + + return 0; +} + +static void mndCloseWal(SMnode *pMnode) { + if (pMnode->pWal != NULL) { + walClose(pMnode->pWal); + pMnode->pWal = NULL; + } +} + static int32_t mndInitSdb(SMnode *pMnode) { SSdbOpt opt = {0}; opt.path = pMnode->path; opt.pMnode = pMnode; + opt.pWal = pMnode->pWal; pMnode->pSdb = sdbInit(&opt); if (pMnode->pSdb == NULL) { @@ -156,7 +186,6 @@ static int32_t mndOpenSdb(SMnode *pMnode) { if (!pMnode->deploy) { return sdbReadFile(pMnode->pSdb); } else { - // return sdbDeploy(pMnode->pSdb);; return 0; } } @@ -182,6 +211,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle } static int32_t mndInitSteps(SMnode *pMnode) { + if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; @@ -201,7 +231,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index df8dc15c4a..a0daa72d9a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,15 +17,27 @@ #include "mndSync.h" #include "mndTrans.h" -int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { +static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); - return tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; } -int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return tmsgSendReq(pEpSet, pMsg); } +static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { + int32_t code = tmsgSendReq(pEpSet, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SMnode *pMnode = pFsm->data; @@ -89,8 +101,8 @@ void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) SSyncMgmt *pMgmt = &pMnode->syncMgmt; pMgmt->errCode = cbMeta.code; - mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, - pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term); + mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId, + cbMeta.code, cbMeta.index, cbMeta.term); if (pMgmt->transId == -1) { if (pMgmt->errCode != 0) { @@ -155,27 +167,9 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); - SWalCfg cfg = { - .vgId = 1, - .fsyncPeriod = 0, - .rollPeriod = -1, - .segSize = -1, - .retentionPeriod = -1, - .retentionSize = -1, - .level = TAOS_WAL_FSYNC, - }; - - pMgmt->pWal = walOpen(path, &cfg); - if (pMgmt->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); - return -1; - } - SSyncInfo syncInfo = {.vgId = 1, .FpSendMsg = mndSyncSendMsg, .FpEqMsg = mndSyncEqMsg}; snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); - syncInfo.pWal = pMgmt->pWal; + syncInfo.pWal = pMnode->pWal; syncInfo.pFsm = mndSyncMakeFsm(pMnode); syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotEnable = true; @@ -208,10 +202,6 @@ void mndCleanupSync(SMnode *pMnode) { mDebug("mnode sync is stopped, id:%" PRId64, pMgmt->sync); tsem_destroy(&pMgmt->syncSem); - if (pMgmt->pWal != NULL) { - walClose(pMgmt->pWal); - } - memset(pMgmt, 0, sizeof(SSyncMgmt)); } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 22f3faf620..8536c451b7 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -22,6 +22,7 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -165,6 +166,7 @@ typedef struct SSdbRow { typedef struct SSdb { SMnode *pMnode; + SWal *pWal; char *currDir; char *tmpDir; int64_t lastCommitVer; @@ -206,6 +208,7 @@ typedef struct { typedef struct SSdbOpt { const char *path; SMnode *pMnode; + SWal *pWal; } SSdbOpt; /** diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 170bf3bc90..c11aaaaa8a 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -52,10 +52,12 @@ SSdb *sdbInit(SSdbOpt *pOption) { pSdb->keyTypes[i] = SDB_KEY_INT32; } + pSdb->pWal = pOption->pWal; pSdb->curVer = -1; pSdb->curTerm = -1; pSdb->lastCommitVer = -1; pSdb->lastCommitTerm = -1; + pSdb->curConfig = -1; pSdb->pMnode = pOption->pMnode; taosThreadMutexInit(&pSdb->filelock, NULL); mDebug("sdb init successfully"); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index bacf4e09bf..f98ecf5343 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -441,12 +441,23 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t sdbWriteFile(SSdb *pSdb) { + int32_t code = 0; if (pSdb->curVer == pSdb->lastCommitVer) { return 0; } taosThreadMutexLock(&pSdb->filelock); - int32_t code = sdbWriteFileImp(pSdb); + if (pSdb->pWal != NULL) { + code = walBeginSnapshot(pSdb->pWal, pSdb->curVer); + } + if (code == 0) { + code = sdbWriteFileImp(pSdb); + } + if (code == 0) { + if (pSdb->pWal != NULL) { + code = walEndSnapshot(pSdb->pWal); + } + } if (code != 0) { mError("failed to write sdb file since %s", terrstr()); } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 9fb1e9a2b3..816c0cfac9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -252,7 +252,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { - .isStandBy = false, + .isStandBy = false, .snapshotEnable = false, .vgId = pVnode->config.vgId, .isStandBy = pVnode->config.standby, From 016a7b698fbc162bb60af434c87f78222481c32b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 9 Jun 2022 17:18:17 +0800 Subject: [PATCH 123/132] feat(stream): stream state window support top and bottom --- include/libs/function/function.h | 1 + source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/executorimpl.c | 33 +++--- source/libs/executor/src/timewindowoperator.c | 32 +++++- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtins.c | 4 +- source/libs/function/src/builtinsimpl.c | 76 ++++++++++++- tests/script/tsim/stream/session0.sim | 105 +++++++++++++++++- 8 files changed, 235 insertions(+), 24 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 810991f770..e8a308a7a7 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -146,6 +146,7 @@ typedef struct SqlFunctionCtx { struct SDiskbasedBuf *pBuf; struct SSDataBlock *pSrcBlock; int32_t curBufPage; + bool increase; char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 68d71a0c24..348f0224b6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -804,7 +804,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -892,8 +892,8 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey); -int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey); +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, + SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t size); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b4579b90b2..a2f673e59b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1130,6 +1130,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; + pCtx->increase = false; pCtx->param = pFunct->pParam; // for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -2008,8 +2009,16 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + if (pCtx[j].increase) { + int64_t ts = *(int64_t*) in; + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, (const char *)&ts, pCtx[j].resultInfo->isNullRes); + ts++; + } + } else { + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } } } } @@ -4676,7 +4685,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo); + bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { int32_t children = 8; @@ -5339,7 +5349,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) { +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t size) { + pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); pSup->pResultRows = taosArrayInit(1024, size); @@ -5358,15 +5369,11 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size if (bufSize <= pageSize) { bufSize = pageSize * 4; } - return createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH); -} - -int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { - return initStreamAggSupporter(pSup, pKey, sizeof(SResultWindowInfo)); -} - -int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { - return initStreamAggSupporter(pSup, pKey, sizeof(SStateWindowInfo)); + int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH); + for(int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].pBuf = pSup->pResultBuf; + } + return code; } int64_t getSmaWaterMark(int64_t interval, double filesFactor) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b309478556..29e5630a50 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1438,9 +1438,15 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt return needed; } +void increaseTs(SqlFunctionCtx* pCtx) { + if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTARTTS) { + pCtx[0].increase = true; + } +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1460,6 +1466,11 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + + if (isStream) { + ASSERT(numOfCols > 0); + increaseTs(pInfo->binfo.pCtx); + } initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); @@ -2128,6 +2139,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + ASSERT(numOfCols > 0); + increaseTs(pInfo->binfo.pCtx); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2212,6 +2225,8 @@ int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, for (int32_t i = 0; i < numOfCols; ++i) { pBasicInfo->pCtx[i].pBuf = NULL; } + ASSERT(numOfCols > 0); + increaseTs(pBasicInfo->pCtx); return TSDB_CODE_SUCCESS; } @@ -2228,6 +2243,10 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } +int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) { + return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo)); +} + SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { @@ -2244,8 +2263,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols); - code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo"); + + code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pInfo->binfo.pCtx, numOfCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3097,6 +3116,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } +int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) { + return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo)); +} + SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; @@ -3130,8 +3153,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys goto _error; } - pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols); - code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo"); + code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pInfo->binfo.pCtx, numOfCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 614be17e08..8313b0c635 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -102,6 +102,8 @@ bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); int32_t bottomFunction(SqlFunctionCtx *pCtx); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e858fb3159..3d92764339 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1364,6 +1364,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = topFunction, .finalizeFunc = topBotFinalize, + .combineFunc = topCombine, }, { .name = "bottom", @@ -1373,7 +1374,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = functionSetup, .processFunc = bottomFunction, - .finalizeFunc = topBotFinalize + .finalizeFunc = topBotFinalize, + .combineFunc = bottomCombine, }, { .name = "spread", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6e969749c6..1ec41a59f2 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1389,6 +1389,18 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple } } +void releaseSource(STuplePos* pPos) { + if (pPos->pageId == -1) { + return ; + } + // Todo(liuyao) relase row +} + +void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { + releaseSource(pDestPos); + *pDestPos = *pSourcePos; +} + int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SMinmaxResInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -1400,10 +1412,12 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 if (pSBuf->assign && ( (((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign ) ) { *(double*) &pDBuf->v = *(double*) &pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); } } else { if ( pSBuf->assign && ( ((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign ) ) { pDBuf->v = pSBuf->v; + replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos); } } pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); @@ -2856,7 +2870,6 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); - pEntryInfo->complete = true; int32_t type = pCtx->input.pData[0]->info.type; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; @@ -2881,6 +2894,67 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pEntryInfo->numOfRes; } +void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, + bool isTopQuery) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = getTopBotOutputInfo(pCtx); + int32_t maxSize = pCtx->param[1].param.i; + STopBotResItem* pItems = pRes->pItems; + assert(pItems != NULL); + + // not full yet + if (pEntryInfo->numOfRes < maxSize) { + STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; + pItem->v = pSourceItem->v; + pItem->uid = pSourceItem->uid; + pItem->tuplePos.pageId = -1; + replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos); + pEntryInfo->numOfRes++; + taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn, + !isTopQuery); + } else { // replace the minimum value in the result + if ((isTopQuery && ( + (IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i > pItems[0].v.i) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u > pItems[0].v.u) || + (IS_FLOAT_TYPE(type) && pSourceItem->v.d > pItems[0].v.d))) + || (!isTopQuery && ( + (IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i < pItems[0].v.i) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u < pItems[0].v.u) || + (IS_FLOAT_TYPE(type) && pSourceItem->v.d < pItems[0].v.d)) + )) { + // replace the old data and the coresponding tuple data + STopBotResItem* pItem = &pItems[0]; + pItem->v = pSourceItem->v; + pItem->uid = pSourceItem->uid; + + // save the data of this tuple by over writing the old data + replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos); + taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, + topBotResComparFn, NULL, !isTopQuery); + } + } +} + +int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + int32_t type = pDestCtx->input.pData[0]->info.type; + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); + for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { + addResult(pDestCtx, pSBuf->pItems + i, type, true); + } + return TSDB_CODE_SUCCESS; +} + +int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + int32_t type = pDestCtx->input.pData[0]->info.type; + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + STopBotRes* pSBuf = getTopBotOutputInfo(pSourceCtx); + for (int32_t i = 0; i < pSResInfo->numOfRes; i++) { + addResult(pDestCtx, pSBuf->pItems + i, type, false); + } + return TSDB_CODE_SUCCESS; +} + bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SSpreadInfo); return true; diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index a2fe773edb..c021e14de7 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -17,7 +17,7 @@ sql use test sql create table t1(ts timestamp, a int, b int , c int, d double,id int); -sql create stream streams2 trigger at_once into streamt as select _wstartts, count(*) c1, sum(a), max(a), min(d), stddev(a), last(a), first(d), max(id) s from t1 session(ts,10s); +sql create stream streams1 trigger at_once into streamt as select _wstartts, count(*) c1, sum(a), max(a), min(d), stddev(a), last(a), first(d), max(id) s from t1 session(ts,10s); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL,1); sql insert into t1 values(1648791223001,10,2,3,1.1,2); sql insert into t1 values(1648791233002,3,2,3,2.1,3); @@ -176,4 +176,107 @@ if $data08 != 13 then return -1 endi +sql create database test2 vgroups 1; +sql use test2; +sql create table t2(ts timestamp, a int, b int , c int, d double, id int); +sql create stream streams2 trigger at_once watermark 1d into streamt2 as select _wstartts,apercentile(a,30) c1, apercentile(a,70), apercentile(a,20,"t-digest") c2, apercentile(a,60,"t-digest") c3, max(id) c4 from t2 session(ts,10s); +sql insert into t2 values(1648791213001,1,1,3,1.0,1); +sql insert into t2 values(1648791213002,2,2,6,3.4,2); +sql insert into t2 values(1648791213003,4,9,3,4.8,3); +sql insert into t2 values(1648791233003,3,4,3,2.1,4); +sql insert into t2 values(1648791233004,3,5,3,3.4,5); +sql insert into t2 values(1648791233005,3,6,3,7.6,6); + +# +sql insert into t2 values(1648791223003,20,7,3,10.1,7); + +$loop_count = 0 +loop2: +sleep 300 +sql select * from streamt2 where c4=7; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======$rows + goto loop2 +endi + +# row 0 +if $data01 != 2.091607978 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 3.274823935 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 1.800000000 then + print ======$data03 + return -1 +endi + +if $data04 != 3.350000000 then + print ======$data04 + return -1 +endi + +sql create database test3 vgroups 1; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams3 trigger at_once watermark 1d into streamt3 as select _wstartts, min(b), a,c from t1 session(ts,10s); +sql create stream streams4 trigger at_once watermark 1d into streamt4 as select _wstartts, max(b), a,c from t1 session(ts,10s); +sql create stream streams5 trigger at_once watermark 1d into streamt5 as select _wstartts, max(b), a,c from t1 session(ts,10s); +sql create stream streams6 trigger at_once watermark 1d into streamt6 as select _wstartts, max(b), a,c from t1 session(ts,10s); +sql insert into t1 values(1648791213001,1,1,1,1.0); +sql insert into t1 values(1648791213002,2,3,2,3.4); +sql insert into t1 values(1648791213003,4,9,3,4.8); +sql insert into t1 values(1648791213004,4,5,4,4.8); + +sql insert into t1 values(1648791233004,3,4,0,2.1); +sql insert into t1 values(1648791233005,3,0,6,3.4); +sql insert into t1 values(1648791233006,3,6,7,7.6); +sql insert into t1 values(1648791233007,3,13,8,7.6); + + +sql insert into t1 values(1648791223004,20,7,9,10.1); + +$loop_count = 0 +loop3: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt3; +if $rows == 0 then + print ======$rows + goto loop3 +endi + +sql select * from streamt4; +if $rows == 0 then + print ======$rows + goto loop3 +endi + +sql select * from streamt5; +if $rows == 0 then + print ======$rows + goto loop3 +endi + +sql select * from streamt6; +if $rows == 0 then + print ======$rows + goto loop3 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From e65dc65e705fd79c4837953ef9b59046f111db72 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 18:39:00 +0800 Subject: [PATCH 124/132] test: disable mnode case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3d398c8ae7..5a8cf562a0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -57,7 +57,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim -./test.sh -f tsim/mnode/basic2.sim +#./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim From c105f0a4dfd5c0198ce17431c79d86f10e4720fe Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 9 Jun 2022 19:11:44 +0800 Subject: [PATCH 125/132] fix(os): start win32 taosd case --- tests/pytest/util/dnodes.py | 3 +-- tests/system-test/fulltest.bat | 8 ++++---- tests/system-test/test-all.bat | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 56989597ec..0d70ab82c0 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -321,6 +321,7 @@ class TDDnode: self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.start(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index)) self.running = 1 else: + os.system("rm -rf %s/taosdlog.0"%self.logDir) if os.system(cmd) != 0: tdLog.exit(cmd) self.running = 1 @@ -338,8 +339,6 @@ class TDDnode: if i > 50: break tailCmdStr = 'tail -f ' - if platform.system().lower() == 'windows': - tailCmdStr = 'tail -n +0 -f ' popen = subprocess.Popen( tailCmdStr + logFile, stdout=subprocess.PIPE, diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index 89caab49d6..c8084c3ad5 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,12 +1,12 @@ -python3 .\test.py -f 0-others\taosShell.py -python3 .\test.py -f 0-others\taosShellError.py +@REM python3 .\test.py -f 0-others\taosShell.py +@REM python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py -@REM python3 .\test.py -f 0-others\taosdMonitor.py +python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py -python3 .\test.py -f 0-others\udf_restart_taosd.py +@REM python3 .\test.py -f 0-others\udf_restart_taosd.py @REM python3 .\test.py -f 0-others\cachelast.py @REM python3 .\test.py -f 0-others\user_control.py diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 9c2e58ea3f..4c67404e01 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -2,14 +2,7 @@ SETLOCAL EnableDelayedExpansion for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") set /a a=0 -@REM echo Windows Taosd Test -@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( -@REM echo Processing %%i -@REM set /a a+=1 -@REM call %%i ARG1 > result_!a!.txt 2>error_!a!.txt -@REM if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) -@REM ) -echo Linux Taosd Test +echo Windows Taosd Test for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( echo Processing %%i @@ -17,10 +10,22 @@ for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( set time1=!_timeTemp! echo Start at %time% set /a a+=1 - call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) ) ) +@REM echo Linux Taosd Test +@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( +@REM for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( +@REM echo Processing %%i +@REM call :GetTimeSeconds %time% +@REM set time1=!_timeTemp! +@REM echo Start at %time% +@REM set /a a+=1 +@REM call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt +@REM if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +@REM ) +@REM ) exit :colorEcho From 62506bac334ec4e8fdf4d4f9f8f1abe7c9655438 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 19:15:09 +0800 Subject: [PATCH 126/132] add elapsed function partial/merge translate --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 80 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 4 ++ 4 files changed, 87 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index d77b914fae..03e11b0502 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -131,6 +131,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_HISTOGRAM_MERGE, FUNCTION_TYPE_HYPERLOGLOG_PARTIAL, FUNCTION_TYPE_HYPERLOGLOG_MERGE, + FUNCTION_TYPE_ELAPSED_PARTIAL, + FUNCTION_TYPE_ELAPSED_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 614be17e08..c4f6f0cc45 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -115,6 +115,7 @@ bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t elapsedFunction(SqlFunctionCtx* pCtx); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t getElapsedInfoSize(); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e858fb3159..04cfa9d360 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -444,6 +444,64 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len return TSDB_CODE_SUCCESS; } +static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + + if (isPartial) { + if (1 != numOfParams && 2 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_TIMESTAMP != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + // param1 + if (2 == numOfParams) { + SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1); + if (QUERY_NODE_VALUE != nodeType(pParamNode1)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)pParamNode1; + + pValue->notReserved = true; + + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (pValue->datum.i == 0) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "ELAPSED function time unit parameter should be greater than db precision"); + } + } + + pFunc->node.resType = (SDataType){.bytes = getElapsedInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + } else { + if (1 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateElapsedPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateElapsedImpl(pFunc, pErrBuf, len, true); +} + +static int32_t translateElapsedMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateElapsedImpl(pFunc, pErrBuf, len, false); +} + static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); if (3 != numOfParams) { @@ -1421,6 +1479,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = elapsedFunction, .finalizeFunc = elapsedFinalize }, + { + .name = "_elapsed_partial", + .type = FUNCTION_TYPE_ELAPSED, + .classification = FUNC_MGT_AGG_FUNC, + .dataRequiredFunc = statisDataRequired, + .translateFunc = translateElapsedPartial, + .getEnvFunc = getElapsedFuncEnv, + .initFunc = elapsedFunctionSetup, + .processFunc = elapsedFunction, + .finalizeFunc = elapsedFinalize + }, + { + .name = "_elapsed_merge", + .type = FUNCTION_TYPE_ELAPSED, + .classification = FUNC_MGT_AGG_FUNC, + .dataRequiredFunc = statisDataRequired, + .translateFunc = translateElapsedMerge, + .getEnvFunc = getElapsedFuncEnv, + .initFunc = elapsedFunctionSetup, + .processFunc = elapsedFunction, + .finalizeFunc = elapsedFinalize + }, { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6e969749c6..d202c8f565 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3027,6 +3027,10 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t getElapsedInfoSize() { + return (int32_t)sizeof(SElapsedInfo); +} + bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SElapsedInfo); return true; From d2efa06742194b13a3b94384bafe7bf93fda8e31 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 19:49:29 +0800 Subject: [PATCH 127/132] add elapsed function merge/partial functions --- source/libs/function/inc/builtinsimpl.h | 2 ++ source/libs/function/src/builtins.c | 4 +-- source/libs/function/src/builtinsimpl.c | 42 +++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index c4f6f0cc45..65d2457e5a 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -114,7 +114,9 @@ int32_t getSpreadInfoSize(); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t elapsedFunction(SqlFunctionCtx* pCtx); +int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t getElapsedInfoSize(); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 04cfa9d360..299e45b4b2 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1488,7 +1488,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getElapsedFuncEnv, .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, - .finalizeFunc = elapsedFinalize + .finalizeFunc = elapsedPartialFinalize }, { .name = "_elapsed_merge", @@ -1498,7 +1498,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateElapsedMerge, .getEnvFunc = getElapsedFuncEnv, .initFunc = elapsedFunctionSetup, - .processFunc = elapsedFunction, + .processFunc = elapsedFunctionMerge, .finalizeFunc = elapsedFinalize }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d202c8f565..bd8b6d0582 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3132,6 +3132,30 @@ _elapsed_over: return TSDB_CODE_SUCCESS; } +int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SElapsedInfo* pInputInfo = (SElapsedInfo *)varDataVal(data); + + pInfo->timeUnit = pInputInfo->timeUnit; + if (pInfo->min > pInputInfo->min) { + pInfo->min = pInputInfo->min; + } + + if (pInfo->max < pInputInfo->max) { + pInfo->max = pInputInfo->max; + } + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + return TSDB_CODE_SUCCESS; +} + int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); double result = (double)pInfo->max - (double)pInfo->min; @@ -3140,6 +3164,24 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getElapsedInfoSize(); + char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + int32_t getHistogramInfoSize() { return (int32_t)sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin); } From 2efbb569978bc85b94676297925ca7e8b7949758 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Jun 2022 20:09:37 +0800 Subject: [PATCH 128/132] enable elapsed function splitting --- source/libs/function/src/builtins.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 299e45b4b2..86b141ac54 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1477,7 +1477,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getElapsedFuncEnv, .initFunc = elapsedFunctionSetup, .processFunc = elapsedFunction, - .finalizeFunc = elapsedFinalize + .finalizeFunc = elapsedFinalize, + .pPartialFunc = "_elapsed_partial", + .pMergeFunc = "_elapsed_merge" }, { .name = "_elapsed_partial", From e85e323863fdc8775a9978351991f875df359d10 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 20:46:15 +0800 Subject: [PATCH 129/132] feat: split vgroup --- source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 51 -------- source/dnode/mnode/impl/src/mndVgroup.c | 150 +++++++++++++++++++++++- 3 files changed, 150 insertions(+), 52 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 89f93d30b5..c50279889e 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -41,6 +41,7 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo); int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray); int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId); +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray); void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby); void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 61d6e7be6c..e6c93a9bfd 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -636,57 +636,6 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { - if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { - return -1; - } - } else { - SVgObj newVgroup = {0}; - memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); - mndTransSetSerial(pTrans); - - if (newVgroup.replica < pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, - pVgroup->vnodeGid[0].dnodeId); - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - } else { - mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); - - SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - - SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; - } - - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; - } - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - } - - return 0; -} - static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 5244bc657b..d8012ed6ed 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1166,7 +1166,155 @@ _OVER: return code; } -static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; } +int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) { + return -1; + } + } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + mndTransSetSerial(pTrans); + + if (newVgroup.replica < pDb->cfg.replications) { + mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + } else if (newVgroup.replica > pDb->cfg.replications) { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + + SVnodeGid del1 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + } else { + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + return 0; +} + +static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + return 0; +} + +static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { + int32_t code = -1; + SSdbRaw *pRaw = NULL; + STrans *pTrans = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode, 0); + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); + if (pTrans == NULL) goto _OVER; + mndTransSetSerial(pTrans); + mDebug("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId); + + SVgObj newVg1 = {0}; + memcpy(&newVg1, pVgroup, sizeof(SVgObj)); + mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica, + newVg1.hashBegin, newVg1.hashEnd); + for (int32_t i = 0; i < newVg1.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId); + } + + if (newVg1.replica == 1) { + if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1], true) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + } else if (newVg1.replica == 3) { + SVnodeGid del1 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + } else { + goto _OVER; + } + + SVgObj newVg2 = {0}; + memcpy(&newVg1, &newVg2, sizeof(SVgObj)); + newVg1.replica = 1; + newVg1.hashEnd = (newVg1.hashBegin + newVg1.hashEnd) / 2; + memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); + + newVg2.replica = 1; + newVg2.hashBegin = newVg1.hashEnd + 1; + memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid)); + memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); + + if (mndAddAdjustVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; + if (mndAddAdjustVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; + + // adjust vgroup + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER; + if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER; + +_OVER: + mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + return code; +} + +static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + int32_t vgId = 2; + SUserObj *pUser = NULL; + SVgObj *pVgroup = NULL; + SDbObj *pDb = NULL; + + mDebug("vgId:%d, start to split", vgId); + + pVgroup = mndAcquireVgroup(pMnode, vgId); + if (pVgroup == NULL) goto _OVER; + + pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb == NULL) goto _OVER; + + pUser = mndAcquireUser(pMnode, pReq->conn.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + goto _OVER; + } + + if (mndCheckNodeAuth(pUser) != 0) { + goto _OVER; + } + + code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + mndReleaseUser(pMnode, pUser); + mndReleaseVgroup(pMnode, pVgroup); + mndReleaseDb(pMnode, pDb); + return code; +} static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pSrc, SDnodeObj *pDst) { From 74ced4e0c0e9b1c76b728d23d4ac242e4f5112c4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Jun 2022 20:54:59 +0800 Subject: [PATCH 130/132] feat: split vgroup --- include/common/tmsgdef.h | 1 + source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/src/mndVgroup.c | 5 +++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 14 ++++++++++++++ 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6b8659f0a8..40701640b7 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -194,6 +194,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) TD_NEW_MSG_SEG(TDMT_QND_MSG) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 1118b5b27e..264dc74e36 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -215,6 +215,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 90e4e30261..ee120576c3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -359,6 +359,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d8012ed6ed..76e65ddd92 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -55,6 +55,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_HASHRANGE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); @@ -1267,8 +1268,8 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid)); memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid)); - if (mndAddAdjustVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; - if (mndAddAdjustVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER; // adjust vgroup if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 1524d8fe6f..da3c8e7c93 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -24,6 +24,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp); int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { @@ -163,6 +164,9 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; + case TDMT_VND_ALTER_HASHRANGE: + vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp); + break; case TDMT_VND_ALTER_CONFIG: break; default: @@ -889,3 +893,13 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void return 0; } + +static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); + + // todo + // 1. stop work + // 2. adjust hash range / compact / remove wals / rename vgroups + // 3. reload sync + return 0; +} \ No newline at end of file From 7ab3a1e42c8590071c93fc5c7de7b4faa3fc1b4a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 9 Jun 2022 21:18:06 +0800 Subject: [PATCH 131/132] feat(wal): support restore from snapshot --- include/libs/stream/tstream.h | 2 ++ include/libs/wal/wal.h | 14 +++++++++++- source/common/src/tdatablock.c | 1 + source/libs/wal/inc/walInt.h | 1 + source/libs/wal/src/walMeta.c | 9 ++++++++ source/libs/wal/src/walMgmt.c | 14 +++++++++++- source/libs/wal/src/walWrite.c | 39 ++++++++++++++++++++++++++++++++-- 7 files changed, 76 insertions(+), 4 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6b5eb3b491..31d13d6cf4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -308,9 +308,11 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc if (pTask->sinkType == TASK_SINK__TABLE) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + taosFreeQitem(pBlock); } else if (pTask->sinkType == TASK_SINK__SMA) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + taosFreeQitem(pBlock); } else { ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); taosWriteQitem(pTask->outputQueue->queue, pBlock); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 95af8ac306..c7d1ccd3de 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -141,6 +141,8 @@ typedef struct SWal { // ctl int64_t refId; TdThreadMutex mutex; + // ref + SHashObj *pRefHash; // ref -> SWalRef // path char path[WAL_PATH_LEN]; // reusable write head @@ -184,7 +186,7 @@ int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walEndSnapshot(SWal *); -void walRestoreFromSnapshot(SWal *, int64_t ver); +int32_t walRestoreFromSnapshot(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); // read @@ -199,6 +201,16 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead); int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead); int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead); +typedef struct { + int64_t refId; + int64_t ver; +} SWalRef; + +SWalRef *walOpenRef(SWal *); +void walCloseRef(SWalRef *); +int32_t walRefVer(SWalRef *, int64_t ver); +int32_t walUnrefVer(SWal *); + // deprecated #if 0 int32_t walRead(SWal *, SWalHead **, int64_t ver); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 509e025363..17ae4e2998 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1772,6 +1772,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } // assign data + // TODO ret = taosMemoryCalloc(1, cap + 46); ret = POINTER_SHIFT(ret, 46); ret->header.vgId = vgId; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 84fe2814ff..7ca105ff2b 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -132,6 +132,7 @@ static inline void walResetVer(SWalVer* pVer) { int walLoadMeta(SWal* pWal); int walSaveMeta(SWal* pWal); +int walRemoveMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); int walCheckAndRepairMeta(SWal* pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9aa848a7bb..8e9cb3a84b 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -419,3 +419,12 @@ int walLoadMeta(SWal* pWal) { taosMemoryFree(buf); return code; } + +int walRemoveMeta(SWal* pWal) { + int metaVer = walFindCurMetaVer(pWal); + if (metaVer == -1) return 0; + char fnameStr[WAL_FILE_LEN]; + walBuildMetaName(pWal, metaVer, fnameStr); + taosRemoveFile(fnameStr); + return 0; +} diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 71cd6de73f..9505b02806 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -75,7 +75,7 @@ void walCleanUp() { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = taosMemoryMalloc(sizeof(SWal)); + SWal *pWal = taosMemoryCalloc(1, sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -92,6 +92,13 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + // init ref + pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (pWal->pRefHash == NULL) { + taosMemoryFree(pWal); + return NULL; + } + // open meta walResetVer(&pWal->vers); pWal->pWriteLogTFile = NULL; @@ -100,6 +107,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + taosHashCleanup(pWal->pRefHash); taosMemoryFree(pWal); return NULL; } @@ -115,12 +123,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); taosMemoryFree(pWal); return NULL; } pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { + taosHashCleanup(pWal->pRefHash); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); taosMemoryFree(pWal); @@ -130,6 +140,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { + taosHashCleanup(pWal->pRefHash); taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -175,6 +186,7 @@ void walClose(SWal *pWal) { walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; + taosHashCleanup(pWal->pRefHash); taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 793ab8b2fb..d30e0b6844 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,12 +18,47 @@ #include "tchecksum.h" #include "walInt.h" -void walRestoreFromSnapshot(SWal *pWal, int64_t ver) { - /*pWal->vers.firstVer = -1;*/ +int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { + taosThreadMutexLock(&pWal->mutex); + + void *pIter = NULL; + while (1) { + taosHashIterate(pWal->pRefHash, pIter); + if (pIter == NULL) break; + SWalRef *pRef = (SWalRef *)pIter; + if (pRef->ver != -1) { + taosHashCancelIterate(pWal->pRefHash, pIter); + return -1; + } + } + + taosCloseFile(&pWal->pWriteLogTFile); + taosCloseFile(&pWal->pWriteIdxTFile); + + if (pWal->vers.firstVer != -1) { + int32_t fileSetSize = taosArrayGetSize(pWal->fileInfoSet); + for (int32_t i = 0; i < fileSetSize; i++) { + SWalFileInfo *pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); + taosRemoveFile(fnameStr); + } + } + walRemoveMeta(pWal); + + pWal->writeCur = -1; + pWal->totSize = 0; + pWal->lastRollSeq = -1; + + taosArrayClear(pWal->fileInfoSet); + pWal->vers.firstVer = -1; pWal->vers.lastVer = ver; pWal->vers.commitVer = ver - 1; pWal->vers.snapshotVer = ver - 1; pWal->vers.verInSnapshotting = -1; + + taosThreadMutexUnlock(&pWal->mutex); + return 0; } int32_t walCommit(SWal *pWal, int64_t ver) { From 0e1328ae7f3ceb77f9f95d6ff2bd0695c57701c3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 10 Jun 2022 10:24:56 +0800 Subject: [PATCH 132/132] fix(stream): set child id --- include/common/tmsg.h | 22 +++++++++++----------- source/dnode/mnode/impl/src/mndScheduler.c | 13 ++++++++++--- source/libs/stream/src/streamData.c | 1 + source/libs/stream/src/streamDispatch.c | 1 + 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4a3c4b0c3f..d9c39cefe6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1205,6 +1205,7 @@ typedef struct { int8_t completed; // all results are returned to client int8_t precision; int8_t compressed; + int8_t streamBlockType; int32_t compLen; int32_t numOfRows; int32_t numOfCols; @@ -2493,14 +2494,14 @@ int32_t tSerializeSTableIndexReq(void* buf, int32_t bufLen, STableIndexReq* pReq int32_t tDeserializeSTableIndexReq(void* buf, int32_t bufLen, STableIndexReq* pReq); typedef struct { - int8_t intervalUnit; - int8_t slidingUnit; - int64_t interval; - int64_t offset; - int64_t sliding; - int64_t dstTbUid; - int32_t dstVgId; // for stream - char* expr; + int8_t intervalUnit; + int8_t slidingUnit; + int64_t interval; + int64_t offset; + int64_t sliding; + int64_t dstTbUid; + int32_t dstVgId; // for stream + char* expr; } STableIndexInfo; typedef struct { @@ -2510,7 +2511,6 @@ typedef struct { int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp); - typedef struct { int8_t mqMsgType; int32_t code; @@ -2751,8 +2751,8 @@ typedef struct { char* msg; } SVDeleteReq; -int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq); -int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq); +int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); +int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); typedef struct { int64_t affectedRows; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index bac76a19d3..55ff34f900 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -35,6 +35,13 @@ extern bool tsStreamSchedV; +static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { + int32_t childId = taosArrayGetSize(pArray); + pTask->childId = childId; + taosArrayPush(pArray, &pTask); + return 0; +} + int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen, double filesFactor) { SNode* pAst = NULL; @@ -195,7 +202,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - taosArrayPush(tasks, &pTask); + mndAddTaskToTaskSet(tasks, pTask); pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -235,7 +242,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - taosArrayPush(tasks, &pTask); + mndAddTaskToTaskSet(tasks, pTask); pTask->nodeId = pStream->fixedSinkVgId; #if 0 @@ -378,7 +385,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { return -1; } sdbRelease(pSdb, pVgroup); - taosArrayPush(taskOneLevel, &pTask); + mndAddTaskToTaskSet(taskOneLevel, pTask); } } else { // merge plan diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 473ad32752..6699e86b1e 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -53,6 +53,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock SSDataBlock* pDataBlock = taosArrayGet(pArray, i); blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor + pDataBlock->info.type = pRetrieve->streamBlockType; pDataBlock->info.childId = pReq->sourceChildId; } pData->blocks = pArray; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 16da418677..38a3509e7b 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -72,6 +72,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->precision = TSDB_DEFAULT_PRECISION; pRetrieve->compressed = 0; pRetrieve->completed = 1; + pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);