Merge branch '3.0' into cpwu/3.0

example/src/tmq.c

@@ -101,8 +101,6 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*const char* sql = "select * from tu1";*/
-  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
   /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
   pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
   if (taos_errno(pRes) != 0) {

include/client/taos.h

@@ -222,6 +222,8 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
+DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
 #if 0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);

include/common/tmsg.h

@@ -491,7 +491,7 @@ typedef struct {
   char    intervalUnit;
   char    slidingUnit;
   char
-          offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+      offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
   int8_t  precision;
   int64_t interval;
   int64_t sliding;
@@ -639,13 +639,18 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 
 typedef struct {
-  SArray* epSetList;  // SArray<SEpSet>
+  SArray* addrsList;  // SArray<SQueryNodeAddr>
 } SQnodeListRsp;
 
 int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 void    tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
 
+typedef struct SQueryNodeAddr {
+  int32_t nodeId;  // vgId or qnodeId
+  SEpSet  epSet;
+} SQueryNodeAddr;
+
 typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
@@ -718,7 +723,7 @@ typedef struct {
 
 int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
 int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
-void tFreeSFuncInfo(SFuncInfo *pInfo);
+void    tFreeSFuncInfo(SFuncInfo* pInfo);
 void    tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
 
 typedef struct {
@@ -1283,14 +1288,14 @@ typedef struct {
 } SMVCreateStreamRsp, SMSCreateStreamRsp;
 
 typedef struct {
-  char    name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
+  char   name[TSDB_TOPIC_FNAME_LEN];  // accout.topic
-  int8_t  igExists;
+  int8_t igExists;
-  int8_t  withTbName;
+  int8_t withTbName;
-  int8_t  withSchema;
+  int8_t withSchema;
-  int8_t  withTag;
+  int8_t withTag;
-  char*   sql;
+  char*  sql;
-  char*   ast;
+  char*  ast;
-  char    subscribeDbName[TSDB_DB_NAME_LEN];
+  char   subscribeDbName[TSDB_DB_NAME_LEN];
 } SCMCreateTopicReq;
 
 int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);

include/common/tmsgdef.h

@@ -175,11 +175,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@@ -205,6 +201,21 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
+  // sync integration
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

include/libs/function/function.h

@@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
 typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -113,7 +113,7 @@ typedef struct SResultRowEntryInfo {
   bool     initialized:1;     // output buffer has been initialized
   bool     complete:1;        // query has completed
   uint8_t  isNullRes:6;       // the result is null
-  uint8_t  numOfRes;        // num of output result in current buffer
+  uint8_t  numOfRes;          // num of output result in current buffer
 } SResultRowEntryInfo;
 
 // determine the real data need to calculated the result

include/libs/qcom/query.h

@@ -133,12 +133,6 @@ typedef struct SMsgSendInfo {
   SDataBuf             msgInfo;
 } SMsgSendInfo;
 
-typedef struct SQueryNodeAddr {
-  int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epSet;
-} SQueryNodeAddr;
-
-
 typedef struct SQueryNodeStat {
   int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
+//#include <tdatablock.h>
-#include "taosdef.h"
+#include "cJSON.h"
-#include "trpc.h"
+#include "tdef.h"
-#include "wal.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
+  uint16_t nodePort;
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
+
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
+
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
+
 typedef struct SSyncFSM {
   void* data;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-                     ESyncState state);
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
-  // user can do something according to the code and isWeak. for example, write data into tsdb
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
-
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
-
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
-
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN   0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
+struct SWal;
-typedef struct SStateMgr {
+typedef struct SWal SWal;
-  void* data;
 
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+struct SEpSet;
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+typedef struct SEpSet SEpSet;
-
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,27 +140,35 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
-struct SSyncNode;
+int32_t     syncInit();
-typedef struct SSyncNode SSyncNode;
+void        syncCleanUp();
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
+void        syncStop(int64_t rid);
+int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
+ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
 
-int32_t syncInit();
+typedef enum {
-void    syncCleanUp();
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
 
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-void       syncStop(int64_t rid);
-int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-ESyncState syncGetMyRole(int64_t rid);
-
-// propose with sequence number, to implement linearizable semantics
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
-
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
 extern int32_t sDebugFlag;
 
-const char *syncStr(ESyncState state);
+//-----------------------------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+struct SSyncBuffer;
+typedef struct SSyncBuffer SSyncBuffer;
+//-----------------------------------------
+
+const char* syncStr(ESyncState state);
 
 #ifdef __cplusplus
 }

include/libs/sync/syncTools.h

@@ -0,0 +1,414 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -38,11 +38,11 @@ typedef struct SRpcConnInfo {
 
 typedef struct SRpcMsg {
   tmsg_t  msgType;
-  void *  pCont;
+  void   *pCont;
   int     contLen;
   int32_t code;
-  void *  handle;         // rpc handle returned to app
+  void   *handle;         // rpc handle returned to app
-  void *  ahandle;        // app handle set by client
+  void   *ahandle;        // app handle set by client
   int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int     persistHandle;  // persist handle or not
 
@@ -54,8 +54,8 @@ typedef struct {
   uint16_t clientPort;
   SRpcMsg  rpcMsg;
   int32_t  rspLen;
-  void *   pRsp;
+  void    *pRsp;
-  void *   pNode;
+  void    *pNode;
 } SNodeMsg;
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
@@ -64,7 +64,7 @@ typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
-  char *   label;         // for debug purpose
+  char    *label;         // for debug purpose
   int      numOfThreads;  // number of threads to handle connections
   int      sessions;      // number of sessions allowed
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
@@ -97,23 +97,23 @@ typedef struct {
 
 typedef struct {
   int32_t msgType;
-  void *  val;
+  void   *val;
   int32_t (*clone)(void *src, void **dst);
   void (*freeFunc)(const void *arg);
 } SRpcBrokenlinkVal;
 
 typedef struct {
-  SHashObj *        args;
+  SHashObj         *args;
   SRpcBrokenlinkVal brokenVal;
 } SRpcCtx;
 
 int32_t rpcInit();
 void    rpcCleanup();
-void *  rpcOpen(const SRpcInit *pRpc);
+void   *rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
-void *  rpcMallocCont(int contLen);
+void   *rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
-void *  rpcReallocCont(void *ptr, int contLen);
+void   *rpcReallocCont(void *ptr, int contLen);
 
 // Because taosd supports multi-process mode
 // These functions should not be used on the server side

include/libs/wal/wal.h

@@ -92,7 +92,7 @@ typedef struct SWalReadHead {
   int8_t  headVer;
   int8_t  reserved;
   int16_t msgType;
-  int32_t len;
+  int32_t bodyLen;
   int64_t ingestTs;  // not implemented
   int64_t version;
 

include/os/osSemaphore.h

@@ -22,21 +22,28 @@ extern "C" {
 
 #include <semaphore.h>
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
-  typedef struct tsem_s *tsem_t;
+
-  int tsem_init(tsem_t *sem, int pshared, unsigned int value);
+typedef struct tsem_s *tsem_t;
-  int tsem_wait(tsem_t *sem);
+
-  int tsem_post(tsem_t *sem);
+int tsem_init(tsem_t *sem, int pshared, unsigned int value);
-  int tsem_destroy(tsem_t *sem);
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+int tsem_post(tsem_t *sem);
+int tsem_destroy(tsem_t *sem);
+
 #else
-  #define tsem_t sem_t
+
-  #define tsem_init sem_init
+#define tsem_t       sem_t
-  int tsem_wait(tsem_t* sem);
+#define tsem_init    sem_init
-  #define tsem_post sem_post
+int tsem_wait(tsem_t *sem);
-  #define tsem_destroy sem_destroy
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+#define tsem_post    sem_post
+#define tsem_destroy sem_destroy
+
 #endif
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
 //  #define TdThreadRwlock TdThreadMutex
 //  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
 //  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
@@ -44,20 +51,20 @@ extern "C" {
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define TdThreadSpinlock TdThreadMutex
+#define TdThreadSpinlock               TdThreadMutex
-  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
+#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
-  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
+#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
-  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
+#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
 int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(TdThread* thread);
+void    taosResetPthread(TdThread *thread);
 bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
-int32_t taosGetAppName(char* name, int32_t* len);
+int32_t taosGetAppName(char *name, int32_t *len);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -411,6 +411,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+// sync integration
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)

include/util/tcache.h

@@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
 /**
  * initialize the cache object
  * @param keyType              key type
- * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
+ * @param refreshTimeInMs      refresh operation interval time, the maximum survival time when one element is expired
  *                             and not referenced by other objects
  * @param extendLifespan       auto extend lifespan, if accessed
  * @param fn                   free resource callback function
  * @return
  */
-SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
+SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName);
 
 /**
@@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
  * @param pCacheObj
  * @return
  */
-size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
+size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj);
 
 /**
  *  move all data node into trash, clear node in trash can if it is not referenced by any clients
@@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
  */
 void taosStopCacheRefreshWorker();
 
-SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
+SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj);
-bool taosCacheIterNext(SCacheIter* pIter);
+bool        taosCacheIterNext(SCacheIter *pIter);
-void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
+void       *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen);
-void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
+void       *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen);
-void taosCacheDestroyIter(SCacheIter* pIter);
+void        taosCacheDestroyIter(SCacheIter *pIter);
 
 #ifdef __cplusplus
 }

source/client/src/clientImpl.c

@@ -95,16 +95,16 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
     if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
       return NULL;
     }
-
-    if (port) {
-      epSet.epSet.eps[0].port = port;
-    }
   } else {
     if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
       return NULL;
     }
   }
 
+  if (port) {
+    epSet.epSet.eps[0].port = port;
+  }
+
   char*          key = getClusterKey(user, secretEncrypt, ip, port);
   SAppInstInfo** pInst = NULL;
 
@@ -439,7 +439,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
       return -1;
     }
 
-    taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return terrno;
+    }
+
     mgmtEpSet->numOfEps++;
   }
 

source/client/src/clientMsgHandler.c

@@ -151,13 +151,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     taosMemoryFreeClear(output.dbVgroup);
 
     tscError("failed to build use db output since %s", terrstr());
-  } else {
+  } else if (output.dbVgroup) {
     struct SCatalog* pCatalog = NULL;
 
     int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (code1 != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
               tstrerror(code1));
+      taosMemoryFreeClear(output.dbVgroup);
     } else {
       catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
     }

source/client/src/tmq.c

@@ -260,6 +260,16 @@ void tmq_list_destroy(tmq_list_t* list) {
   taosArrayDestroy(container);
 }
 
+int32_t tmq_list_get_size(const tmq_list_t* list) {
+  const SArray* container = &list->container;
+  return taosArrayGetSize(container);
+}
+
+char** tmq_list_to_c_array(const tmq_list_t* list) {
+  const SArray* container = &list->container;
+  return container->pData;
+}
+
 static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
   return sprintf(dst, "%s:%d", topicName, vg);
 }
@@ -387,7 +397,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->commit_cb = conf->commit_cb;
   pTmq->resetOffsetCfg = conf->resetOffset;
 
-  pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
+  pTmq->consumerId = tGenIdPI64();
   pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
   if (pTmq->clientTopics == NULL) {
     taosMemoryFree(pTmq);

source/common/src/tdatablock.c

@@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
 
   if (pEp->port == 0) {
     pEp->port = tsServerPort;
-    return -1;
   }
 
   return 0;

source/common/src/tmsg.c

@@ -114,6 +114,19 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
   return 0;
 }
 
+int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) {
+  if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
+  if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+
 int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pEp->inUse);
@@ -2058,11 +2071,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp)
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  int32_t num = taosArrayGetSize(pRsp->epSetList);
+  int32_t num = taosArrayGetSize(pRsp->addrsList);
   if (tEncodeI32(&encoder, num) < 0) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    SEpSet *epSet = taosArrayGet(pRsp->epSetList, i);
+    SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i);
-    if (tEncodeSEpSet(&encoder, epSet) < 0) return -1;
+    if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1;
   }
   tEndEncode(&encoder);
 
@@ -2078,10 +2091,10 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   if (tStartDecode(&decoder) < 0) return -1;
   int32_t num = 0;
   if (tDecodeI32(&decoder, &num) < 0) return -1;
-  pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet));
+  pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
-  if (NULL == pRsp->epSetList) return -1;
+  if (NULL == pRsp->addrsList) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1;
+    if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
   }
   tEndDecode(&decoder);
 
@@ -2089,7 +2102,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   return 0;
 }
 
-void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
+void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); }
 
 int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
   SCoder encoder = {0};

source/dnode/mgmt/implement/src/dmExec.c

@@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
 
 void dmCloseNode(SMgmtWrapper *pWrapper) {
   dInfo("node:%s, start to close", pWrapper->name);
+  pWrapper->deployed = false;
 
   while (pWrapper->refCount > 0) {
     taosMsleep(10);

source/dnode/mgmt/implement/src/dmHandle.c

@@ -370,9 +370,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "dnode-transport", "initialized");
 
-  if (dmStartUdfd(pDnode) != 0) {
+//  if (dmStartUdfd(pDnode) != 0) {
-    dError("failed to start udfd");
+//    dError("failed to start udfd");
-  }
+//  }
 
   dInfo("dnode-mgmt is initialized");
   return 0;

source/dnode/mgmt/interface/src/dmEnv.c

@@ -53,6 +53,7 @@ void dmCleanup() {
   }
 
   monCleanup();
+  syncCleanUp();
   walCleanUp();
   taosStopCacheRefreshWorker();
   dInfo("dnode env is cleaned up");

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -218,9 +218,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
 
   // Requests handled by VNODE
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -105,6 +105,7 @@ void    vmStopWorker(SVnodesMgmt *pMgmt);
 int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 void    vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);  // sync integration
 int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -121,6 +121,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
   pCfg->hashMethod = pCreate->hashMethod;
+
+  // sync integration
+  pCfg->syncCfg.myIndex = pCreate->selfIndex;
+  pCfg->syncCfg.replicaNum = pCreate->replica;
+  memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo));
+  for (int i = 0; i < pCreate->replica; ++i) {
+    (pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port;
+    snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s",
+             (pCreate->replicas)[i].fqdn);
+  }
 }
 
 static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
@@ -172,6 +182,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+  msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
   msgCb.qsizeFp = vmGetQueueSize;
 
   SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -252,7 +263,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -265,10 +276,6 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
@@ -283,4 +290,17 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
 
   dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+
+  // sync integration
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -73,6 +73,12 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
     return -1;
   }
 
+  // sync integration
+  vnodeSyncSetQ(pImpl, NULL);
+  vnodeSyncSetRpc(pImpl, NULL);
+  int32_t ret = vnodeSyncStart(pImpl);
+  assert(ret == 0);
+
   taosWLockLatch(&pMgmt->latch);
   int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
   taosWUnLockLatch(&pMgmt->latch);
@@ -137,6 +143,7 @@ static void *vmOpenVnodeFunc(void *param) {
     msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+    msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
     msgCb.qsizeFp = vmGetQueueSize;
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
     SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -266,6 +273,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
   // walCleanUp();
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
+
+  // syncCleanUp();
+
   dInfo("vnode-mgmt is cleaned up");
 }
 
@@ -306,6 +316,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "vnode-wal", "initialized");
 
+  // sync integration
+  if (syncInit() != 0) {
+    dError("failed to open sync since %s", terrstr());
+    return -1;
+  }
+
   if (vnodeInit(tsNumOfCommitThreads) != 0) {
     dError("failed to init vnode since %s", terrstr());
     goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -14,8 +14,12 @@
  */
 
 #define _DEFAULT_SOURCE
+
 #include "vmInt.h"
 
+#include "sync.h"
+#include "syncTools.h"
+
 static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
                  .ahandle = pMsg->rpcMsg.ahandle,
@@ -88,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 
 static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    version;
+  SRpcMsg    rsp;
 
   SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
   if (pArray == NULL) {
@@ -107,13 +111,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     }
   }
 
+#if 0
+  int64_t version;
+
   vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version);
 
   numOfMsgs = taosArrayGetSize(pArray);
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
     SRpcMsg  *pRpc = &pMsg->rpcMsg;
-    SRpcMsg   rsp;
 
     rsp.pCont = NULL;
     rsp.contLen = 0;
@@ -123,17 +129,36 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
     tmsgSendRsp(&rsp);
-
-#if 0
-    if (pRsp != NULL) {
-      pRsp->ahandle = pRpc->ahandle;
-      taosMemoryFree(pRsp);
-    } else {
-      if (code != 0 && terrno != 0) code = terrno;
-      vmSendRsp(pVnode->pWrapper, pMsg, code);
-    }
-#endif
   }
+#else
+  // sync integration response
+  for (int i = 0; i < taosArrayGetSize(pArray); i++) {
+    SNodeMsg *pMsg;
+    SRpcMsg  *pRpc;
+
+    pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    pRpc = &pMsg->rpcMsg;
+
+    rsp.ahandle = pRpc->ahandle;
+    rsp.handle = pRpc->handle;
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
+    if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      rsp.code = TSDB_CODE_SYN_NOT_LEADER;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) {
+      rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) {
+      // ok
+      // send response in applyQ
+    } else {
+      assert(0);
+    }
+  }
+#endif
 
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
@@ -148,13 +173,44 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SNodeMsg  *pMsg = NULL;
+  SRpcMsg    rsp;
+
+  // static int64_t version = 0;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
+#if 1
+    // sync integration
+
     taosGetQitem(qall, (void **)&pMsg);
 
-    // todo
+    // init response rpc msg
-    SRpcMsg *pRsp = NULL;
+    rsp.code = 0;
-    // (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    // get original rpc msg
+    assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg);
+    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
+    SRpcMsg originalRpcMsg;
+    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
+
+    // apply data into tsdb
+    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
+      rsp.code = terrno;
+      dTrace("vnodeProcessWriteReq error, code:%d", terrno);
+    }
+
+    syncApplyMsgDestroy(pSyncApplyMsg);
+    rpcFreeCont(originalRpcMsg.pCont);
+
+    // if leader, send response
+    if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+      rsp.ahandle = pMsg->rpcMsg.ahandle;
+      rsp.handle = pMsg->rpcMsg.handle;
+      tmsgSendRsp(&rsp);
+    }
+#endif
   }
 }
 
@@ -307,6 +363,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
         dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
         taosWriteQitem(pVnode->pMergeQ, pMsg);
         break;
+      case SYNC_QUEUE:  // sync integration
+        dTrace("msg:%p, will be put into vnode-sync queue", pMsg);
+        taosWriteQitem(pVnode->pSyncQ, pMsg);
+        break;
       default:
         code = -1;
         terrno = TSDB_CODE_INVALID_PARA;
@@ -333,6 +393,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
   return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE);
 }
 
+// sync integration
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE);
+}
+
 int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
   int32_t    size = -1;
   SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -30,6 +30,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
 int32_t  mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
 SEpSet   mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
 int32_t  mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
+int32_t  mndGetGlobalVgroupVersion(int32_t *vgId);
 
 void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndDb.c

@@ -1191,6 +1191,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
   char *p = strchr(usedbReq.db, '.');
   if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
     memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
+    //mndGetGlobalVgroupVersion(); TODO
     static int32_t vgVersion = 1;
     if (usedbReq.vgVersion < vgVersion) {
       usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
@@ -1202,16 +1203,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
       mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
       usedbRsp.vgVersion = vgVersion++;
 
-      if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) {
-        terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      } else {
-        code = 0;
-      }
     } else {
       usedbRsp.vgVersion = usedbReq.vgVersion;
-      code = 0;
     }
     usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
+    code = 0;
 
     // no jump, need to construct rsp
   } else {

source/dnode/mnode/impl/src/mndDnode.c

@@ -52,9 +52,9 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq);
 static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp);
 static int32_t mndProcessStatusReq(SNodeMsg *pReq);
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextDnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitDnode(SMnode *pMnode) {
@@ -335,10 +335,13 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
       }
       bool roleChanged = false;
       for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
-        if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
+        // sync integration
-          roleChanged = true;
+        if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
+          if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
+            roleChanged = true;
+          }
+          pVgroup->vnodeGid[vg].role = pVload->syncState;
         }
-        pVgroup->vnodeGid[vg].role = pVload->syncState;
       }
       if (roleChanged) {
         // notify scheduler role has changed
@@ -634,7 +637,7 @@ static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode *pMnode = pReq->pNode;
   int32_t totalRows = 0;
   int32_t numOfRows = 0;
@@ -667,12 +670,12 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
     STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN);
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) buf, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
 
     STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) bufVal, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false);
 
     numOfRows++;
   }
@@ -683,7 +686,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
 static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {}
 
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -698,8 +701,8 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     cols = 0;
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->id, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false);
 
     char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]);
@@ -709,19 +712,19 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int16_t id = mndGetVnodesNum(pMnode, pDnode->id);
-    colDataAppend(pColInfo, numOfRows, (const char*) &id, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&id, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
 
     char b1[9] = {0};
-    STR_TO_VARSTR(b1, online? "ready":"offline");
+    STR_TO_VARSTR(b1, online ? "ready" : "offline");
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, b1, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
 
     char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);

source/dnode/mnode/impl/src/mndOffset.c

@@ -172,7 +172,7 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) {
     bool          create = false;
     SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
     if (pOffsetObj == NULL) {
-      pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset));
+      pOffsetObj = taosMemoryMalloc(sizeof(SMqOffsetObj));
       memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
       create = true;
     }

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -49,6 +49,7 @@ static const SPerfsTableSchema topicSchema[] = {
 
 static const SPerfsTableSchema consumerSchema[] = {
     {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -61,6 +62,7 @@ static const SPerfsTableSchema subscribeSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
 };
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -58,7 +58,8 @@ static void      mndCancelGetNextQuery(SMnode *pMnode, void *pIter);
 int32_t mndInitProfile(SMnode *pMnode) {
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
-  int32_t connCheckTime = tsShellActivityTimer * 2;
+  // in ms
+  int32_t connCheckTime = tsShellActivityTimer * 2 * 1000;
   pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn");
   if (pMgmt->cache == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -174,10 +175,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
 }
 
 static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
-  SMnode *    pMnode = pReq->pNode;
+  SMnode     *pMnode = pReq->pNode;
-  SUserObj *  pUser = NULL;
+  SUserObj   *pUser = NULL;
-  SDbObj *    pDb = NULL;
+  SDbObj     *pDb = NULL;
-  SConnObj *  pConn = NULL;
+  SConnObj   *pConn = NULL;
   int32_t     code = -1;
   SConnectReq connReq = {0};
   char        ip[30] = {0};
@@ -464,7 +465,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
   taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
 
   int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp);
-  void *  buf = rpcMallocCont(tlen);
+  void   *buf = rpcMallocCont(tlen);
   tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp);
 
   int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps);
@@ -486,7 +487,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
-  SMnode *      pMnode = pReq->pNode;
+  SMnode       *pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -520,7 +521,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
-  SMnode *      pMnode = pReq->pNode;
+  SMnode       *pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -552,11 +553,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode *  pMnode = pReq->pNode;
+  SMnode   *pMnode = pReq->pNode;
   int32_t   numOfRows = 0;
   SConnObj *pConn = NULL;
   int32_t   cols = 0;
-  char *    pWrite;
+  char     *pWrite;
   char      ipStr[TSDB_IPv4ADDR_LEN + 6];
 
   if (pShow->pIter == NULL) {
@@ -611,8 +612,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int
 }
 
 static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode *pMnode = pReq->pNode;
+  SMnode   *pMnode = pReq->pNode;
-  int32_t numOfRows = 0;
+  int32_t   numOfRows = 0;
 #if 0
   SConnObj *pConn = NULL;
   int32_t   cols = 0;

source/dnode/mnode/impl/src/mndQnode.c

@@ -444,8 +444,8 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet));
+  qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr));
-  if (NULL == qlistRsp.epSetList) {
+  if (NULL == qlistRsp.addrsList) {
     mError("failed to alloc epSet while process qnode list req");
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _OVER;
@@ -455,11 +455,13 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
     void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
     if (pIter == NULL) break;
 
-    SEpSet epSet = {.numOfEps = 1};
+    SQueryNodeAddr nodeAddr = {0};
-    tstrncpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
+    nodeAddr.nodeId = QNODE_HANDLE;
-    epSet.eps[0].port = pObj->pDnode->port;
+    nodeAddr.epSet.numOfEps = 1;
+    tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
+    nodeAddr.epSet.eps[0].port = pObj->pDnode->port;
 
-    (void)taosArrayPush(qlistRsp.epSetList, &epSet);
+    (void)taosArrayPush(qlistRsp.addrsList, &nodeAddr);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);

source/dnode/mnode/impl/src/mndShow.c

@@ -28,7 +28,7 @@ static int32_t   mndProcessRetrieveSysTableReq(SNodeMsg *pReq);
 int32_t mndInitShow(SMnode *pMnode) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
 
-  pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show");
+  pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
   if (pMgmt->cache == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("failed to alloc show cache since %s", terrstr());

source/dnode/mnode/impl/src/mndTopic.c

@@ -534,8 +534,8 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    char *sql = taosMemoryCalloc(1, strlen(pTopic->sql) + 1 + VARSTR_HEADER_SIZE);
+    char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
-    strcpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql);
+    tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
     varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
     colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
 

source/dnode/mnode/impl/test/db/db.cpp

@@ -89,6 +89,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
     void*   pReq = rpcMallocCont(contLen);
     tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
 
+    taosMsleep(1000); // Wait for the vnode to become the leader
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -258,6 +258,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateDbReq(dbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(1000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -304,6 +304,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -438,6 +439,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -497,6 +499,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -538,6 +541,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -602,6 +606,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -655,6 +660,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -715,6 +721,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -775,6 +782,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/vnode/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(
     "src/vnd/vnodeStateMgr.c"
     "src/vnd/vnodeModule.c"
     "src/vnd/vnodeSvr.c"
+    "src/vnd/vnodeSync.c"
 
     # meta
     "src/meta/metaOpen.c"

source/dnode/vnode/inc/vnode.h

@@ -21,6 +21,7 @@
 #include "tqueue.h"
 #include "trpc.h"
 
+#include "sync.h"
 #include "tarray.h"
 #include "tfs.h"
 #include "wal.h"
@@ -60,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int     vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
 
+int64_t vnodeGetSyncHandle(SVnode *pVnode);
+void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
+
 // meta
 typedef struct SMeta      SMeta;  // todo: remove
 typedef struct SMTbCursor SMTbCursor;
@@ -147,6 +151,7 @@ struct SVnodeCfg {
   bool     isWeak;
   STsdbCfg tsdbCfg;
   SWalCfg  walCfg;
+  SSyncCfg syncCfg;  // sync integration
   uint32_t hashBegin;
   uint32_t hashEnd;
   int8_t   hashMethod;
@@ -166,6 +171,11 @@ typedef struct {
   uint64_t uid;
 } STableKeyInfo;
 
+// sync integration
+void    vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void    vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+int32_t vnodeSyncStart(SVnode *pVnode);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tq.h

@@ -18,8 +18,10 @@
 
 #include "executor.h"
 #include "os.h"
+#include "tcache.h"
 #include "thash.h"
 #include "tmsg.h"
+#include "trpc.h"
 #include "ttimer.h"
 #include "wal.h"
 
@@ -81,7 +83,6 @@ typedef struct STqOffsetStore STqOffsetStore;
 
 struct STqReadHandle {
   int64_t           ver;
-  int64_t           tbUid;
   SHashObj*         tbIdHash;
   const SSubmitReq* pMsg;
   SSubmitBlk*       pBlock;
@@ -142,26 +143,37 @@ typedef struct {
 } STqMetaStore;
 
 typedef struct {
-  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int64_t  consumerId;
-  int64_t consumerId;
+  int32_t  epoch;
-  int32_t epoch;
+  int32_t  skipLogNum;
-  int8_t  subType;
+  int64_t  reqOffset;
-  int8_t  withTbName;
+  SRWLatch lock;
-  int8_t  withSchema;
+  SRpcMsg* handle;
-  int8_t  withTag;
+} STqPushHandle;
-  int8_t  withTagSchema;
+
-  char*   qmsg;
+typedef struct {
+  char          subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int64_t       consumerId;
+  int32_t       epoch;
+  int8_t        subType;
+  int8_t        withTbName;
+  int8_t        withSchema;
+  int8_t        withTag;
+  int8_t        withTagSchema;
+  char*         qmsg;
+  STqPushHandle pushHandle;
   // SRWLatch        lock;
   SWalReadHandle* pWalReader;
-  // number should be identical to fetch thread num
+  // task number should be the same with fetch thread
-  STqReadHandle* pStreamReader[4];
+  STqReadHandle* pExecReader[5];
-  qTaskInfo_t    task[4];
+  qTaskInfo_t    task[5];
 } STqExec;
 
 struct STQ {
   char* path;
   // STqMetaStore* tqMeta;
-  SHashObj* execs;  // subKey -> tqExec
+  SHashObj* pushMgr;  // consumerId -> STqExec*
+  SHashObj* execs;    // subKey -> STqExec
   SHashObj* pStreamTasks;
   SVnode*   pVnode;
   SWal*     pWal;
@@ -205,20 +217,6 @@ typedef struct {
   SArray* topics;  // SArray<STqTopic>
 } STqConsumer;
 
-enum {
-  TQ_PUSHER_TYPE__CLIENT = 1,
-  TQ_PUSHER_TYPE__STREAM,
-};
-
-typedef struct {
-  int8_t   type;
-  int8_t   reserved[3];
-  int32_t  ttl;
-  int64_t  consumerId;
-  SRpcMsg* pMsg;
-  // SMqPollRsp* rsp;
-} STqClientPusher;
-
 typedef struct {
   int8_t      type;
   int8_t      nodeType;
@@ -228,10 +226,6 @@ typedef struct {
   // TODO sync function
 } STqStreamPusher;
 
-typedef struct {
-  SHashObj* pHash;  // <id, STqPush*>
-} STqPushMgr;
-
 typedef struct {
   int8_t inited;
   tmr_h  timer;
@@ -247,14 +241,11 @@ void tqCleanUp();
 STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
 void tqClose(STQ*);
 // required by vnode
-int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
+int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int tqCommit(STQ*);
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-// int32_t tqProcessRebReq(STQ* pTq, char* msg);
-// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@@ -295,16 +286,6 @@ int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t
 int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
 int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
 
-// tqPush
-int32_t tqPushMgrInit();
-void    tqPushMgrCleanUp();
-
-STqPushMgr* tqPushMgrOpen();
-void        tqPushMgrClose(STqPushMgr* pushMgr);
-
-STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
-STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -68,13 +68,13 @@ typedef struct {
 } SStreamSinkInfo;
 
 typedef struct {
-  SVnode*   pVnode;
+  SVnode   *pVnode;
-  SHashObj* pHash;  // streamId -> SStreamSinkInfo
+  SHashObj *pHash;  // streamId -> SStreamSinkInfo
 } SSink;
 
 // SVState
 struct SVState {
-  int64_t processed;
+  // int64_t processed;
   int64_t committed;
   int64_t applied;
 };
@@ -85,25 +85,26 @@ struct SVnodeInfo {
 };
 
 struct SVnode {
-  char*      path;
+  char      *path;
   SVnodeCfg  config;
   SVState    state;
-  STfs*      pTfs;
+  STfs      *pTfs;
   SMsgCb     msgCb;
-  SVBufPool* pBufPool;
+  SVBufPool *pBufPool;
-  SMeta*     pMeta;
+  SMeta     *pMeta;
-  STsdb*     pTsdb;
+  STsdb     *pTsdb;
-  SWal*      pWal;
+  SWal      *pWal;
-  STQ*       pTq;
+  STQ       *pTq;
-  SSink*     pSink;
+  SSink     *pSink;
+  int64_t    sync;  // sync integration
   tsem_t     canCommit;
-  SQHandle*  pQuery;
+  SQHandle  *pQuery;
 };
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
 // sma
-void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
+void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
 
 #include "vnd.h"
 
@@ -113,6 +114,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 #include "tq.h"
 
+#include "vnodeSync.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeSync.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_SYNC_H_
+#define _TD_VNODE_SYNC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
+int32_t vnodeSyncStart(SVnode *pVnode);
+void    vnodeSyncClose(SVnode *pVnode);
+
+void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+
+int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
+int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
+
+void    vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
+
+SSyncFSM *syncVnodeMakeFsm();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SYNC_H_*/

source/dnode/vnode/src/tq/tq.c

@@ -15,9 +15,12 @@
 
 #include "vnodeInt.h"
 
-int32_t tqInit() { return tqPushMgrInit(); }
+int32_t tqInit() {
+  //
+  return 0;
+}
 
-void tqCleanUp() { tqPushMgrCleanUp(); }
+void tqCleanUp() {}
 
 STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
   STQ* pTq = taosMemoryMalloc(sizeof(STQ));
@@ -41,6 +44,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
+  pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+
   return pTq;
 }
 
@@ -52,8 +57,139 @@ void tqClose(STQ* pTq) {
   // TODO
 }
 
-int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) {
+int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
   if (msgType != TDMT_VND_SUBMIT) return 0;
+  void*       pIter = NULL;
+  STqExec*    pExec = NULL;
+  SSubmitReq* pReq = (SSubmitReq*)msg;
+  int32_t     workerId = 4;
+  int64_t     fetchOffset = ver;
+
+  while (1) {
+    pIter = taosHashIterate(pTq->pushMgr, pIter);
+    if (pIter == NULL) break;
+    pExec = *(STqExec**)pIter;
+
+    taosWLockLatch(&pExec->pushHandle.lock);
+
+    SRpcMsg* pMsg = atomic_load_ptr(&pExec->pushHandle.handle);
+    ASSERT(pMsg);
+
+    SMqDataBlkRsp rsp = {0};
+    rsp.reqOffset = pExec->pushHandle.reqOffset;
+    rsp.blockData = taosArrayInit(0, sizeof(int32_t));
+    rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+
+    if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
+      qTaskInfo_t task = pExec->task[workerId];
+      ASSERT(task);
+      qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+      while (1) {
+        SSDataBlock* pDataBlock = NULL;
+        uint64_t     ts = 0;
+        if (qExecTask(task, &pDataBlock, &ts) < 0) {
+          ASSERT(0);
+        }
+        if (pDataBlock == NULL) break;
+
+        ASSERT(pDataBlock->info.rows != 0);
+        ASSERT(pDataBlock->info.numOfCols != 0);
+
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        pRetrieve->useconds = ts;
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
+      }
+    } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
+      STqReadHandle* pReader = pExec->pExecReader[workerId];
+      tqReadHandleSetMsg(pReader, pReq, 0);
+      while (tqNextDataBlock(pReader)) {
+        SSDataBlock block = {0};
+        if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+                                &block.info.numOfCols) < 0) {
+          ASSERT(0);
+        }
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        /*pRetrieve->useconds = 0;*/
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(block.info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
+      }
+    } else {
+      ASSERT(0);
+    }
+
+    if (rsp.blockNum == 0) {
+      taosWUnLockLatch(&pExec->pushHandle.lock);
+      continue;
+    }
+
+    ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
+    ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
+
+    rsp.rspOffset = fetchOffset;
+
+    int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
+    void*   buf = rpcMallocCont(tlen);
+    if (buf == NULL) {
+      pMsg->code = -1;
+      return -1;
+    }
+
+    ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+    ((SMqRspHead*)buf)->epoch = pExec->pushHandle.epoch;
+    ((SMqRspHead*)buf)->consumerId = pExec->pushHandle.consumerId;
+
+    void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+    tEncodeSMqDataBlkRsp(&abuf, &rsp);
+    pMsg->pCont = buf;
+    pMsg->contLen = tlen;
+    pMsg->code = 0;
+    tmsgSendRsp(pMsg);
+
+    atomic_store_ptr(&pExec->pushHandle.handle, NULL);
+    taosWUnLockLatch(&pExec->pushHandle.lock);
+
+    vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+           TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
+           rsp.reqOffset, rsp.rspOffset);
+
+    // TODO destroy
+    taosArrayDestroy(rsp.blockData);
+    taosArrayDestroy(rsp.blockDataLen);
+  }
+
+  return 0;
+}
+
+int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
+  if (msgType != TDMT_VND_SUBMIT) return 0;
+
   void* data = taosMemoryMalloc(msgLen);
   if (data == NULL) {
     return -1;
@@ -61,7 +197,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
   memcpy(data, msg, msgLen);
 
   if (msgType == TDMT_VND_SUBMIT) {
-    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, version) != 0) {
+    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
       return -1;
     }
   }
@@ -71,6 +207,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
       .pCont = data,
       .contLen = msgLen,
   };
+
   tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
 
 #if 0
@@ -240,6 +377,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReqV2* pReq = pMsg->pCont;
   int64_t       consumerId = pReq->consumerId;
+  int64_t       waitTime = pReq->blockingTime;
   int32_t       reqEpoch = pReq->epoch;
   int64_t       fetchOffset;
 
@@ -265,8 +403,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
-  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
   rsp.blockData = taosArrayInit(0, sizeof(void*));
+  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
 
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
@@ -283,6 +421,28 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // response to user
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
+
+#if 0
+      // add to pushMgr
+      taosWLockLatch(&pExec->pushHandle.lock);
+
+      pExec->pushHandle.consumerId = consumerId;
+      pExec->pushHandle.epoch = reqEpoch;
+      pExec->pushHandle.reqOffset = rsp.reqOffset;
+      pExec->pushHandle.skipLogNum = rsp.skipLogNum;
+      pExec->pushHandle.handle = pMsg;
+
+      taosWUnLockLatch(&pExec->pushHandle.lock);
+
+      // TODO add timer
+
+      // TODO: the pointer will always be valid?
+      taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*));
+      taosArrayDestroy(rsp.blockData);
+      taosArrayDestroy(rsp.blockDataLen);
+      return 0;
+#endif
+
       break;
     }
 
@@ -325,7 +485,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           rsp.blockNum++;
         }
       } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
-        STqReadHandle* pReader = pExec->pStreamReader[workerId];
+        STqReadHandle* pReader = pExec->pExecReader[workerId];
         tqReadHandleSetMsg(pReader, pCont, 0);
         while (tqNextDataBlock(pReader)) {
           SSDataBlock block = {0};
@@ -635,10 +795,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     req.qmsg = NULL;
 
     pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
-    for (int32_t i = 0; i < 4; i++) {
+    for (int32_t i = 0; i < 5; i++) {
-      pExec->pStreamReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+      pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle handle = {
-          .reader = pExec->pStreamReader[i],
+          .reader = pExec->pExecReader[i],
           .meta = pTq->pVnode->pMeta,
       };
       pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);

source/dnode/vnode/src/tq/tqPush.c

@@ -12,73 +12,3 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
-#include "vnodeInt.h"
-
-int32_t tqPushMgrInit() {
-  //
-  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
-  if (old == 1) return 0;
-
-  tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
-  return 0;
-}
-
-void tqPushMgrCleanUp() {
-  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
-  if (old == 0) return;
-  taosTmrStop(tqPushMgmt.timer);
-  taosTmrCleanUp(tqPushMgmt.timer);
-}
-
-STqPushMgr* tqPushMgrOpen() {
-  STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr));
-  if (mgr == NULL) {
-    return NULL;
-  }
-  mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-  return mgr;
-}
-
-void tqPushMgrClose(STqPushMgr* pushMgr) {
-  taosHashCleanup(pushMgr->pHash);
-  taosMemoryFree(pushMgr);
-}
-
-STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
-  STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher));
-  if (clientPusher == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
-  clientPusher->pMsg = pMsg;
-  clientPusher->consumerId = consumerId;
-  clientPusher->ttl = ttl;
-  if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(clientPusher);
-    // TODO send rsp back
-    return NULL;
-  }
-  return clientPusher;
-}
-
-STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
-  STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher));
-  if (streamPusher == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  streamPusher->type = TQ_PUSHER_TYPE__STREAM;
-  streamPusher->nodeType = 0;
-  streamPusher->streamId = streamId;
-  /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/
-
-  if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(streamPusher);
-    return NULL;
-  }
-  return streamPusher;
-}

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -82,6 +82,18 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
 
+  // sync integration
+  if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
+  SJson *pNodeInfoArr = tjsonCreateArray();
+  tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr);
+  for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) {
+    SJson *pNodeInfo = tjsonCreateObject();
+    tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
+    tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
+    tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
+  }
+
   return 0;
 }
 
@@ -120,6 +132,21 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
 
+  // sync integration
+  if (tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
+
+  SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
+  int    arraySize = tjsonGetArraySize(pNodeInfoArr);
+  assert(arraySize == pCfg->syncCfg.replicaNum);
+
+  for (int i = 0; i < arraySize; ++i) {
+    SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i);
+    assert(pNodeInfo != NULL);
+    tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
+    tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
+  }
+
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -158,12 +158,23 @@ int vnodeSyncCommit(SVnode *pVnode) {
 }
 
 static int vnodeCommit(void *arg) {
-  SVnode *pVnode = (SVnode *)arg;
+  SVnode    *pVnode = (SVnode *)arg;
+  char       dir[TSDB_FILENAME_LEN];
+  SVnodeInfo info = {0};
+
+  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
+  info.config = pVnode->config;
+  info.state.committed = pVnode->state.applied;
+  info.state.applied = pVnode->state.applied;
+
+  vnodeSaveInfo(dir, &info);
 
   // metaCommit(pVnode->pMeta);
   tqCommit(pVnode->pTq);
   tsdbCommit(pVnode->pTsdb);
 
+  vnodeCommitInfo(dir, &info);
+
   vnodeBufPoolRecycle(pVnode);
   tsem_post(&(pVnode->canCommit));
   return 0;
@@ -185,6 +196,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
   const SVState *pState = (SVState *)pObj;
 
   if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "applied version", pState->applied) < 0) return -1;
 
   return 0;
 }
@@ -193,6 +205,7 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   SVState *pState = (SVState *)pObj;
 
   if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "applied version", pState->applied) < 0) return -1;
 
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -35,6 +35,8 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
 
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
   info.config = *pCfg;
+  info.state.committed = -1;
+  info.state.applied = -1;
 
   if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
     vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno));
@@ -75,8 +77,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->path = (char *)&pVnode[1];
   strcpy(pVnode->path, path);
   pVnode->config = info.config;
-  pVnode->state.committed = info.state.committed;
+  pVnode->state = info.state;
-  pVnode->state.processed = pVnode->state.applied = pVnode->state.committed;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
 
@@ -124,6 +125,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
     goto _err;
   }
 
+  // sync integration
+  // open sync
+  if (vnodeSyncOpen(pVnode, dir)) {
+    goto _err;
+  }
+
 #if 0
   if (vnodeBegin() < 0) {
     goto _err;
@@ -149,6 +156,10 @@ void vnodeClose(SVnode *pVnode) {
     vnodeSyncCommit(pVnode);
     // close vnode
     vnodeQueryClose(pVnode);
+
+    // sync integration
+    vnodeSyncClose(pVnode);
+
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
     tsdbClose(pVnode->pTsdb);
@@ -159,3 +170,7 @@ void vnodeClose(SVnode *pVnode) {
     taosMemoryFree(pVnode);
   }
 }
+
+int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
+
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "sync.h"
+#include "syncTools.h"
 #include "vnodeInt.h"
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
@@ -21,6 +23,7 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
 static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
 
 int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
+#if 0
   SNodeMsg *pMsg;
   SRpcMsg  *pRpc;
 
@@ -38,6 +41,7 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
 
   walFsync(pVnode->pWal, false);
 
+#endif
   return 0;
 }
 
@@ -86,21 +90,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
         // TODO: handle error
       }
       break;
-#if 0
-    case TDMT_VND_MQ_SET_CONN: {
-      if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-        // TODO: handle error
-      }
-    } break;
-    case TDMT_VND_MQ_REB: {
-      if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-      }
-    } break;
-    case TDMT_VND_MQ_CANCEL_CONN: {
-      if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-      }
-    } break;
-#endif
     case TDMT_VND_TASK_DEPLOY: {
       if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
@@ -202,8 +191,86 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
 }
 
+// sync integration
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
-  /*vInfo("sync message is processed");*/
+  SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
+  assert(pSyncNode != NULL);
+
+  ESyncState state = syncGetMyRole(pVnode->sync);
+  SyncTerm   currentTerm = syncGetMyTerm(pVnode->sync);
+
+  SMsgHead *pHead = pMsg->pCont;
+
+  char  logBuf[512];
+  char *syncNodeStr = sync2SimpleStr(pVnode->sync);
+  snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr);
+  syncRpcMsgLog2(logBuf, pMsg);
+  taosMemoryFree(syncNodeStr);
+
+  SRpcMsg *pRpcMsg = pMsg;
+
+  if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
+    SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+    syncTimeoutDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
+    SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnPingCb(pSyncNode, pSyncMsg);
+    syncPingDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
+    SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+    syncPingReplyDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
+    SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+    syncClientRequestDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
+    SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+    syncRequestVoteDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
+    SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+    syncRequestVoteReplyDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
+    SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+    syncAppendEntriesDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
+    SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+    syncAppendEntriesReplyDestroy(pSyncMsg);
+
+  } else {
+    vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
+  }
+
+  syncNodeRelease(pSyncNode);
+
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync.h"
+#include "syncTools.h"
+#include "tmsgcb.h"
+#include "vnodeInt.h"
+
+// sync integration
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = pVnode->config.vgId;
+  SSyncCfg *pCfg = &(syncInfo.syncCfg);
+  pCfg->replicaNum = pVnode->config.syncCfg.replicaNum;
+  pCfg->myIndex = pVnode->config.syncCfg.myIndex;
+  memcpy(pCfg->nodeInfo, pVnode->config.syncCfg.nodeInfo, sizeof(pCfg->nodeInfo));
+
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", path);
+  syncInfo.pWal = pVnode->pWal;
+
+  syncInfo.pFsm = syncVnodeMakeFsm(pVnode);
+  syncInfo.rpcClient = NULL;
+  syncInfo.FpSendMsg = vnodeSendMsg;
+  syncInfo.queue = NULL;
+  syncInfo.FpEqMsg = vnodeSyncEqMsg;
+
+  pVnode->sync = syncOpen(&syncInfo);
+  assert(pVnode->sync > 0);
+
+  // for test
+  setPingTimerMS(pVnode->sync, 3000);
+  setElectTimerMS(pVnode->sync, 500);
+  setHeartbeatTimerMS(pVnode->sync, 100);
+
+  return 0;
+}
+
+int32_t vnodeSyncStart(SVnode *pVnode) {
+  syncStart(pVnode->sync);
+  return 0;
+}
+
+void vnodeSyncClose(SVnode *pVnode) {
+  // stop by ref id
+  syncStop(pVnode->sync);
+}
+
+void vnodeSyncSetQ(SVnode *pVnode, void *qHandle) { syncSetQ(pVnode->sync, (void *)(&(pVnode->msgCb))); }
+
+void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle) { syncSetRpc(pVnode->sync, (void *)(&(pVnode->msgCb))); }
+
+int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg) {
+  int32_t ret = 0;
+  SMsgCb *pMsgCb = qHandle;
+  if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
+    tmsgPutToQueue(qHandle, SYNC_QUEUE, pMsg);
+  } else {
+    vError("vnodeSyncEqMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
+  }
+  return ret;
+}
+
+int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  int32_t ret = 0;
+  SMsgCb *pMsgCb = rpcHandle;
+  if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
+    tmsgSendReq(rpcHandle, pEpSet, pMsg);
+  } else {
+    vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
+  }
+  return ret;
+}
+
+int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+  SVnode *pVnode = (SVnode *)(pFsm->data);
+  vnodeGetSnapshot(pVnode, pSnapshot);
+
+  /*
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = 0;
+  pSnapshot->lastApplyTerm = 0;
+  */
+
+  return 0;
+}
+
+void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  if (pFsm->FpGetSnapshot != NULL) {
+    SSnapshot snapshot;
+    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    char logBuf[256];
+    snprintf(
+        logBuf, sizeof(logBuf),
+        "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
+        pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+
+    SVnode       *pVnode = (SVnode *)(pFsm->data);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
+    SRpcMsg       applyMsg;
+    syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
+    syncApplyMsgDestroy(pSyncApplyMsg);
+
+    /*
+        SRpcMsg applyMsg;
+        applyMsg = *pMsg;
+        applyMsg.pCont = rpcMallocCont(applyMsg.contLen);
+        assert(applyMsg.contLen == pMsg->contLen);
+        memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen);
+    */
+
+    // recover handle for response
+    SRpcMsg saveRpcMsg;
+    int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
+    if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
+      applyMsg.handle = saveRpcMsg.handle;
+      applyMsg.ahandle = saveRpcMsg.ahandle;
+    } else {
+      applyMsg.handle = NULL;
+      applyMsg.ahandle = NULL;
+    }
+
+    // put to applyQ
+    tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
+
+  } else {
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf),
+             "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
+             "beginIndex :%ld\n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
+             beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+  }
+}
+
+void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
+           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+SSyncFSM *syncVnodeMakeFsm(SVnode *pVnode) {
+  SSyncFSM *pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM));
+  pFsm->data = pVnode;
+  pFsm->FpCommitCb = vnodeSyncCommitCb;
+  pFsm->FpPreCommitCb = vnodeSyncPreCommitCb;
+  pFsm->FpRollBackCb = vnodeSyncRollBackCb;
+  pFsm->FpGetSnapshot = vnodeSyncGetSnapshotCb;
+  return pFsm;
+}

source/libs/executor/inc/executil.h

@@ -60,7 +60,7 @@ typedef struct SResultRow {
   uint32_t      numOfRows;   // number of rows of current time window
   struct SResultRowEntryInfo* pEntryInfo;  // For each result column, there is a resultInfo
   STimeWindow   win;
-  char         *key;               // start key of current result row
+  char         *key;         // start key of current result row
 } SResultRow;
 
 typedef struct SResultRowPosition {

source/libs/executor/inc/executorimpl.h

@@ -608,8 +608,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
 int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols,
                     SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey);
 void    initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
-void    doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo,
+void    doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo,
-                     SDiskbasedBuf* pBuf, int32_t* rowCellOffset);
+                     SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx);
 void    finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
                                       SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
 void    doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,

source/libs/executor/src/executorimpl.c

@@ -230,10 +230,10 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) {
 
 void operatorDummyCloseFn(void* param, int32_t numOfCols) {}
 
-static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
+static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
-                                  int32_t orderType, int32_t* rowCellOffset);
+                                  int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx);
-static void    initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
 
+static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
 static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo);
 static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable);
 static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo);
@@ -2809,16 +2809,6 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
   }
 }
 
-void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
-  for (int32_t j = 0; j < numOfOutput; ++j) {
-    if (pCtx[j].functionId == -1) {
-      continue;
-    }
-
-    pCtx[j].fpSet.finalize(&pCtx[j]);
-  }
-}
-
 void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
                                    SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
@@ -2841,7 +2831,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
       }
 
       if (pCtx[j].fpSet.process) {  // TODO set the dummy function, to avoid the check for null ptr.
-        pCtx[j].fpSet.finalize(&pCtx[j]);
+//        pCtx[j].fpSet.finalize(&pCtx[j]);
       }
 
       if (pRow->numOfRows < pResInfo->numOfRes) {
@@ -2872,7 +2862,7 @@ void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbased
       }
 
       if (pCtx[j].fpSet.process) {  // TODO set the dummy function.
-        pCtx[j].fpSet.finalize(&pCtx[j]);
+//        pCtx[j].fpSet.finalize(&pCtx[j]);
         pResInfo->initialized = true;
       }
 
@@ -3107,7 +3097,8 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin
  * @param pQInfo
  * @param result
  */
-int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset) {
+int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
+    int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) {
   int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
   int32_t numOfResult = pBlock->info.rows;  // there are already exists result rows
 
@@ -3125,8 +3116,6 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo*
     step = -1;
   }
 
-  int32_t nrows = pBlock->info.rows;
-
   for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
     SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i);
     SFilePage*          page = getBufPage(pBuf, pPos->pageId);
@@ -3139,7 +3128,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo*
 
     // TODO copy multiple rows?
     int32_t numOfRowsToCopy = pRow->numOfRows;
-    if (numOfResult + numOfRowsToCopy >= rowCapacity) {
+    if (numOfResult + numOfRowsToCopy >= pBlock->info.capacity) {
       break;
     }
 
@@ -3148,31 +3137,32 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, int32_t rowCapacity, SExprInfo*
     for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
       int32_t slotId = pExprInfo[j].base.resSchema.slotId;
 
-      SColumnInfoData*     pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
+      pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset);
-      SResultRowEntryInfo* pEntryInfo = getResultCell(pRow, j, rowCellOffset);
+      if (pCtx[j].fpSet.process) {
+        pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId);
+      } else {
+        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
 
-      char* in = GET_ROWCELL_INTERBUF(pEntryInfo);
+        char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
-      colDataAppend(pColInfoData, nrows, in, pEntryInfo->isNullRes);
+        colDataAppend(pColInfoData, pBlock->info.rows, in, pCtx[j].resultInfo->isNullRes);
+      }
     }
 
     releaseBufPage(pBuf, page);
-    nrows += 1;
 
-    numOfResult += numOfRowsToCopy;
+    pBlock->info.rows += pRow->numOfRows;
-    if (numOfResult == rowCapacity) {  // output buffer is full
+    if (pBlock->info.rows >= pBlock->info.capacity) {  // output buffer is full
       break;
     }
   }
 
   // qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv));
-  pBlock->info.rows = numOfResult;
   blockDataUpdateTsWindow(pBlock);
-
   return 0;
 }
 
-void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf,
+void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf,
-                  int32_t* rowCellOffset) {
+                  int32_t* rowCellOffset, SqlFunctionCtx* pCtx) {
   assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup);
 
   blockDataCleanup(pBlock);
@@ -3181,7 +3171,7 @@ void doBuildResultDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResI
   }
 
   int32_t orderType = TSDB_ORDER_ASC;
-  doCopyToSDataBlock(pBlock, rowCapacity, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset);
+  doCopyToSDataBlock(pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx);
 
   // add condition (pBlock->info.rows >= 1) just to runtime happy
   blockDataUpdateTsWindow(pBlock);
@@ -4405,7 +4395,7 @@ static void doFinalizeResultImpl(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
     //      SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
     //      doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
     //    } else {
-    pCtx[j].fpSet.finalize(&pCtx[j]);
+//    pCtx[j].fpSet.finalize(&pCtx[j]);
   }
 }
 
@@ -4805,7 +4795,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup)
   }
 
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pInfo->pRes, pOperator->resultInfo.capacity, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset);
+  doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset, pInfo->pCtx);
   if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }
@@ -5129,8 +5119,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro
     }
 
     blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
-    doBuildResultDatablock(pBlock, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr,
+    doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
-                           pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
 
     if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
@@ -5149,7 +5138,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup
   }
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
+    doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
     if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
     }
@@ -5184,7 +5173,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup
 
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
+  doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
 
   ASSERT(pInfo->binfo.pRes->info.rows > 0);
   pOperator->status = OP_RES_TO_RETURN;
@@ -5229,7 +5218,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) {
   pOperator->status = OP_RES_TO_RETURN;
   closeAllResultRows(&pSliceInfo->binfo.resultRowInfo);
   setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
-  finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput);
+//  finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput);
 
   initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
   //  doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes);
@@ -5286,8 +5275,8 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup
   OPTR_SET_OPENED(pOperator);
 
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr,
+  doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr,
-               pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
+               pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
 
   if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
@@ -5377,7 +5366,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) {
   SOptrBasicInfo* pBInfo    = &pInfo->binfo;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset);
+    doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
       return NULL;
@@ -5409,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) {
 
   initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset);
+  doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }
@@ -5426,7 +5415,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup)
   SOptrBasicInfo*          pBInfo = &pInfo->binfo;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset);
+    doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
       return NULL;
@@ -5458,7 +5447,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup)
 
   initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset);
+  doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }

source/libs/executor/src/groupoperator.c

@@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
   SSDataBlock* pRes = pInfo->binfo.pRes;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
+    doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
     if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
     }
@@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
   initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
 
   while(1) {
-    doBuildResultDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
+    doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);
     doFilter(pInfo->pCondition, pRes);
 
     bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo);

source/libs/function/inc/builtinsimpl.h

@@ -24,7 +24,7 @@ extern "C" {
 #include "functionMgt.h"
 
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
-void functionFinalize(SqlFunctionCtx *pCtx);
+int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
 
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
@@ -43,12 +43,12 @@ int32_t maxFunction(SqlFunctionCtx *pCtx);
 bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t stddevFunction(SqlFunctionCtx* pCtx);
-void stddevFinalize(SqlFunctionCtx* pCtx);
+int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
 
 bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t percentileFunction(SqlFunctionCtx *pCtx);
-void percentileFinalize(SqlFunctionCtx* pCtx);
+int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
 
 bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@@ -60,6 +60,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx);
 
 bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 int32_t topFunction(SqlFunctionCtx *pCtx);
+int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
 
 #ifdef __cplusplus
 }

source/libs/function/src/builtins.c

@@ -507,7 +507,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = topFunction,
-    .finalizeFunc = functionFinalize
+    .finalizeFunc = topBotFinalize,
   },
   {
     .name = "bottom",

source/libs/function/src/builtinsimpl.c

@@ -49,11 +49,17 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
   return true;
 }
 
-void functionFinalize(SqlFunctionCtx *pCtx) {
+int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
-  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
 
-  cleanupResultRowEntry(pResInfo);
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0;
+  cleanupResultRowEntry(pResInfo);
+
+  char* in = GET_ROWCELL_INTERBUF(pResInfo);
+  colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
+
+  return pResInfo->numOfRes;
 }
 
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
@@ -612,12 +618,11 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-void stddevFinalize(SqlFunctionCtx* pCtx) {
+int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
-  functionFinalize(pCtx);
-
   SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
   double avg = pStddevRes->isum / ((double) pStddevRes->count);
   pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
+  return functionFinalize(pCtx, pBlock, slotId);
 }
 
 typedef struct SPercentileInfo {
@@ -739,7 +744,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-void percentileFinalize(SqlFunctionCtx* pCtx) {
+int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
   SVariant* pVal = &pCtx->param[1].param;
   double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d;
 
@@ -752,7 +757,7 @@ void percentileFinalize(SqlFunctionCtx* pCtx) {
   }
 
   tMemBucketDestroy(pMemBucket);
-  functionFinalize(pCtx);
+  return functionFinalize(pCtx, pBlock, slotId);
 }
 
 bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@@ -1173,16 +1178,14 @@ typedef struct STopBotResItem {
 } STopBotResItem;
 
 typedef struct STopBotRes {
-  int32_t   num;
+  int32_t         pageId;
+//  int32_t   num;
   STopBotResItem *pItems;
 } STopBotRes;
 
 bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
-  SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0);
-  int32_t bytes = pColNode->node.resType.bytes;
   SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
-
+  pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem);
-  pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes;
   return true;
 }
 
@@ -1194,13 +1197,14 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) {
   return pRes;
 }
 
-static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid);
+static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock,
+                            uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo);
 
 int32_t topFunction(SqlFunctionCtx *pCtx) {
   int32_t numOfElems = 0;
+  SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
 
   STopBotRes *pRes = getTopBotOutputInfo(pCtx);
-  assert(pRes->num >= 0);
 
 //  if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) {
 //    buildTopBotStruct(pRes, pCtx);
@@ -1221,11 +1225,9 @@ int32_t topFunction(SqlFunctionCtx *pCtx) {
     numOfElems++;
 
     char* data = colDataGetData(pCol, i);
-    doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid);
+    doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo);
   }
 
-  // treat the result as only one result
-  SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1256,7 +1258,9 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par
   return (val1->v.d > val2->v.d) ? 1 : -1;
 }
 
-void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) {
+
+void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type,
+    uint64_t uid, SResultRowEntryInfo* pEntryInfo) {
   SVariant val = {0};
   taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
 
@@ -1264,29 +1268,71 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t ty
   assert(pItems != NULL);
 
   // not full yet
-  if (pRes->num < maxSize) {
+  if (pEntryInfo->numOfRes < maxSize) {
-    STopBotResItem* pItem = &pItems[pRes->num];
+    STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes];
     pItem->v   = val;
     pItem->uid = uid;
     pItem->tuplePos.pageId = -1;  // todo set the corresponding tuple data in the disk-based buffer
 
-    pRes->num++;
+    if (pRes->pageId == -1) {
-    taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false);
+      SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId);
+      pPage->num = sizeof(SFilePage);
+
+      // keep the current row data
+      for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
+        SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
+        bool isNull = colDataIsNull_s(pCol, rowIndex);
+
+
+        colDataGetData(pCol, rowIndex);
+      }
+
+    }
+
+    // allocate the buffer and keep the data of this row into the new allocated buffer
+    pEntryInfo->numOfRes++;
+    taosheapsort((void *) pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void *) &type, topBotResComparFn, false);
   } else { // replace the minimum value in the result
     if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) ||
         (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) ||
         (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) {
-      STopBotResItem* pItem = &pItems[pRes->num];
+      STopBotResItem* pItem = &pItems[0];
       pItem->v   = val;
       pItem->uid = uid;
       pItem->tuplePos.pageId = -1;  // todo set the corresponding tuple data in the disk-based buffer
 
-      taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false);
+      taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false);
     }
   }
 }
 
-void topBotFinalize(SqlFunctionCtx* pCtx) {
+int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
-  functionFinalize(pCtx);
+  SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx);
+  STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
+  pEntryInfo->complete = true;
 
+  int32_t type = pCtx->input.pData[0]->info.type;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  // todo assign the tag value and the corresponding row data
+  int32_t currentRow = pBlock->info.rows;
+  switch(type) {
+    case TSDB_DATA_TYPE_INT: {
+      for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) {
+        STopBotResItem* pItem = &pRes->pItems[i];
+        colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i);
+
+        int32_t pageId = pItem->tuplePos.pageId;
+        int32_t offset = pItem->tuplePos.offset;
+        if (pageId != -1) {
+          // todo
+        }
+      }
+      break;
+    }
+  }
+
+  return pEntryInfo->numOfRes;
+
+//  return functionFinalize(pCtx, pBlock, slotId);
 }

source/libs/qcom/src/querymsg.c

@@ -359,10 +359,10 @@ PROCESS_QLIST_OVER:
 
   if (code != 0) {
     tFreeSQnodeListRsp(&out);
-    out.epSetList = NULL;
+    out.addrsList = NULL;
   }
 
-  *(SArray **)output = out.epSetList;
+  *(SArray **)output = out.addrsList;
 
   return code;
 }

source/libs/qworker/src/qworker.c

@@ -1388,6 +1388,14 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
 
   void *pIter = taosHashIterate(mgmt->schHash, NULL);
   while (pIter) {
+    SQWSchStatus *sch = (SQWSchStatus *)pIter;
+    if (NULL == sch->hbConnInfo.handle) {
+      uint64_t *sId = taosHashGetKey(pIter, NULL);
+      QW_DLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId);
+      pIter = taosHashIterate(mgmt->schHash, pIter);
+      continue;
+    }
+    
     code = qwGenerateSchHbRsp(mgmt, (SQWSchStatus *)pIter, &rspList[i]);
     if (code) {
       taosHashCancelIterate(mgmt->schHash, pIter);

source/libs/scalar/inc/filterInt.h

@@ -186,6 +186,7 @@ typedef struct SFilterColCtx {
 
 typedef struct SFilterCompare {
   uint8_t       type;
+  int8_t        precision;
   uint8_t       optr;
   uint8_t       optr2;
 } SFilterCompare;
@@ -218,6 +219,7 @@ typedef struct SFltTreeStat {
   int32_t      code;
   int8_t       precision;
   bool         scalarMode;
+  SArray*      nodeList;
   SFilterInfo* info;
 } SFltTreeStat;
 
@@ -303,6 +305,7 @@ struct SFilterInfo {
 #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
 #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
 #define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type)
+#define FILTER_GET_COL_FIELD_PRECISION(fi) (((SColumnNode *)((fi)->desc))->node.resType.precision)
 #define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes)
 #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId)
 #define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId)
@@ -317,6 +320,7 @@ struct SFilterInfo {
 #define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
 #define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
 #define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
+#define FILTER_UNIT_DATA_PRECISION(u) ((u)->compare.precision)
 #define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
 #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))

source/libs/scalar/src/filter.c

@@ -21,6 +21,7 @@
 #include "sclInt.h"
 #include "tcompare.h"
 #include "tdatablock.h"
+#include "ttime.h"
 
 OptrStr gOptrStr[] = {
   {0,                                      "invalid"},
@@ -986,6 +987,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
   assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN));
   
   info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col);
+  info->units[info->unitNum].compare.precision = FILTER_GET_COL_FIELD_PRECISION(col);
 
   *uidx = info->unitNum;
 
@@ -1748,6 +1750,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
     assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE));
 
     uint32_t type = FILTER_UNIT_DATA_TYPE(unit);
+    int8_t precision = FILTER_UNIT_DATA_PRECISION(unit);
     SFilterField* fi = right;
     
     SValueNode* var = (SValueNode *)fi->desc;
@@ -1801,6 +1804,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
     } else {
       SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
       out.columnData->info.type = type;
+      out.columnData->info.precision = precision;
       if (IS_VAR_DATA_TYPE(type)) {
         out.columnData->info.bytes = bytes;
       } else {
@@ -3475,6 +3479,33 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode* pNode) {
+  if (NULL == stat->nodeList) {
+    stat->nodeList = taosArrayInit(10, POINTER_BYTES);
+    if (NULL == stat->nodeList) {
+      FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+  }
+
+  if (NULL == taosArrayPush(stat->nodeList, &pNode)) {
+    FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void fltConvertToTsValueNode(SFltTreeStat *stat, SValueNode* valueNode) {
+  char *timeStr = valueNode->datum.p;
+  if (convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, stat->precision, &valueNode->datum.i) !=
+      TSDB_CODE_SUCCESS) {
+    valueNode->datum.i = 0;
+  }
+  taosMemoryFree(timeStr);
+  
+  valueNode->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
+  valueNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
 EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
   SFltTreeStat *stat = (SFltTreeStat *)pContext;
 
@@ -3504,25 +3535,23 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
     }
     
     SValueNode *valueNode = (SValueNode *)*pNode;
-    if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type) {
+    if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) {
       return DEAL_RES_CONTINUE;
     }
 
-#if 0    
     if (stat->precision < 0) {
-      //TODO
+      int32_t code = fltAddValueNodeToConverList(stat, valueNode);
+      if (code) {
+        stat->code = code;
+        return DEAL_RES_ERROR;
+      }
+      
       return DEAL_RES_CONTINUE;
     }
 
-    char *timeStr = valueNode->datum.p;
+    fltConvertToTsValueNode(stat, valueNode);
-    if (taosParseTime(valueNode->datum.p, &valueNode->datum.i, valueNode->node.resType.bytes, stat->precision, tsDaylight) !=
+
-        TSDB_CODE_SUCCESS) {
-      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
-    }
-    TODO
-#else
     return DEAL_RES_CONTINUE;
-#endif
   }
 
   if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
@@ -3619,9 +3648,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
 }
 
 int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
+  int32_t code = 0;
   nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat);
 
-  FLT_RET(pStat->code);
+  FLT_ERR_JRET(pStat->code);
+
+  int32_t nodeNum = taosArrayGetSize(pStat->nodeList);
+  for (int32_t i = 0; i < nodeNum; ++i) {
+    SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i);
+    
+    fltConvertToTsValueNode(pStat, valueNode);
+  }
+
+_return:
+
+  taosArrayDestroy(pStat->nodeList);
+  FLT_RET(code);
 }
 
 int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {

source/libs/scalar/src/sclvector.c

@@ -24,6 +24,7 @@
 #include "tcompare.h"
 #include "tdatablock.h"
 #include "ttypes.h"
+#include "ttime.h"
 
 #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData))
 #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData))
@@ -252,6 +253,15 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
     return p;
 }
 
+static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t rowIndex) {
+  int64_t value = 0;
+  if (taosParseTime(buf, &value, strlen(buf), pOut->columnData->info.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
+    value = 0;
+  }
+  
+  colDataAppendInt64(pOut->columnData, rowIndex, &value);
+}
+
 static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) {
   int64_t value = strtoll(buf, NULL, 10);
   colDataAppendInt64(pOut->columnData, rowIndex, &value);
@@ -295,7 +305,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
   _bufConverteFunc func = NULL;
   if (TSDB_DATA_TYPE_BOOL == outType) {
     func = varToBool;
-  } else if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType) {
+  } else if (IS_SIGNED_NUMERIC_TYPE(outType)) {
     func = varToSigned;
   } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) {
     func = varToUnsigned;
@@ -305,6 +315,8 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     ASSERT(inType == TSDB_DATA_TYPE_VARCHAR);
     func = varToNchar;
     vton = true;
+  } else if (TSDB_DATA_TYPE_TIMESTAMP == outType) {
+    func = varToTimestamp;
   } else {
     sclError("invalid convert outType:%d", outType);
     return TSDB_CODE_QRY_APP_ERROR;
@@ -594,8 +606,8 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
 /*BIGI*/   0,   0,   0,   0,   0,   0,   6,   7,   7,   0,   7,   5,   5,   5,   7,   0,   7,   0,   0,
 /*FLOA*/   0,   0,   0,   0,   0,   0,   0,   7,   7,   6,   7,   6,   6,   6,   6,   0,   7,   0,   0,
 /*DOUB*/   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   7,   7,   7,   0,   7,   0,   0,
-/*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   0,   7,   7,   7,   7,   0,   0,   0,   0,
+/*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   0,   7,   7,   7,   7,   0,   0,   0,   0,
-/*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   9,   9,   9,   7,   0,   7,   0,   0,
+/*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   9,   9,   9,   7,   0,   7,   0,   0,
 /*NCHA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   0,   0,   0,   0,
 /*UTIN*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   12,  13,  14,  0,   7,   0,   0,
 /*USMA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   13,  14,  0,   7,   0,   0,

source/libs/sync/CMakeLists.txt

@@ -11,7 +11,7 @@ target_link_libraries(
 
 target_include_directories(
     sync
-    PUBLIC "${TD_SOURCE_DIR}/include/libs/sync"
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/sync/inc/syncEnv.h

@@ -31,10 +31,10 @@ extern "C" {
 #define TIMER_MAX_MS 0x7FFFFFFF
 #define ENV_TICK_TIMER_MS 1000
 #define PING_TIMER_MS 1000
-#define ELECT_TIMER_MS_MIN 150
+#define ELECT_TIMER_MS_MIN 500
 #define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
 #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
-#define HEARTBEAT_TIMER_MS 30
+#define HEARTBEAT_TIMER_MS 100
 
 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})
 

source/libs/sync/inc/syncInt.h

@@ -25,6 +25,7 @@ extern "C" {
 #include <stdlib.h>
 #include "cJSON.h"
 #include "sync.h"
+#include "syncTools.h"
 #include "taosdef.h"
 #include "tglobal.h"
 #include "tlog.h"
@@ -67,6 +68,43 @@ extern "C" {
     }                                                                   \
   }
 
+#define sFatalLong(...)                                                 \
+  {                                                                     \
+    if (sDebugFlag & DEBUG_FATAL) {                                     \
+      taosPrintLongString("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
+    }                                                                   \
+  }
+#define sErrorLong(...)                                                 \
+  {                                                                     \
+    if (sDebugFlag & DEBUG_ERROR) {                                     \
+      taosPrintLongString("SYN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
+    }                                                                   \
+  }
+#define sWarnLong(...)                                                \
+  {                                                                   \
+    if (sDebugFlag & DEBUG_WARN) {                                    \
+      taosPrintLongString("SYN WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
+    }                                                                 \
+  }
+#define sInfoLong(...)                                                \
+  {                                                                   \
+    if (sDebugFlag & DEBUG_INFO) {                                    \
+      taosPrintLongString("SYN INFO ", DEBUG_INFO, 255, __VA_ARGS__); \
+    }                                                                 \
+  }
+#define sDebugLong(...)                                                        \
+  {                                                                            \
+    if (sDebugFlag & DEBUG_DEBUG) {                                            \
+      taosPrintLongString("SYN DEBUG ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \
+    }                                                                          \
+  }
+#define sTraceLong(...)                                                        \
+  {                                                                            \
+    if (sDebugFlag & DEBUG_TRACE) {                                            \
+      taosPrintLongString("SYN TRACE ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \
+    }                                                                          \
+  }
+
 struct SyncTimeout;
 typedef struct SyncTimeout SyncTimeout;
 
@@ -106,17 +144,19 @@ typedef struct SVotesRespond SVotesRespond;
 struct SSyncIndexMgr;
 typedef struct SSyncIndexMgr SSyncIndexMgr;
 
-typedef struct SRaftId {
+struct SRaftCfg;
-  SyncNodeId  addr;  // typedef uint64_t SyncNodeId;
+typedef struct SRaftCfg SRaftCfg;
-  SyncGroupId vgId;  // typedef int32_t  SyncGroupId;
+
-} SRaftId;
+struct SSyncRespMgr;
+typedef struct SSyncRespMgr SSyncRespMgr;
 
 typedef struct SSyncNode {
   // init by SSyncInfo
   SyncGroupId vgId;
-  SSyncCfg    syncCfg;
+  SRaftCfg*   pRaftCfg;
   char        path[TSDB_FILENAME_LEN];
   char        raftStorePath[TSDB_FILENAME_LEN * 2];
+  char        configPath[TSDB_FILENAME_LEN * 2];
 
   // sync io
   SWal* pWal;
@@ -199,10 +239,14 @@ typedef struct SSyncNode {
   int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
   int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg);
 
+  // tools
+  SSyncRespMgr* pSyncRespMgr;
+
 } SSyncNode;
 
 // open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
+void       syncNodeStart(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
 // ping --------------
@@ -222,10 +266,12 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
 
 // utils --------------
-int32_t    syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
-int32_t    syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
-cJSON*     syncNode2Json(const SSyncNode* pSyncNode);
+cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
-char*      syncNode2Str(const SSyncNode* pSyncNode);
+char*   syncNode2Str(const SSyncNode* pSyncNode);
+char*   syncNode2SimpleStr(const SSyncNode* pSyncNode);
+
 SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
 

source/libs/sync/inc/syncMessage.h

@@ -27,22 +27,6 @@ extern "C" {
 #include "syncInt.h"
 #include "taosdef.h"
 
-// encode as uint32
-typedef enum ESyncMessageType {
-  SYNC_UNKNOWN = 9999,
-  SYNC_TIMEOUT = 99,
-  SYNC_PING = 101,
-  SYNC_PING_REPLY = 103,
-  SYNC_CLIENT_REQUEST = 105,
-  SYNC_CLIENT_REQUEST_REPLY = 107,
-  SYNC_REQUEST_VOTE = 109,
-  SYNC_REQUEST_VOTE_REPLY = 111,
-  SYNC_APPEND_ENTRIES = 113,
-  SYNC_APPEND_ENTRIES_REPLY = 115,
-  SYNC_RESPONSE = 119,
-
-} ESyncMessageType;
-
 // ---------------------------------------------
 cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg);
 cJSON* syncRpcUnknownMsg2Json();
@@ -53,263 +37,7 @@ void syncRpcMsgPrint(SRpcMsg* pMsg);
 void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
 void syncRpcMsgLog(SRpcMsg* pMsg);
 void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
-
 // ---------------------------------------------
-typedef enum ESyncTimeoutType {
-  SYNC_TIMEOUT_PING = 100,
-  SYNC_TIMEOUT_ELECTION,
-  SYNC_TIMEOUT_HEARTBEAT,
-} ESyncTimeoutType;
-
-typedef struct SyncTimeout {
-  uint32_t         bytes;
-  uint32_t         msgType;
-  ESyncTimeoutType timeoutType;
-  uint64_t         logicClock;
-  int32_t          timerMS;
-  void*            data;  // need optimized
-} SyncTimeout;
-
-SyncTimeout* syncTimeoutBuild();
-SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, void* data);
-void         syncTimeoutDestroy(SyncTimeout* pMsg);
-void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
-void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
-char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);  //
-SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);         //
-void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
-void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
-SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);  //
-cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
-char*        syncTimeout2Str(const SyncTimeout* pMsg);  //
-
-// for debug ----------------------
-void syncTimeoutPrint(const SyncTimeout* pMsg);
-void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
-void syncTimeoutLog(const SyncTimeout* pMsg);
-void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncPing {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  uint32_t dataLen;
-  char     data[];
-} SyncPing;
-
-SyncPing* syncPingBuild(uint32_t dataLen);
-SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
-SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId);
-void      syncPingDestroy(SyncPing* pMsg);
-void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
-void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
-char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
-SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
-void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
-void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
-SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*    syncPing2Json(const SyncPing* pMsg);
-char*     syncPing2Str(const SyncPing* pMsg);
-
-// for debug ----------------------
-void syncPingPrint(const SyncPing* pMsg);
-void syncPingPrint2(char* s, const SyncPing* pMsg);
-void syncPingLog(const SyncPing* pMsg);
-void syncPingLog2(char* s, const SyncPing* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncPingReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  uint32_t dataLen;
-  char     data[];
-} SyncPingReply;
-
-SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
-SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
-SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId);
-void           syncPingReplyDestroy(SyncPingReply* pMsg);
-void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
-void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
-char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);  //
-SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);           //
-void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
-void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
-SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);  //
-cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
-char*          syncPingReply2Str(const SyncPingReply* pMsg);  //
-
-// for debug ----------------------
-void syncPingReplyPrint(const SyncPingReply* pMsg);
-void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
-void syncPingReplyLog(const SyncPingReply* pMsg);
-void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncClientRequest {
-  uint32_t bytes;
-  uint32_t msgType;
-  uint32_t originalRpcType;
-  uint64_t seqNum;
-  bool     isWeak;
-  uint32_t dataLen;
-  char     data[];
-} SyncClientRequest;
-
-SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
-SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak);  // step 1
-void               syncClientRequestDestroy(SyncClientRequest* pMsg);
-void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
-void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
-char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
-SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
-void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
-void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
-SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
-cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
-char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
-
-// for debug ----------------------
-void syncClientRequestPrint(const SyncClientRequest* pMsg);
-void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
-void syncClientRequestLog(const SyncClientRequest* pMsg);
-void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncClientRequestReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  int32_t  errCode;
-  SRaftId  leaderHint;
-} SyncClientRequestReply;
-
-// ---------------------------------------------
-typedef struct SyncRequestVote {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  SyncIndex lastLogIndex;
-  SyncTerm  lastLogTerm;
-} SyncRequestVote;
-
-SyncRequestVote* syncRequestVoteBuild();
-void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
-void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
-void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
-char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
-SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
-void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
-void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
-SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
-char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
-
-// for debug ----------------------
-void syncRequestVotePrint(const SyncRequestVote* pMsg);
-void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
-void syncRequestVoteLog(const SyncRequestVote* pMsg);
-void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncRequestVoteReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm term;
-  bool     voteGranted;
-} SyncRequestVoteReply;
-
-SyncRequestVoteReply* syncRequestVoteReplyBuild();
-void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
-void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
-void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
-char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
-SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
-void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
-void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
-SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
-char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
-
-// for debug ----------------------
-void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncAppendEntries {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  SyncIndex prevLogIndex;
-  SyncTerm  prevLogTerm;
-  SyncIndex commitIndex;
-  uint32_t  dataLen;
-  char      data[];
-} SyncAppendEntries;
-
-SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen);
-void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
-void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
-void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
-char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
-SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
-void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
-void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
-SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
-char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
-
-// for debug ----------------------
-void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
-void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
-void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
-void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncAppendEntriesReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  bool      success;
-  SyncIndex matchIndex;
-} SyncAppendEntriesReply;
-
-SyncAppendEntriesReply* syncAppendEntriesReplyBuild();
-void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
-void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
-void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
-char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
-SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
-void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
-void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
-SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
-char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
-
-// for debug ----------------------
-void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncRaftCfg.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_CFG_H
+#define _TD_LIBS_SYNC_RAFT_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "cJSON.h"
+#include "syncInt.h"
+#include "taosdef.h"
+
+typedef struct SRaftCfg {
+  SSyncCfg  cfg;
+  TdFilePtr pFile;
+  char      path[TSDB_FILENAME_LEN * 2];
+} SRaftCfg;
+
+SRaftCfg *raftCfgOpen(const char *path);
+int32_t   raftCfgClose(SRaftCfg *pRaftCfg);
+int32_t   raftCfgPersist(SRaftCfg *pRaftCfg);
+
+cJSON * syncCfg2Json(SSyncCfg *pSyncCfg);
+char *  syncCfg2Str(SSyncCfg *pSyncCfg);
+int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
+int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg);
+
+cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
+char * raftCfg2Str(SRaftCfg *pRaftCfg);
+
+int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path);
+
+// for debug ----------------------
+void syncCfgPrint(SSyncCfg *pCfg);
+void syncCfgPrint2(char *s, SSyncCfg *pCfg);
+void syncCfgLog(SSyncCfg *pCfg);
+void syncCfgLog2(char *s, SSyncCfg *pCfg);
+
+void raftCfgPrint(SRaftCfg *pCfg);
+void raftCfgPrint2(char *s, SRaftCfg *pCfg);
+void raftCfgLog(SRaftCfg *pCfg);
+void raftCfgLog2(char *s, SRaftCfg *pCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_RAFT_CFG_H*/

source/libs/sync/inc/syncRaftEntry.h

@@ -27,29 +27,22 @@ extern "C" {
 #include "syncMessage.h"
 #include "taosdef.h"
 
-typedef enum EntryType {
-  SYNC_RAFT_ENTRY_NOOP = 0,
-  SYNC_RAFT_ENTRY_DATA = 1,
-  SYNC_RAFT_ENTRY_CONFIG = 2,
-} EntryType;
-
 typedef struct SSyncRaftEntry {
   uint32_t  bytes;
-  uint32_t  msgType;
+  uint32_t  msgType;          // SyncClientRequest msgType
-  uint32_t  originalRpcType;
+  uint32_t  originalRpcType;  // user RpcMsg msgType
   uint64_t  seqNum;
   bool      isWeak;
   SyncTerm  term;
   SyncIndex index;
-  EntryType entryType;
+  uint32_t  dataLen;  // user RpcMsg.contLen
-  uint32_t  dataLen;
+  char      data[];   // user RpcMsg.pCont
-  char      data[];
 } SSyncRaftEntry;
 
 SSyncRaftEntry* syncEntryBuild(uint32_t dataLen);
 SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);  // step 4
-SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType);
+SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);
-SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index);
+SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId);
 void            syncEntryDestory(SSyncRaftEntry* pEntry);
 char*           syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len);  // step 5
 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len);              // step 6

source/libs/sync/inc/syncRaftLog.h

@@ -27,9 +27,6 @@ extern "C" {
 #include "syncRaftEntry.h"
 #include "taosdef.h"
 
-#define SYNC_INDEX_BEGIN 0
-#define SYNC_INDEX_INVALID -1
-
 typedef struct SSyncLogStoreData {
   SSyncNode* pSyncNode;
   SWal*      pWal;

source/libs/sync/inc/syncRaftStore.h

@@ -28,7 +28,7 @@ extern "C" {
 #include "taosdef.h"
 
 #define RAFT_STORE_BLOCK_SIZE 512
-#define RAFT_STORE_PATH_LEN 128
+#define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2)
 
 typedef struct SRaftStore {
   SyncTerm  currentTerm;

source/libs/sync/inc/syncRespMgr.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RESP_MGR_H
+#define _TD_LIBS_SYNC_RESP_MGR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "syncInt.h"
+#include "taosdef.h"
+
+typedef struct SRespStub {
+  SRpcMsg rpcMsg;
+  int64_t createTime;
+} SRespStub;
+
+typedef struct SSyncRespMgr {
+  SHashObj *    pRespHash;
+  int64_t       ttl;
+  void *        data;
+  TdThreadMutex mutex;
+  uint64_t      seqNum;
+} SSyncRespMgr;
+
+SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl);
+void          syncRespMgrDestroy(SSyncRespMgr *pObj);
+int64_t       syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub);
+int32_t       syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index);
+int32_t       syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub);
+int32_t       syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub);
+void          syncRespClean(SSyncRespMgr *pObj);
+void          syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_RESP_MGR_H*/

source/libs/sync/inc/syncSnapshot.h

@@ -27,7 +27,6 @@ extern "C" {
 #include "taosdef.h"
 
 int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
-
 int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
 
 #ifdef __cplusplus

source/libs/sync/inc/syncUtil.h

@@ -55,6 +55,8 @@ char*       syncUtilprintBin(char* ptr, uint32_t len);
 char*       syncUtilprintBin2(char* ptr, uint32_t len);
 SyncIndex   syncUtilMinIndex(SyncIndex a, SyncIndex b);
 SyncIndex   syncUtilMaxIndex(SyncIndex a, SyncIndex b);
+void        syncUtilMsgHtoN(void* msg);
+void        syncUtilMsgNtoH(void* msg);
 
 #ifdef __cplusplus
 }

source/libs/sync/src/syncAppendEntries.c

@@ -125,7 +125,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         "logOK:%d",
         pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK);
 
-    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
     pReply->srcId = ths->myRaftId;
     pReply->destId = pMsg->srcId;
     pReply->term = ths->pRaftStore->currentTerm;
@@ -199,13 +199,20 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
-            // maybe is a NOOP ENTRY
+            if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
-            // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA);
+              SRpcMsg rpcMsg;
+              syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
+
+              SFsmCbMeta cbMeta;
+              cbMeta.index = pRollBackEntry->index;
+              cbMeta.isWeak = pRollBackEntry->isWeak;
+              cbMeta.code = 0;
+              cbMeta.state = ths->state;
+              cbMeta.seqNum = pRollBackEntry->seqNum;
+              ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
+              rpcFreeCont(rpcMsg.pCont);
+            }
 
-            SRpcMsg rpcMsg;
-            syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
-            ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state);
-            rpcFreeCont(rpcMsg.pCont);
             syncEntryDestory(pRollBackEntry);
           }
         }
@@ -220,8 +227,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-            ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state);
+            SFsmCbMeta cbMeta;
+            cbMeta.index = pAppendEntry->index;
+            cbMeta.isWeak = pAppendEntry->isWeak;
+            cbMeta.code = 2;
+            cbMeta.state = ths->state;
+            cbMeta.seqNum = pAppendEntry->seqNum;
+            ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
           }
         }
         rpcFreeCont(rpcMsg.pCont);
@@ -245,8 +258,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-          ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state);
+          SFsmCbMeta cbMeta;
+          cbMeta.index = pAppendEntry->index;
+          cbMeta.isWeak = pAppendEntry->isWeak;
+          cbMeta.code = 3;
+          cbMeta.state = ths->state;
+          cbMeta.seqNum = pAppendEntry->seqNum;
+          ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
         }
       }
       rpcFreeCont(rpcMsg.pCont);
@@ -261,7 +280,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       assert(0);
     }
 
-    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
     pReply->srcId = ths->myRaftId;
     pReply->destId = pMsg->srcId;
     pReply->term = ths->pRaftStore->currentTerm;
@@ -301,8 +320,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+              if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-                ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state);
+                SFsmCbMeta cbMeta;
+                cbMeta.index = pEntry->index;
+                cbMeta.isWeak = pEntry->isWeak;
+                cbMeta.code = 0;
+                cbMeta.state = ths->state;
+                cbMeta.seqNum = pEntry->seqNum;
+                ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
               }
 
               rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncAppendEntriesReply.c

@@ -56,6 +56,15 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p
   //    syncNodeUpdateTerm(ths, pMsg->term);
   //  }
 
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128];
+    snprintf(logBuf, sizeof(logBuf), "syncNodeOnAppendEntriesReplyCb error term, receive:%lu current:%lu", pMsg->term,
+             ths->pRaftStore->currentTerm);
+    syncNodeLog2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
+
   assert(pMsg->term == ths->pRaftStore->currentTerm);
 
   if (pMsg->success) {

source/libs/sync/src/syncCommit.c

@@ -97,8 +97,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-            pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state);
+            SFsmCbMeta cbMeta;
+            cbMeta.index = pEntry->index;
+            cbMeta.isWeak = pEntry->isWeak;
+            cbMeta.code = 0;
+            cbMeta.state = pSyncNode->state;
+            cbMeta.seqNum = pEntry->seqNum;
+            pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
           }
 
           rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncElection.c

@@ -35,7 +35,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
 
   int32_t ret = 0;
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    SyncRequestVote* pMsg = syncRequestVoteBuild();
+    SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId);
     pMsg->srcId = pSyncNode->myRaftId;
     pMsg->destId = pSyncNode->peersId[i];
     pMsg->term = pSyncNode->pRaftStore->currentTerm;

source/libs/sync/src/syncEnv.c

@@ -29,8 +29,10 @@ static void      syncEnvTick(void *param, void *tmrId);
 int32_t syncEnvStart() {
   int32_t ret = 0;
   taosSeedRand(taosGetTimestampSec());
-  gSyncEnv = doSyncEnvStart(gSyncEnv);
+  //gSyncEnv = doSyncEnvStart(gSyncEnv);
+  gSyncEnv = doSyncEnvStart();
   assert(gSyncEnv != NULL);
+  sTrace("syncEnvStart ok!");
   return ret;
 }
 
@@ -90,7 +92,12 @@ static SSyncEnv *doSyncEnvStart() {
 }
 
 static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) {
-  taosTmrCleanUp(pSyncEnv->pTimerManager);
+  assert(pSyncEnv == gSyncEnv);
+  if (pSyncEnv != NULL) {
+    taosTmrCleanUp(pSyncEnv->pTimerManager);
+    taosMemoryFree(pSyncEnv);
+  }
+  gSyncEnv = NULL;
   return 0;
 }
 

source/libs/sync/src/syncIO.c

@@ -29,7 +29,7 @@ static int32_t  syncIODestroy(SSyncIO *io);
 static int32_t  syncIOStartInternal(SSyncIO *io);
 static int32_t  syncIOStopInternal(SSyncIO *io);
 
-static void   *syncIOConsumerFunc(void *param);
+static void *  syncIOConsumerFunc(void *param);
 static void    syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static void    syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
@@ -70,9 +70,15 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
   assert(pEpSet->numOfEps == 1);
 
   int32_t ret = 0;
-  char    logBuf[256];
+  {
-  snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
+    syncUtilMsgNtoH(pMsg->pCont);
-  syncRpcMsgPrint2(logBuf, pMsg);
+
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
+    syncRpcMsgLog2(logBuf, pMsg);
+
+    syncUtilMsgHtoN(pMsg->pCont);
+  }
 
   pMsg->handle = NULL;
   pMsg->noResp = 1;
@@ -83,7 +89,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
 int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) {
   int32_t ret = 0;
   char    logBuf[128];
-  syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg);
+  syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg);
 
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
@@ -235,9 +241,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) {
 }
 
 static void *syncIOConsumerFunc(void *param) {
-  SSyncIO   *io = param;
+  SSyncIO *  io = param;
   STaosQall *qall;
-  SRpcMsg   *pRpcMsg, rpcMsg;
+  SRpcMsg *  pRpcMsg, rpcMsg;
   qall = taosAllocateQall();
 
   while (1) {
@@ -252,7 +258,7 @@ static void *syncIOConsumerFunc(void *param) {
       syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg);
 
       // use switch case instead of if else
-      if (pRpcMsg->msgType == SYNC_PING) {
+      if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
         if (io->FpOnSyncPing != NULL) {
           SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -260,7 +266,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncPingDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_PING_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
         if (io->FpOnSyncPingReply != NULL) {
           SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -268,7 +274,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncPingReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
         if (io->FpOnSyncClientRequest != NULL) {
           SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -276,7 +282,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncClientRequestDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
         if (io->FpOnSyncRequestVote != NULL) {
           SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -284,7 +290,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncRequestVoteDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
         if (io->FpOnSyncRequestVoteReply != NULL) {
           SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -292,7 +298,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncRequestVoteReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
         if (io->FpOnSyncAppendEntries != NULL) {
           SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -300,7 +306,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncAppendEntriesDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
         if (io->FpOnSyncAppendEntriesReply != NULL) {
           SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -308,7 +314,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncAppendEntriesReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_TIMEOUT) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
         if (io->FpOnSyncTimeout != NULL) {
           SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -336,7 +342,7 @@ static void *syncIOConsumerFunc(void *param) {
               rpcMsg.handle = pRpcMsg->handle;
               rpcMsg.code = 0;
 
-              syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
+              syncRpcMsgLog2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
               rpcSendResponse(&rpcMsg);
             }
       */
@@ -350,7 +356,9 @@ static void *syncIOConsumerFunc(void *param) {
 }
 
 static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg);
+  syncUtilMsgNtoH(pMsg->pCont);
+
+  syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg);
   SSyncIO *io = pParent;
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
@@ -359,10 +367,10 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
 }
 
 static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  if (pMsg->msgType == SYNC_RESPONSE) {
+  if (pMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) {
     sTrace("==syncIOProcessReply==");
   } else {
-    syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg);
+    syncRpcMsgLog2((char *)"==syncIOProcessReply==", pMsg);
   }
   rpcFreeCont(pMsg->pCont);
 }
@@ -408,14 +416,14 @@ static void syncIOTickQ(void *param, void *tmrId) {
   srcId.vgId = -1;
   destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
   destId.vgId = -1;
-  SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ");
+  SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, -1, "syncIOTickQ");
 
   SRpcMsg rpcMsg;
   syncPingReply2RpcMsg(pMsg, &rpcMsg);
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
   memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg));
-  syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg);
+  syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg);
   taosWriteQitem(io->pMsgQ, pTemp);
   syncPingReplyDestroy(pMsg);
 
@@ -430,12 +438,12 @@ static void syncIOTickPing(void *param, void *tmrId) {
   srcId.vgId = -1;
   destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
   destId.vgId = -1;
-  SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing");
+  SyncPing *pMsg = syncPingBuild2(&srcId, &destId, -1, "syncIOTickPing");
   // SyncPing *pMsg = syncPingBuild3(&srcId, &destId);
 
   SRpcMsg rpcMsg;
   syncPing2RpcMsg(pMsg, &rpcMsg);
-  syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg);
+  syncRpcMsgLog2((char *)"==syncIOTickPing==", &rpcMsg);
   rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL);
   syncPingDestroy(pMsg);
 

source/libs/sync/src/syncMain.c

@@ -23,11 +23,13 @@
 #include "syncIndexMgr.h"
 #include "syncInt.h"
 #include "syncMessage.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncReplication.h"
 #include "syncRequestVote.h"
 #include "syncRequestVoteReply.h"
+#include "syncRespMgr.h"
 #include "syncTimeout.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
@@ -44,9 +46,9 @@ static int32_t syncNodeEqNoop(SSyncNode* ths);
 static int32_t syncNodeAppendNoop(SSyncNode* ths);
 
 // process message ----
-static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
-static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
-static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
 
 // life cycle
 static void syncFreeNode(void* param);
@@ -76,10 +78,12 @@ void syncCleanUp() {
   }
 }
 
-int64_t syncStart(const SSyncInfo* pSyncInfo) {
+int64_t syncOpen(const SSyncInfo* pSyncInfo) {
   SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
   assert(pSyncNode != NULL);
 
+  syncNodeLog2("syncNodeOpen open success", pSyncNode);
+
   pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode);
   if (pSyncNode->rid < 0) {
     syncFreeNode(pSyncNode);
@@ -89,6 +93,16 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) {
   return pSyncNode->rid;
 }
 
+void syncStart(int64_t rid) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  syncNodeStart(pSyncNode);
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
 void syncStop(int64_t rid) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
@@ -105,11 +119,6 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
   return ret;
 }
 
-int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
-  int32_t ret = syncPropose2(rid, pMsg, isWeak, 0);
-  return ret;
-}
-
 int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
   int32_t ret = syncPropose(rid, pMsg, isWeak);
   return ret;
@@ -121,28 +130,163 @@ ESyncState syncGetMyRole(int64_t rid) {
     return TAOS_SYNC_STATE_ERROR;
   }
   assert(rid == pSyncNode->rid);
-  return pSyncNode->state;
+  ESyncState state = pSyncNode->state;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return state;
 }
 
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum) {
+const char* syncGetMyRoleStr(int64_t rid) {
-  int32_t    ret = 0;
+  const char* s = syncUtilState2String(syncGetMyRole(rid));
+  return s;
+}
+
+SyncTerm syncGetMyTerm(int64_t rid) {
   SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
   if (pSyncNode == NULL) {
-    return -1;
+    return TAOS_SYNC_STATE_ERROR;
+  }
+  assert(rid == pSyncNode->rid);
+  SyncTerm term = pSyncNode->pRaftStore->currentTerm;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return term;
+}
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return TAOS_SYNC_STATE_ERROR;
+  }
+  assert(rid == pSyncNode->rid);
+
+  SRespStub stub;
+  int32_t   ret = syncRespMgrGet(pSyncNode->pSyncRespMgr, index, &stub);
+  if (ret == 1) {
+    memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg));
+  }
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return ret;
+}
+
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return TAOS_SYNC_STATE_ERROR;
+  }
+  assert(rid == pSyncNode->rid);
+
+  SRespStub stub;
+  int32_t   ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub);
+  if (ret == 1) {
+    memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg));
+  }
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return ret;
+}
+
+void syncSetQ(int64_t rid, void* queue) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    sTrace("syncSetQ get pSyncNode is NULL, rid:%ld", rid);
+    return;
+  }
+  assert(rid == pSyncNode->rid);
+  pSyncNode->queue = queue;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
+void syncSetRpc(int64_t rid, void* rpcHandle) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid);
+    return;
+  }
+  assert(rid == pSyncNode->rid);
+  pSyncNode->rpcClient = rpcHandle;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
+char* sync2SimpleStr(int64_t rid) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid);
+    return NULL;
+  }
+  assert(rid == pSyncNode->rid);
+  char* s = syncNode2SimpleStr(pSyncNode);
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+
+  return s;
+}
+
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  assert(rid == pSyncNode->rid);
+  pSyncNode->pingBaseLine = pingTimerMS;
+  pSyncNode->pingTimerMS = pingTimerMS;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
+void setElectTimerMS(int64_t rid, int32_t electTimerMS) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  assert(rid == pSyncNode->rid);
+  pSyncNode->electBaseLine = electTimerMS;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return;
+  }
+  assert(rid == pSyncNode->rid);
+  pSyncNode->hbBaseLine = hbTimerMS;
+  pSyncNode->heartbeatTimerMS = hbTimerMS;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+}
+
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
+  int32_t    ret = TAOS_SYNC_PROPOSE_SUCCESS;
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return TAOS_SYNC_PROPOSE_OTHER_ERROR;
   }
   assert(rid == pSyncNode->rid);
 
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
-    SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak);
+    SRespStub stub;
+    stub.createTime = taosGetTimestampMs();
+    stub.rpcMsg = *pMsg;
+    uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub);
+
+    SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak, pSyncNode->vgId);
     SRpcMsg            rpcMsg;
     syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
-    pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    if (pSyncNode->FpEqMsg != NULL) {
+      pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    } else {
+      sTrace("syncPropose pSyncNode->FpEqMsg is NULL");
+    }
     syncClientRequestDestroy(pSyncMsg);
-    ret = 0;
+    ret = TAOS_SYNC_PROPOSE_SUCCESS;
 
   } else {
     sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state));
-    ret = -1;  // todo : need define err code !!
+    ret = TAOS_SYNC_PROPOSE_NOT_LEADER;
   }
 
   taosReleaseRef(tsNodeRefId, pSyncNode->rid);
@@ -155,49 +299,62 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   assert(pSyncNode != NULL);
   memset(pSyncNode, 0, sizeof(SSyncNode));
 
-  if (taosMkDir(pSyncInfo->path) != 0) {
+  int32_t ret = 0;
-    terrno = TAOS_SYSTEM_ERROR(errno);
+  if (!taosDirExist((char*)(pSyncInfo->path))) {
-    sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr());
+    if (taosMkDir(pSyncInfo->path) != 0) {
-    return NULL;
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr());
+      return NULL;
+    }
+
+    // create raft config file
+    snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path);
+    ret = syncCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), pSyncNode->configPath);
+    assert(ret == 0);
   }
 
   // init by SSyncInfo
   pSyncNode->vgId = pSyncInfo->vgId;
-  pSyncNode->syncCfg = pSyncInfo->syncCfg;
   memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
   snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s/raft_store.json", pSyncInfo->path);
+  snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path);
+
   pSyncNode->pWal = pSyncInfo->pWal;
   pSyncNode->rpcClient = pSyncInfo->rpcClient;
   pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg;
   pSyncNode->queue = pSyncInfo->queue;
   pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg;
 
+  // init raft config
+  pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
+  assert(pSyncNode->pRaftCfg != NULL);
+
   // init internal
-  pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
+  pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
-  syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId);
+  syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId);
 
   // init peersNum, peers, peersId
-  pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
+  pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
   int j = 0;
-  for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
+  for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
-    if (i != pSyncInfo->syncCfg.myIndex) {
+    if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
-      pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i];
+      pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
       j++;
     }
   }
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]);
+    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]);
   }
 
   // init replicaNum, replicasId
-  pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum;
+  pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
-  for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
+  for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
-    syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]);
+    syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
   }
 
   // init raft algorithm
   pSyncNode->pFsm = pSyncInfo->pFsm;
-  pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum);
+  pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
   pSyncNode->leaderCache = EMPTY_RAFT_ID;
 
   // init life cycle outside
@@ -287,10 +444,25 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb;
   pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb;
 
+  // tools
+  pSyncNode->pSyncRespMgr = syncRespMgrCreate(NULL, 0);
+  assert(pSyncNode->pSyncRespMgr != NULL);
+
+  // start in syncNodeStart
+  // start raft
+  // syncNodeBecomeFollower(pSyncNode);
+
+  return pSyncNode;
+}
+
+void syncNodeStart(SSyncNode* pSyncNode) {
   // start raft
   syncNodeBecomeFollower(pSyncNode);
 
-  return pSyncNode;
+  // for test
+  int32_t ret = 0;
+  // ret = syncNodeStartPingTimer(pSyncNode);
+  assert(ret == 0);
 }
 
 void syncNodeClose(SSyncNode* pSyncNode) {
@@ -305,12 +477,18 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   syncIndexMgrDestroy(pSyncNode->pNextIndex);
   syncIndexMgrDestroy(pSyncNode->pMatchIndex);
   logStoreDestory(pSyncNode->pLogStore);
+  raftCfgClose(pSyncNode->pRaftCfg);
 
   syncNodeStopPingTimer(pSyncNode);
   syncNodeStopElectTimer(pSyncNode);
   syncNodeStopHeartbeatTimer(pSyncNode);
 
-  taosMemoryFree(pSyncNode);
+  if (pSyncNode->pFsm != NULL) {
+    taosMemoryFree(pSyncNode->pFsm);
+  }
+
+  // free memory in syncFreeNode
+  // taosMemoryFree(pSyncNode);
 }
 
 // ping --------------
@@ -328,7 +506,7 @@ int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing*
 
 int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
   int32_t   ret = 0;
-  SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
+  SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId);
   ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg);
   assert(ret == 0);
 
@@ -339,10 +517,9 @@ int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
 int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
   int32_t ret = 0;
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    SRaftId destId;
+    SRaftId*  destId = &(pSyncNode->peersId[i]);
-    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
+    ret = syncNodePing(pSyncNode, destId, pMsg);
-    ret = syncNodePing(pSyncNode, &destId, pMsg);
     assert(ret == 0);
     syncPingDestroy(pMsg);
   }
@@ -351,11 +528,10 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
 
 int32_t syncNodePingAll(SSyncNode* pSyncNode) {
   int32_t ret = 0;
-  for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) {
+  for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
-    SRaftId destId;
+    SRaftId*  destId = &(pSyncNode->replicasId[i]);
-    syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
+    ret = syncNodePing(pSyncNode, destId, pMsg);
-    ret = syncNodePing(pSyncNode, &destId, pMsg);
     assert(ret == 0);
     syncPingDestroy(pMsg);
   }
@@ -430,14 +606,30 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;
   syncUtilraftId2EpSet(destRaftId, &epSet);
-  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  if (pSyncNode->FpSendMsg != NULL) {
+    pMsg->noResp = 1;
+    // htonl
+    syncUtilMsgHtoN(pMsg->pCont);
+
+    pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  } else {
+    sTrace("syncNodeSendMsgById pSyncNode->FpSendMsg is NULL");
+  }
   return 0;
 }
 
 int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;
   syncUtilnodeInfo2EpSet(nodeInfo, &epSet);
-  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  if (pSyncNode->FpSendMsg != NULL) {
+    pMsg->noResp = 1;
+    // htonl
+    syncUtilMsgHtoN(pMsg->pCont);
+
+    pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  } else {
+    sTrace("syncNodeSendMsgByInfo pSyncNode->FpSendMsg is NULL");
+  }
   return 0;
 }
 
@@ -448,7 +640,11 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   if (pSyncNode != NULL) {
     // init by SSyncInfo
     cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId);
+    cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg));
     cJSON_AddStringToObject(pRoot, "path", pSyncNode->path);
+    cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath);
+    cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath);
+
     snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal);
     cJSON_AddStringToObject(pRoot, "pWal", u64buf);
 
@@ -501,9 +697,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
     // tla+ server vars
     cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state);
     cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state));
-    char tmpBuf[RAFT_STORE_BLOCK_SIZE];
+    cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore));
-    raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf));
-    cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf);
 
     // tla+ candidate vars
     cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted));
@@ -518,6 +712,11 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
     snprintf(u64buf, sizeof(u64buf), "%" PRId64 "", pSyncNode->commitIndex);
     cJSON_AddStringToObject(pRoot, "commitIndex", u64buf);
 
+    // timer ms init
+    cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine);
+    cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine);
+    cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine);
+
     // ping timer
     snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer);
     cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf);
@@ -586,6 +785,19 @@ char* syncNode2Str(const SSyncNode* pSyncNode) {
   return serialized;
 }
 
+char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
+  int   len = 256;
+  char* s = (char*)taosMemoryMalloc(len);
+  snprintf(s, len,
+           "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, electTimerLogicClock:%lu, "
+           "electTimerLogicClockUser:%lu, "
+           "electTimerMS:%d",
+           pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state,
+           syncUtilState2String(pSyncNode->state), pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser,
+           pSyncNode->electTimerMS);
+  return s;
+}
+
 SSyncNode* syncNodeAcquire(int64_t rid) {
   SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid);
   if (pNode == NULL) {
@@ -672,24 +884,37 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
   assert(voteGrantedMajority(pSyncNode->pVotesGranted));
   syncNodeBecomeLeader(pSyncNode);
 
+  syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode);
+
   // Raft 3.6.2 Committing entries from previous terms
+
+  // use this now
   syncNodeAppendNoop(pSyncNode);
+  syncMaybeAdvanceCommitIndex(pSyncNode);  // maybe only one replica
+
+  // do not use this
   // syncNodeEqNoop(pSyncNode);
 }
 
 void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
   assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER);
   pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE;
+
+  syncNodeLog2("==state change syncNodeFollower2Candidate==", pSyncNode);
 }
 
 void syncNodeLeader2Follower(SSyncNode* pSyncNode) {
   assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
   syncNodeBecomeFollower(pSyncNode);
+
+  syncNodeLog2("==state change syncNodeLeader2Follower==", pSyncNode);
 }
 
 void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {
   assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
   syncNodeBecomeFollower(pSyncNode);
+
+  syncNodeLog2("==state change syncNodeCandidate2Follower==", pSyncNode);
 }
 
 // raft vote --------------
@@ -707,7 +932,7 @@ void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId)
 void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
   syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId));
 
-  SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild();
+  SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild(pSyncNode->vgId);
   pMsg->srcId = pSyncNode->myRaftId;
   pMsg->destId = pSyncNode->myRaftId;
   pMsg->term = pSyncNode->pRaftStore->currentTerm;
@@ -735,13 +960,13 @@ void syncNodePrint2(char* s, SSyncNode* pObj) {
 
 void syncNodeLog(SSyncNode* pObj) {
   char* serialized = syncNode2Str(pObj);
-  sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized);
+  sTraceLong("syncNodeLog | len:%lu | %s", strlen(serialized), serialized);
   taosMemoryFree(serialized);
 }
 
 void syncNodeLog2(char* s, SSyncNode* pObj) {
   char* serialized = syncNode2Str(pObj);
-  sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  sTraceLong("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
   taosMemoryFree(serialized);
 }
 
@@ -751,11 +976,15 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) {
   SSyncNode* pSyncNode = (SSyncNode*)param;
   if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) {
     SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock),
-                                              pSyncNode->pingTimerMS, pSyncNode);
+                                              pSyncNode->pingTimerMS, pSyncNode->vgId, pSyncNode);
     SRpcMsg      rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
     syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg);
-    pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    if (pSyncNode->FpEqMsg != NULL) {
+      pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    } else {
+      sTrace("syncNodeEqPingTimer pSyncNode->FpEqMsg is NULL");
+    }
     syncTimeoutDestroy(pSyncMsg);
 
     taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
@@ -770,17 +999,21 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
   SSyncNode* pSyncNode = (SSyncNode*)param;
   if (atomic_load_64(&pSyncNode->electTimerLogicClockUser) <= atomic_load_64(&pSyncNode->electTimerLogicClock)) {
     SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_ELECTION, atomic_load_64(&pSyncNode->electTimerLogicClock),
-                                              pSyncNode->electTimerMS, pSyncNode);
+                                              pSyncNode->electTimerMS, pSyncNode->vgId, pSyncNode);
     SRpcMsg      rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
     syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg);
-    pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    if (pSyncNode->FpEqMsg != NULL) {
+      pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    } else {
+      sTrace("syncNodeEqElectTimer pSyncNode->FpEqMsg is NULL");
+    }
     syncTimeoutDestroy(pSyncMsg);
 
     // reset timer ms
     pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
-    taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
+    taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager,
-                 &pSyncNode->pPingTimer);
+                 &pSyncNode->pElectTimer);
   } else {
     sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "",
            pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser);
@@ -793,11 +1026,15 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
       atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) {
     SyncTimeout* pSyncMsg =
         syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock),
-                          pSyncNode->heartbeatTimerMS, pSyncNode);
+                          pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode);
     SRpcMsg rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
     syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg);
-    pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    if (pSyncNode->FpEqMsg != NULL) {
+      pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    } else {
+      sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL");
+    }
     syncTimeoutDestroy(pSyncMsg);
 
     taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager,
@@ -815,7 +1052,7 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) {
 
   SyncIndex       index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1;
   SyncTerm        term = ths->pRaftStore->currentTerm;
-  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index);
+  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId);
   assert(pEntry != NULL);
 
   uint32_t           entryLen;
@@ -826,7 +1063,11 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) {
 
   SRpcMsg rpcMsg;
   syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
-  ths->FpEqMsg(ths->queue, &rpcMsg);
+  if (ths->FpEqMsg != NULL) {
+    ths->FpEqMsg(ths->queue, &rpcMsg);
+  } else {
+    sTrace("syncNodeEqNoop pSyncNode->FpEqMsg is NULL");
+  }
 
   taosMemoryFree(serialized);
   syncClientRequestDestroy(pSyncMsg);
@@ -839,7 +1080,7 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 
   SyncIndex       index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1;
   SyncTerm        term = ths->pRaftStore->currentTerm;
-  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index);
+  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId);
   assert(pEntry != NULL);
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
@@ -851,18 +1092,34 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
 }
 
 // on message ----
-static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
+  // log state
+  char logBuf[1024];
+  snprintf(logBuf, sizeof(logBuf),
+           "==syncNodeOnPingCb== vgId:%d, state: %d, %s, term:%lu electTimerLogicClock:%lu, "
+           "electTimerLogicClockUser:%lu, electTimerMS:%d",
+           ths->vgId, ths->state, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm,
+           ths->electTimerLogicClock, ths->electTimerLogicClockUser, ths->electTimerMS);
+
   int32_t ret = 0;
-  syncPingLog2("==syncNodeOnPingCb==", pMsg);
+  syncPingLog2(logBuf, pMsg);
-  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId);
+  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId, ths->vgId);
   SRpcMsg        rpcMsg;
   syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
+
+  /*
+    // htonl
+    SMsgHead* pHead = rpcMsg.pCont;
+    pHead->contLen = htonl(pHead->contLen);
+    pHead->vgId = htonl(pHead->vgId);
+  */
+
   syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
 
   return ret;
 }
 
-static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
   int32_t ret = 0;
   syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg);
   return ret;
@@ -878,7 +1135,7 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
 //     /\ UNCHANGED <<messages, serverVars, candidateVars,
 //                    leaderVars, commitIndex>>
 //
-static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
   int32_t ret = 0;
   syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
 
@@ -898,8 +1155,14 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-        ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state);
+        SFsmCbMeta cbMeta;
+        cbMeta.index = pEntry->index;
+        cbMeta.isWeak = pEntry->isWeak;
+        cbMeta.code = 0;
+        cbMeta.state = ths->state;
+        cbMeta.seqNum = pEntry->seqNum;
+        ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
       }
     }
     rpcFreeCont(rpcMsg.pCont);
@@ -913,8 +1176,14 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
+      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
-        ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state);
+        SFsmCbMeta cbMeta;
+        cbMeta.index = pEntry->index;
+        cbMeta.isWeak = pEntry->isWeak;
+        cbMeta.code = 1;
+        cbMeta.state = ths->state;
+        cbMeta.seqNum = pEntry->seqNum;
+        ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
       }
     }
     rpcFreeCont(rpcMsg.pCont);
@@ -926,7 +1195,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
 
 static void syncFreeNode(void* param) {
   SSyncNode* pNode = param;
-  syncNodePrint2((char*)"==syncFreeNode==", pNode);
+  // inner object already free
+  // syncNodePrint2((char*)"==syncFreeNode==", pNode);
 
   taosMemoryFree(pNode);
 }

source/libs/sync/src/syncMessage.c

@@ -22,50 +22,50 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
   cJSON* pRoot;
 
   // in compiler optimization, switch case = if else constants
-  if (pRpcMsg->msgType == SYNC_TIMEOUT) {
+  if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
     SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncTimeout2Json(pSyncMsg);
     syncTimeoutDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_PING) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
     SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncPing2Json(pSyncMsg);
     syncPingDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_PING_REPLY) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
     SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncPingReply2Json(pSyncMsg);
     syncPingReplyDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
     SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncClientRequest2Json(pSyncMsg);
     syncClientRequestDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST_REPLY) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST_REPLY) {
     pRoot = syncRpcUnknownMsg2Json();
 
-  } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
     SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncRequestVote2Json(pSyncMsg);
     syncRequestVoteDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
     SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncRequestVoteReply2Json(pSyncMsg);
     syncRequestVoteReplyDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
     SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncAppendEntries2Json(pSyncMsg);
     syncAppendEntriesDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
     SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
     pRoot = syncAppendEntriesReply2Json(pSyncMsg);
     syncAppendEntriesReplyDestroy(pSyncMsg);
 
-  } else if (pRpcMsg->msgType == SYNC_RESPONSE) {
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) {
     pRoot = cJSON_CreateObject();
     char* s;
     s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen);
@@ -98,7 +98,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
 
 cJSON* syncRpcUnknownMsg2Json() {
   cJSON* pRoot = cJSON_CreateObject();
-  cJSON_AddNumberToObject(pRoot, "msgType", SYNC_UNKNOWN);
+  cJSON_AddNumberToObject(pRoot, "msgType", TDMT_VND_SYNC_UNKNOWN);
   cJSON_AddStringToObject(pRoot, "data", "unknown message");
 
   cJSON* pJson = cJSON_CreateObject();
@@ -146,12 +146,14 @@ SyncTimeout* syncTimeoutBuild() {
   SyncTimeout* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_TIMEOUT;
+  pMsg->msgType = TDMT_VND_SYNC_TIMEOUT;
   return pMsg;
 }
 
-SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, void* data) {
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data) {
   SyncTimeout* pMsg = syncTimeoutBuild();
+  pMsg->vgId = vgId;
   pMsg->timeoutType = timeoutType;
   pMsg->logicClock = logicClock;
   pMsg->timerMS = timerMS;
@@ -217,6 +219,7 @@ cJSON* syncTimeout2Json(const SyncTimeout* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
     cJSON_AddNumberToObject(pRoot, "timeoutType", pMsg->timeoutType);
     snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->logicClock);
@@ -271,22 +274,23 @@ SyncPing* syncPingBuild(uint32_t dataLen) {
   SyncPing* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_PING;
+  pMsg->msgType = TDMT_VND_SYNC_PING;
   pMsg->dataLen = dataLen;
   return pMsg;
 }
 
-SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) {
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str) {
   uint32_t  dataLen = strlen(str) + 1;
   SyncPing* pMsg = syncPingBuild(dataLen);
+  pMsg->vgId = vgId;
   pMsg->srcId = *srcId;
   pMsg->destId = *destId;
   snprintf(pMsg->data, pMsg->dataLen, "%s", str);
   return pMsg;
 }
 
-SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) {
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId) {
-  SyncPing* pMsg = syncPingBuild2(srcId, destId, "ping");
+  SyncPing* pMsg = syncPingBuild2(srcId, destId, vgId, "ping");
   return pMsg;
 }
 
@@ -326,6 +330,98 @@ SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) {
   return pMsg;
 }
 
+int32_t syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+  if (tStartEncode(&encoder) < 0) {
+    return -1;
+  }
+
+  if (tEncodeU32(&encoder, pMsg->bytes) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU32(&encoder, pMsg->msgType) < 0) {
+    return -1;
+  }
+  if (tEncodeU64(&encoder, pMsg->srcId.addr) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->srcId.vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU64(&encoder, pMsg->destId.addr) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->destId.vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU32(&encoder, pMsg->dataLen) < 0) {
+    return -1;
+  }
+  if (tEncodeBinary(&encoder, pMsg->data, pMsg->dataLen)) {
+    return -1;
+  }
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) {
+    return NULL;
+  }
+
+  SyncPing* pMsg = NULL;
+  uint32_t  bytes;
+  if (tDecodeU32(&decoder, &bytes) < 0) {
+    return NULL;
+  }
+
+  pMsg = taosMemoryMalloc(bytes);
+  assert(pMsg != NULL);
+  pMsg->bytes = bytes;
+
+  if (tDecodeI32(&decoder, &pMsg->vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU32(&decoder, &pMsg->msgType) < 0) {
+    return NULL;
+  }
+  if (tDecodeU64(&decoder, &pMsg->srcId.addr) < 0) {
+    return NULL;
+  }
+  if (tDecodeI32(&decoder, &pMsg->srcId.vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU64(&decoder, &pMsg->destId.addr) < 0) {
+    return NULL;
+  }
+  if (tDecodeI32(&decoder, &pMsg->destId.vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU32(&decoder, &pMsg->dataLen) < 0) {
+    return NULL;
+  }
+  uint64_t len;
+  char*    data = NULL;
+  if (tDecodeBinary(&decoder, (const void**)(&data), &len) < 0) {
+    return NULL;
+  }
+  assert(len = pMsg->dataLen);
+  memcpy(pMsg->data, data, len);
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return pMsg;
+}
+
 void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg) {
   memset(pRpcMsg, 0, sizeof(*pRpcMsg));
   pRpcMsg->msgType = pMsg->msgType;
@@ -349,6 +445,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -436,22 +533,23 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) {
   SyncPingReply* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_PING_REPLY;
+  pMsg->msgType = TDMT_VND_SYNC_PING_REPLY;
   pMsg->dataLen = dataLen;
   return pMsg;
 }
 
-SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str) {
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str) {
   uint32_t       dataLen = strlen(str) + 1;
   SyncPingReply* pMsg = syncPingReplyBuild(dataLen);
+  pMsg->vgId = vgId;
   pMsg->srcId = *srcId;
   pMsg->destId = *destId;
   snprintf(pMsg->data, pMsg->dataLen, "%s", str);
   return pMsg;
 }
 
-SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) {
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId) {
-  SyncPingReply* pMsg = syncPingReplyBuild2(srcId, destId, "pang");
+  SyncPingReply* pMsg = syncPingReplyBuild2(srcId, destId, vgId, "pang");
   return pMsg;
 }
 
@@ -491,6 +589,98 @@ SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) {
   return pMsg;
 }
 
+int32_t syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+  if (tStartEncode(&encoder) < 0) {
+    return -1;
+  }
+
+  if (tEncodeU32(&encoder, pMsg->bytes) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU32(&encoder, pMsg->msgType) < 0) {
+    return -1;
+  }
+  if (tEncodeU64(&encoder, pMsg->srcId.addr) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->srcId.vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU64(&encoder, pMsg->destId.addr) < 0) {
+    return -1;
+  }
+  if (tEncodeI32(&encoder, pMsg->destId.vgId) < 0) {
+    return -1;
+  }
+  if (tEncodeU32(&encoder, pMsg->dataLen) < 0) {
+    return -1;
+  }
+  if (tEncodeBinary(&encoder, pMsg->data, pMsg->dataLen)) {
+    return -1;
+  }
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) {
+    return NULL;
+  }
+
+  SyncPingReply* pMsg = NULL;
+  uint32_t       bytes;
+  if (tDecodeU32(&decoder, &bytes) < 0) {
+    return NULL;
+  }
+
+  pMsg = taosMemoryMalloc(bytes);
+  assert(pMsg != NULL);
+  pMsg->bytes = bytes;
+
+  if (tDecodeI32(&decoder, &pMsg->vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU32(&decoder, &pMsg->msgType) < 0) {
+    return NULL;
+  }
+  if (tDecodeU64(&decoder, &pMsg->srcId.addr) < 0) {
+    return NULL;
+  }
+  if (tDecodeI32(&decoder, &pMsg->srcId.vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU64(&decoder, &pMsg->destId.addr) < 0) {
+    return NULL;
+  }
+  if (tDecodeI32(&decoder, &pMsg->destId.vgId) < 0) {
+    return NULL;
+  }
+  if (tDecodeU32(&decoder, &pMsg->dataLen) < 0) {
+    return NULL;
+  }
+  uint64_t len;
+  char*    data = NULL;
+  if (tDecodeBinary(&decoder, (const void**)(&data), &len) < 0) {
+    return NULL;
+  }
+  assert(len = pMsg->dataLen);
+  memcpy(pMsg->data, data, len);
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return pMsg;
+}
+
 void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg) {
   memset(pRpcMsg, 0, sizeof(*pRpcMsg));
   pRpcMsg->msgType = pMsg->msgType;
@@ -514,6 +704,7 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -601,7 +792,7 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) {
   SyncClientRequest* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_CLIENT_REQUEST;
+  pMsg->msgType = TDMT_VND_SYNC_CLIENT_REQUEST;
   pMsg->seqNum = 0;
   pMsg->isWeak = false;
   pMsg->dataLen = dataLen;
@@ -609,8 +800,9 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) {
 }
 
 // step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak
-SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) {
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, int32_t vgId) {
   SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen);
+  pMsg->vgId = vgId;
   pMsg->originalRpcType = pOriginalRpcMsg->msgType;
   pMsg->seqNum = seqNum;
   pMsg->isWeak = isWeak;
@@ -678,6 +870,7 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
     cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType);
     snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum);
@@ -734,12 +927,13 @@ void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) {
 }
 
 // ---- message process SyncRequestVote----
-SyncRequestVote* syncRequestVoteBuild() {
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId) {
   uint32_t         bytes = sizeof(SyncRequestVote);
   SyncRequestVote* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_REQUEST_VOTE;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE;
   return pMsg;
 }
 
@@ -801,6 +995,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -880,12 +1075,13 @@ void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) {
 }
 
 // ---- message process SyncRequestVoteReply----
-SyncRequestVoteReply* syncRequestVoteReplyBuild() {
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId) {
   uint32_t              bytes = sizeof(SyncRequestVoteReply);
   SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_REQUEST_VOTE_REPLY;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE_REPLY;
   return pMsg;
 }
 
@@ -947,6 +1143,7 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -1023,12 +1220,13 @@ void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) {
 }
 
 // ---- message process SyncAppendEntries----
-SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) {
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId) {
   uint32_t           bytes = sizeof(SyncAppendEntries) + dataLen;
   SyncAppendEntries* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_APPEND_ENTRIES;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES;
   pMsg->dataLen = dataLen;
   return pMsg;
 }
@@ -1092,6 +1290,7 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -1186,12 +1385,13 @@ void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) {
 }
 
 // ---- message process SyncAppendEntriesReply----
-SyncAppendEntriesReply* syncAppendEntriesReplyBuild() {
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) {
   uint32_t                bytes = sizeof(SyncAppendEntriesReply);
   SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes);
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
-  pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES_REPLY;
   return pMsg;
 }
 
@@ -1253,6 +1453,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) {
 
   if (pMsg != NULL) {
     cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
     cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
 
     cJSON* pSrcId = cJSON_CreateObject();
@@ -1330,3 +1531,154 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
   sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
   taosMemoryFree(serialized);
 }
+
+// ---- message process SyncApplyMsg----
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
+  uint32_t      bytes = sizeof(SyncApplyMsg) + dataLen;
+  SyncApplyMsg* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->msgType = TDMT_VND_SYNC_APPLY_MSG;
+  pMsg->dataLen = dataLen;
+  return pMsg;
+}
+
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta) {
+  SyncApplyMsg* pMsg = syncApplyMsgBuild(pOriginalRpcMsg->contLen);
+  pMsg->vgId = vgId;
+  pMsg->originalRpcType = pOriginalRpcMsg->msgType;
+  pMsg->fsmMeta = *pMeta;
+  memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen);
+  return pMsg;
+}
+
+void syncApplyMsgDestroy(SyncApplyMsg* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen) {
+  assert(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg) {
+  memcpy(pMsg, buf, len);
+  assert(len == pMsg->bytes);
+}
+
+char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  assert(buf != NULL);
+  syncApplyMsgSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len) {
+  uint32_t      bytes = *((uint32_t*)buf);
+  SyncApplyMsg* pMsg = taosMemoryMalloc(bytes);
+  assert(pMsg != NULL);
+  syncApplyMsgDeserialize(buf, len, pMsg);
+  assert(len == pMsg->bytes);
+  return pMsg;
+}
+
+// SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncApplyMsgSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+// get SRpcMsg from ApplyQ, to SyncApplyMsg
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg) {
+  syncApplyMsgDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncApplyMsg* pMsg = syncApplyMsgDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  return pMsg;
+}
+
+// SyncApplyMsg to OriginalRpcMsg
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg) {
+  memset(pOriginalRpcMsg, 0, sizeof(*pOriginalRpcMsg));
+  pOriginalRpcMsg->msgType = pMsg->originalRpcType;
+  pOriginalRpcMsg->contLen = pMsg->dataLen;
+  pOriginalRpcMsg->pCont = rpcMallocCont(pOriginalRpcMsg->contLen);
+  memcpy(pOriginalRpcMsg->pCont, pMsg->data, pOriginalRpcMsg->contLen);
+}
+
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+    cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->fsmMeta.index);
+    cJSON_AddStringToObject(pRoot, "fsmMeta.index", u64buf);
+    cJSON_AddNumberToObject(pRoot, "fsmMeta.isWeak", pMsg->fsmMeta.isWeak);
+    cJSON_AddNumberToObject(pRoot, "fsmMeta.code", pMsg->fsmMeta.code);
+    cJSON_AddNumberToObject(pRoot, "fsmMeta.state", pMsg->fsmMeta.state);
+    cJSON_AddStringToObject(pRoot, "fsmMeta.state.str", syncUtilState2String(pMsg->fsmMeta.state));
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->fsmMeta.seqNum);
+    cJSON_AddStringToObject(pRoot, "fsmMeta.seqNum", u64buf);
+
+    cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen);
+    char* s;
+    s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen);
+    cJSON_AddStringToObject(pRoot, "data", s);
+    taosMemoryFree(s);
+    s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen);
+    cJSON_AddStringToObject(pRoot, "data2", s);
+    taosMemoryFree(s);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncApplyMsg", pRoot);
+  return pJson;
+}
+
+char* syncApplyMsg2Str(const SyncApplyMsg* pMsg) {
+  cJSON* pJson = syncApplyMsg2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg) {
+  char* serialized = syncApplyMsg2Str(pMsg);
+  printf("syncApplyMsgPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg) {
+  char* serialized = syncApplyMsg2Str(pMsg);
+  printf("syncApplyMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncApplyMsgLog(const SyncApplyMsg* pMsg) {
+  char* serialized = syncApplyMsg2Str(pMsg);
+  sTrace("ssyncApplyMsgLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) {
+  char* serialized = syncApplyMsg2Str(pMsg);
+  sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  taosMemoryFree(serialized);
+}

source/libs/sync/src/syncRaftCfg.c

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncRaftCfg.h"
+#include "cJSON.h"
+#include "syncEnv.h"
+#include "syncUtil.h"
+
+// file must already exist!
+SRaftCfg *raftCfgOpen(const char *path) {
+  SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg));
+  snprintf(pCfg->path, sizeof(pCfg->path), "%s", path);
+
+  pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE);
+  assert(pCfg->pFile != NULL);
+
+  taosLSeekFile(pCfg->pFile, 0, SEEK_SET);
+
+  char buf[1024];
+  int  len = taosReadFile(pCfg->pFile, buf, sizeof(buf));
+  assert(len > 0);
+
+  int32_t ret = syncCfgFromStr(buf, &(pCfg->cfg));
+  assert(ret == 0);
+
+  return pCfg;
+}
+
+int32_t raftCfgClose(SRaftCfg *pRaftCfg) {
+  int64_t ret = taosCloseFile(&(pRaftCfg->pFile));
+  assert(ret == 0);
+  taosMemoryFree(pRaftCfg);
+  return 0;
+}
+
+int32_t raftCfgPersist(SRaftCfg *pRaftCfg) {
+  assert(pRaftCfg != NULL);
+
+  char *s = syncCfg2Str(&(pRaftCfg->cfg));
+  taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET);
+  int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1);
+  assert(ret == strlen(s) + 1);
+  taosMemoryFree(s);
+
+  taosFsyncFile(pRaftCfg->pFile);
+  return 0;
+}
+
+cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  if (pSyncCfg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum);
+    cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex);
+
+    cJSON *pNodeInfoArr = cJSON_CreateArray();
+    cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr);
+    for (int i = 0; i < pSyncCfg->replicaNum; ++i) {
+      cJSON *pNodeInfo = cJSON_CreateObject();
+      cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort);
+      cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn);
+      cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo);
+    }
+  }
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SSyncCfg", pRoot);
+  return pJson;
+}
+
+char *syncCfg2Str(SSyncCfg *pSyncCfg) {
+  cJSON *pJson = syncCfg2Json(pSyncCfg);
+  char * serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {
+  memset(pSyncCfg, 0, sizeof(SSyncCfg));
+  cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg");
+
+  cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum");
+  assert(cJSON_IsNumber(pReplicaNum));
+  pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum);
+
+  cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex");
+  assert(cJSON_IsNumber(pMyIndex));
+  pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex);
+
+  cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo");
+  int    arraySize = cJSON_GetArraySize(pNodeInfoArr);
+  assert(arraySize == pSyncCfg->replicaNum);
+
+  for (int i = 0; i < arraySize; ++i) {
+    cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i);
+    assert(pNodeInfo != NULL);
+
+    cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort");
+    assert(cJSON_IsNumber(pNodePort));
+    ((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort);
+
+    cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn");
+    assert(cJSON_IsString(pNodeFqdn));
+    snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s",
+             pNodeFqdn->valuestring);
+  }
+
+  return 0;
+}
+
+int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) {
+  cJSON *pRoot = cJSON_Parse(s);
+  assert(pRoot != NULL);
+
+  int32_t ret = syncCfgFromJson(pRoot, pSyncCfg);
+  assert(ret == 0);
+
+  cJSON_Delete(pRoot);
+  return 0;
+}
+
+cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
+  cJSON *pJson = syncCfg2Json(&(pRaftCfg->cfg));
+  return pJson;
+}
+
+char *raftCfg2Str(SRaftCfg *pRaftCfg) {
+  char *s = syncCfg2Str(&(pRaftCfg->cfg));
+  return s;
+}
+
+int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path) {
+  assert(pCfg != NULL);
+
+  TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
+  assert(pFile != NULL);
+
+  char *  s = syncCfg2Str(pCfg);
+  int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
+  assert(ret == strlen(s) + 1);
+
+  taosMemoryFree(s);
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+// for debug ----------------------
+void syncCfgPrint(SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  printf("syncCfgPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgPrint2(char *s, SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  printf("syncCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgLog(SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  sTrace("syncCfgLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgLog2(char *s, SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  sTrace("syncCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgPrint(SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  printf("raftCfgPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgPrint2(char *s, SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  printf("raftCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgLog(SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  sTrace("raftCfgLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgLog2(char *s, SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  sTrace("raftCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  taosMemoryFree(serialized);
+}

source/libs/sync/src/syncRaftEntry.c

@@ -28,13 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) {
 
 // step 4. SyncClientRequest => SSyncRaftEntry, add term, index
 SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) {
-  SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA);
+  SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index);
   assert(pEntry != NULL);
 
   return pEntry;
 }
 
-SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) {
+SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) {
   SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen);
   assert(pEntry != NULL);
 
@@ -44,19 +44,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde
   pEntry->isWeak = pMsg->isWeak;
   pEntry->term = term;
   pEntry->index = index;
-  pEntry->entryType = entryType;
   pEntry->dataLen = pMsg->dataLen;
   memcpy(pEntry->data, pMsg->data, pMsg->dataLen);
 
   return pEntry;
 }
 
-SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) {
+SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) {
-  SSyncRaftEntry* pEntry = syncEntryBuild(0);
+  // init rpcMsg
+  SMsgHead head;
+  head.vgId = vgId;
+  head.contLen = sizeof(SMsgHead);
+  SRpcMsg rpcMsg;
+  memset(&rpcMsg, 0, sizeof(SRpcMsg));
+  rpcMsg.contLen = head.contLen;
+  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  rpcMsg.msgType = TDMT_VND_SYNC_NOOP;
+  memcpy(rpcMsg.pCont, &head, sizeof(head));
+
+  SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen);
   assert(pEntry != NULL);
+
+  pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST;
+  pEntry->originalRpcType = TDMT_VND_SYNC_NOOP;
+  pEntry->seqNum = 0;
+  pEntry->isWeak = 0;
   pEntry->term = term;
   pEntry->index = index;
-  pEntry->entryType = SYNC_RAFT_ENTRY_NOOP;
+
+  assert(pEntry->dataLen == rpcMsg.contLen);
+  memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen);
+  rpcFreeCont(rpcMsg.pCont);
 
   return pEntry;
 }
@@ -103,7 +121,6 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) {
     cJSON_AddStringToObject(pRoot, "term", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index);
     cJSON_AddStringToObject(pRoot, "index", u64buf);
-    cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType);
     cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen);
 
     char* s;

source/libs/sync/src/syncRaftLog.c

@@ -50,38 +50,46 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
 
   SyncIndex lastIndex = logStoreLastIndex(pLogStore);
   assert(pEntry->index == lastIndex + 1);
-  uint32_t len;
-  char*    serialized = syncEntrySerialize(pEntry, &len);
-  assert(serialized != NULL);
 
-  int code = 0;
+  int          code = 0;
-  /*
+  SSyncLogMeta syncMeta;
-    code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len);
+  syncMeta.isWeek = pEntry->isWeak;
-    assert(code == 0);
+  syncMeta.seqNum = pEntry->seqNum;
-  */
+  syncMeta.term = pEntry->term;
-  assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0);
+  code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
+  assert(code == 0);
 
   walFsync(pWal, true);
-  taosMemoryFree(serialized);
   return code;
 }
 
 SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  SSyncRaftEntry*    pEntry = NULL;
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
     SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
     assert(walReadWithHandle(pWalHandle, index) == 0);
-    pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len);
+
+    SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
     assert(pEntry != NULL);
 
+    pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST;
+    pEntry->originalRpcType = pWalHandle->pHead->head.msgType;
+    pEntry->seqNum = pWalHandle->pHead->head.syncMeta.seqNum;
+    pEntry->isWeak = pWalHandle->pHead->head.syncMeta.isWeek;
+    pEntry->term = pWalHandle->pHead->head.syncMeta.term;
+    pEntry->index = index;
+    assert(pEntry->dataLen == pWalHandle->pHead->head.bodyLen);
+    memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
+
     // need to hold, do not new every time!!
     walCloseReadHandle(pWalHandle);
-  }
+    return pEntry;
 
-  return pEntry;
+  } else {
+    return NULL;
+  }
 }
 
 int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
@@ -207,20 +215,20 @@ void logStorePrint(SSyncLogStore* pLogStore) {
 
 void logStorePrint2(char* s, SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  printf("logStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
   fflush(NULL);
   taosMemoryFree(serialized);
 }
 
 void logStoreLog(SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized);
+  sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized);
   taosMemoryFree(serialized);
 }
 
 void logStoreLog2(char* s, SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
   taosMemoryFree(serialized);
 }
 

source/libs/sync/src/syncReplication.c

@@ -77,7 +77,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
     SyncAppendEntries* pMsg = NULL;
     SSyncRaftEntry*    pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex);
     if (pEntry != NULL) {
-      pMsg = syncAppendEntriesBuild(pEntry->bytes);
+      pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId);
       assert(pMsg != NULL);
 
       // add pEntry into msg
@@ -91,7 +91,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
 
     } else {
       // maybe overflow, send empty record
-      pMsg = syncAppendEntriesBuild(0);
+      pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId);
       assert(pMsg != NULL);
     }
 

source/libs/sync/src/syncRequestVote.c

@@ -64,7 +64,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
     raftStoreVote(ths->pRaftStore, &(pMsg->srcId));
   }
 
-  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild();
+  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId);
   pReply->srcId = ths->myRaftId;
   pReply->destId = pMsg->srcId;
   pReply->term = ths->pRaftStore->currentTerm;

source/libs/sync/src/syncRequestVoteReply.c

@@ -49,11 +49,20 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg)
     return ret;
   }
 
-  assert(!(pMsg->term > ths->pRaftStore->currentTerm));
+  // assert(!(pMsg->term > ths->pRaftStore->currentTerm));
-  // no need this code, because if I receive reply.term, then I must have sent for that term.
+  //  no need this code, because if I receive reply.term, then I must have sent for that term.
-  //  if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //   if (pMsg->term > ths->pRaftStore->currentTerm) {
-  //    syncNodeUpdateTerm(ths, pMsg->term);
+  //     syncNodeUpdateTerm(ths, pMsg->term);
-  //  }
+  //   }
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128];
+    snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term,
+             ths->pRaftStore->currentTerm);
+    syncNodePrint2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
 
   assert(pMsg->term == ths->pRaftStore->currentTerm);
 

source/libs/sync/src/syncRespMgr.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncRespMgr.h"
+
+SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
+  SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr));
+  memset(pObj, 0, sizeof(SSyncRespMgr));
+
+  pObj->pRespHash =
+      taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  assert(pObj->pRespHash != NULL);
+  pObj->ttl = ttl;
+  pObj->data = data;
+  pObj->seqNum = 0;
+  taosThreadMutexInit(&(pObj->mutex), NULL);
+
+  return pObj;
+}
+
+void syncRespMgrDestroy(SSyncRespMgr *pObj) {
+  taosThreadMutexLock(&(pObj->mutex));
+  taosHashCleanup(pObj->pRespHash);
+  taosThreadMutexUnlock(&(pObj->mutex));
+  taosThreadMutexDestroy(&(pObj->mutex));
+  taosMemoryFree(pObj);
+}
+
+int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  uint64_t keyCode = ++(pObj->seqNum);
+  taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub));
+
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return keyCode;
+}
+
+int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  taosHashRemove(pObj->pRespHash, &index, sizeof(index));
+
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;
+}
+
+int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    memcpy(pStub, pTmp, sizeof(SRespStub));
+    taosThreadMutexUnlock(&(pObj->mutex));
+    return 1;  // get one object
+  }
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;  // get none object
+}
+
+int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    memcpy(pStub, pTmp, sizeof(SRespStub));
+    taosThreadMutexUnlock(&(pObj->mutex));
+    taosHashRemove(pObj->pRespHash, &index, sizeof(index));
+    return 1;  // get one object
+  }
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;  // get none object
+}
+
+void syncRespClean(SSyncRespMgr *pObj) {
+  taosThreadMutexLock(&(pObj->mutex));
+  syncRespCleanByTTL(pObj, pObj->ttl);
+  taosThreadMutexUnlock(&(pObj->mutex));
+}
+
+void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {}

source/libs/sync/src/syncUtil.c

@@ -16,6 +16,8 @@
 #include "syncUtil.h"
 #include "syncEnv.h"
 
+void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
 // ---- encode / decode
 uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
   uint64_t u64;
@@ -197,3 +199,17 @@ SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) {
   SyncIndex r = a > b ? a : b;
   return r;
 }
+
+void syncUtilMsgHtoN(void* msg) {
+  // htonl
+  SMsgHead* pHead = msg;
+  pHead->contLen = htonl(pHead->contLen);
+  pHead->vgId = htonl(pHead->vgId);
+}
+
+void syncUtilMsgNtoH(void* msg) {
+  // ntohl
+  SMsgHead* pHead = msg;
+  pHead->contLen = ntohl(pHead->contLen);
+  pHead->vgId = ntohl(pHead->vgId);
+}

source/libs/sync/test/CMakeLists.txt

@@ -28,16 +28,15 @@ add_executable(syncRpcMsgTest "")
 add_executable(syncPingTimerTest2 "")
 add_executable(syncPingSelfTest "")
 add_executable(syncElectTest "")
-add_executable(syncElectTest2 "")
-add_executable(syncElectTest3 "")
 add_executable(syncEncodeTest "")
 add_executable(syncWriteTest "")
 add_executable(syncReplicateTest "")
-add_executable(syncReplicateTest2 "")
-add_executable(syncReplicateTest3 "")
-add_executable(syncReplicateLoadTest "")
 add_executable(syncRefTest "")
 add_executable(syncLogStoreCheck "")
+add_executable(syncRaftCfgTest "")
+add_executable(syncRespMgrTest "")
+add_executable(syncSnapshotTest "")
+add_executable(syncApplyMsgTest "")
 
 
 target_sources(syncTest
@@ -160,14 +159,6 @@ target_sources(syncElectTest
     PRIVATE
     "syncElectTest.cpp"
 )
-target_sources(syncElectTest2
-    PRIVATE
-    "syncElectTest2.cpp"
-)
-target_sources(syncElectTest3
-    PRIVATE
-    "syncElectTest3.cpp"
-)
 target_sources(syncEncodeTest
     PRIVATE
     "syncEncodeTest.cpp"
@@ -180,18 +171,6 @@ target_sources(syncReplicateTest
     PRIVATE
     "syncReplicateTest.cpp"
 )
-target_sources(syncReplicateTest2
-    PRIVATE
-    "syncReplicateTest2.cpp"
-)
-target_sources(syncReplicateTest3
-    PRIVATE
-    "syncReplicateTest3.cpp"
-)
-target_sources(syncReplicateLoadTest
-    PRIVATE
-    "syncReplicateLoadTest.cpp"
-)
 target_sources(syncRefTest
     PRIVATE
     "syncRefTest.cpp"
@@ -200,206 +179,217 @@ target_sources(syncLogStoreCheck
     PRIVATE
     "syncLogStoreCheck.cpp"
 )
+target_sources(syncRaftCfgTest
+    PRIVATE
+    "syncRaftCfgTest.cpp"
+)
+target_sources(syncRespMgrTest
+    PRIVATE
+    "syncRespMgrTest.cpp"
+)
+target_sources(syncSnapshotTest
+    PRIVATE
+    "syncSnapshotTest.cpp"
+)
+target_sources(syncApplyMsgTest
+    PRIVATE
+    "syncApplyMsgTest.cpp"
+)
 
 
 target_include_directories(syncTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEnvTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTimerTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOTickQTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOTickPingTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOSendMsgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOClientTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOServerTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRaftStoreTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEnqTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIndexTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncInitTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncUtilTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncVotesGrantedTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncVotesRespondTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIndexMgrTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncLogStoreTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEntryTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRequestVoteTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRequestVoteReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncAppendEntriesTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncAppendEntriesReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncClientRequestTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncTimeoutTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRpcMsgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTimerTest2
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingSelfTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncElectTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncElectTest2
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncElectTest3
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEncodeTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncWriteTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncReplicateTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncReplicateTest2
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncReplicateTest3
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncReplicateLoadTest
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRefTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncLogStoreCheck
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncRaftCfgTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncRespMgrTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncSnapshotTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncApplyMsgTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 
@@ -524,14 +514,6 @@ target_link_libraries(syncElectTest
     sync
     gtest_main
 )
-target_link_libraries(syncElectTest2
-    sync
-    gtest_main
-)
-target_link_libraries(syncElectTest3
-    sync
-    gtest_main
-)
 target_link_libraries(syncEncodeTest
     sync
     gtest_main
@@ -544,18 +526,6 @@ target_link_libraries(syncReplicateTest
     sync
     gtest_main
 )
-target_link_libraries(syncReplicateTest2
-    sync
-    gtest_main
-)
-target_link_libraries(syncReplicateTest3
-    sync
-    gtest_main
-)
-target_link_libraries(syncReplicateLoadTest
-    sync
-    gtest_main
-)
 target_link_libraries(syncRefTest
     sync
     gtest_main
@@ -564,6 +534,22 @@ target_link_libraries(syncLogStoreCheck
     sync
     gtest_main
 )
+target_link_libraries(syncRaftCfgTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncRespMgrTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncSnapshotTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncApplyMsgTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()

source/libs/sync/test/syncAppendEntriesReplyTest.cpp

@@ -15,7 +15,7 @@ void logTest() {
 }
 
 SyncAppendEntriesReply *createMsg() {
-  SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(1000);
   pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
@@ -27,7 +27,7 @@ SyncAppendEntriesReply *createMsg() {
 
 void test1() {
   SyncAppendEntriesReply *pMsg = createMsg();
-  syncAppendEntriesReplyPrint2((char *)"test1:", pMsg);
+  syncAppendEntriesReplyLog2((char *)"test1:", pMsg);
   syncAppendEntriesReplyDestroy(pMsg);
 }
 
@@ -36,10 +36,10 @@ void test2() {
   uint32_t                len = pMsg->bytes;
   char *                  serialized = (char *)taosMemoryMalloc(len);
   syncAppendEntriesReplySerialize(pMsg, serialized, len);
-  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000);
   syncAppendEntriesReplyDeserialize(serialized, len, pMsg2);
-  syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ",
+  syncAppendEntriesReplyLog2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ",
-                               pMsg2);
+                             pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesReplyDestroy(pMsg);
@@ -51,8 +51,8 @@ void test3() {
   uint32_t                len;
   char *                  serialized = syncAppendEntriesReplySerialize2(pMsg, &len);
   SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyDeserialize2(serialized, len);
-  syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ",
+  syncAppendEntriesReplyLog2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ",
-                               pMsg2);
+                             pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesReplyDestroy(pMsg);
@@ -63,11 +63,12 @@ void test4() {
   SyncAppendEntriesReply *pMsg = createMsg();
   SRpcMsg                 rpcMsg;
   syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
-  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000);
   syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2);
-  syncAppendEntriesReplyPrint2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ",
+  syncAppendEntriesReplyLog2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ",
-                               pMsg2);
+                             pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesReplyDestroy(pMsg);
   syncAppendEntriesReplyDestroy(pMsg2);
 }
@@ -77,17 +78,17 @@ void test5() {
   SRpcMsg                 rpcMsg;
   syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyFromRpcMsg2(&rpcMsg);
-  syncAppendEntriesReplyPrint2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ",
+  syncAppendEntriesReplyLog2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ",
-                               pMsg2);
+                             pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesReplyDestroy(pMsg);
   syncAppendEntriesReplyDestroy(pMsg2);
 }
 
 int main() {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();
 
   test1();

source/libs/sync/test/syncAppendEntriesTest.cpp

@@ -15,7 +15,7 @@ void logTest() {
 }
 
 SyncAppendEntries *createMsg() {
-  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20);
+  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000);
   pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
@@ -29,7 +29,7 @@ SyncAppendEntries *createMsg() {
 
 void test1() {
   SyncAppendEntries *pMsg = createMsg();
-  syncAppendEntriesPrint2((char *)"test1:", pMsg);
+  syncAppendEntriesLog2((char *)"test1:", pMsg);
   syncAppendEntriesDestroy(pMsg);
 }
 
@@ -38,9 +38,9 @@ void test2() {
   uint32_t           len = pMsg->bytes;
   char *             serialized = (char *)taosMemoryMalloc(len);
   syncAppendEntriesSerialize(pMsg, serialized, len);
-  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen);
+  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen, 1000);
   syncAppendEntriesDeserialize(serialized, len, pMsg2);
-  syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
+  syncAppendEntriesLog2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesDestroy(pMsg);
@@ -52,7 +52,7 @@ void test3() {
   uint32_t           len;
   char *             serialized = syncAppendEntriesSerialize2(pMsg, &len);
   SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len);
-  syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
+  syncAppendEntriesLog2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesDestroy(pMsg);
@@ -65,8 +65,9 @@ void test4() {
   syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen);
   syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2);
-  syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
+  syncAppendEntriesLog2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesDestroy(pMsg);
   syncAppendEntriesDestroy(pMsg2);
 }
@@ -76,16 +77,16 @@ void test5() {
   SRpcMsg            rpcMsg;
   syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntries *pMsg2 = syncAppendEntriesFromRpcMsg2(&rpcMsg);
-  syncAppendEntriesPrint2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
+  syncAppendEntriesLog2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesDestroy(pMsg);
   syncAppendEntriesDestroy(pMsg2);
 }
 
 int main() {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();
 
   test1();

source/libs/sync/test/syncApplyMsgTest.cpp

@@ -0,0 +1,123 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncApplyMsg *createMsg() {
+  SRpcMsg rpcMsg;
+  memset(&rpcMsg, 0, sizeof(rpcMsg));
+  rpcMsg.msgType = 12345;
+  rpcMsg.contLen = 20;
+  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  strcpy((char *)rpcMsg.pCont, "hello rpc");
+
+  SFsmCbMeta meta;
+  meta.code = 11;
+  meta.index = 22;
+  meta.isWeak = 1;
+  meta.seqNum = 33;
+  meta.state = TAOS_SYNC_STATE_LEADER;
+
+  SyncApplyMsg *pMsg = syncApplyMsgBuild2(&rpcMsg, 123, &meta);
+  rpcFreeCont(rpcMsg.pCont);
+  return pMsg;
+}
+
+void test1() {
+  SyncApplyMsg *pMsg = createMsg();
+  syncApplyMsgLog2((char *)"test1:", pMsg);
+  syncApplyMsgDestroy(pMsg);
+}
+
+void test2() {
+  SyncApplyMsg *pMsg = createMsg();
+  uint32_t      len = pMsg->bytes;
+  char *        serialized = (char *)taosMemoryMalloc(len);
+  syncApplyMsgSerialize(pMsg, serialized, len);
+  SyncApplyMsg *pMsg2 = syncApplyMsgBuild(pMsg->dataLen);
+  syncApplyMsgDeserialize(serialized, len, pMsg2);
+  syncApplyMsgLog2((char *)"test2: syncApplyMsgSerialize -> syncApplyMsgDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test3() {
+  SyncApplyMsg *pMsg = createMsg();
+  uint32_t      len;
+  char *        serialized = syncApplyMsgSerialize2(pMsg, &len);
+  SyncApplyMsg *pMsg2 = syncApplyMsgDeserialize2(serialized, len);
+  syncApplyMsgLog2((char *)"test3: syncApplyMsgSerialize2 -> syncApplyMsgDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test4() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = (SyncApplyMsg *)taosMemoryMalloc(rpcMsg.contLen);
+  syncApplyMsgFromRpcMsg(&rpcMsg, pMsg2);
+  syncApplyMsgLog2((char *)"test4: syncApplyMsg2RpcMsg -> syncApplyMsgFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test5() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg);
+  syncApplyMsgLog2((char *)"test5: syncClientRequest2RpcMsg -> syncApplyMsgFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test6() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg);
+
+  SRpcMsg originalRpcMsg;
+  syncApplyMsg2OriginalRpcMsg(pMsg2, &originalRpcMsg);
+  syncRpcMsgLog2((char *)"test6", &originalRpcMsg);
+
+  rpcFreeCont(originalRpcMsg.pCont);
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+  test6();
+
+  return 0;
+}

source/libs/sync/test/syncClientRequestTest.cpp

@@ -21,13 +21,14 @@ SyncClientRequest *createMsg() {
   rpcMsg.contLen = 20;
   rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
   strcpy((char *)rpcMsg.pCont, "hello rpc");
-  SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true);
+  SyncClientRequest *pMsg = syncClientRequestBuild2(&rpcMsg, 123, true, 1000);
+  rpcFreeCont(rpcMsg.pCont);
   return pMsg;
 }
 
 void test1() {
   SyncClientRequest *pMsg = createMsg();
-  syncClientRequestPrint2((char *)"test1:", pMsg);
+  syncClientRequestLog2((char *)"test1:", pMsg);
   syncClientRequestDestroy(pMsg);
 }
 
@@ -38,7 +39,7 @@ void test2() {
   syncClientRequestSerialize(pMsg, serialized, len);
   SyncClientRequest *pMsg2 = syncClientRequestBuild(pMsg->dataLen);
   syncClientRequestDeserialize(serialized, len, pMsg2);
-  syncClientRequestPrint2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2);
+  syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2);
 
   taosMemoryFree(serialized);
   syncClientRequestDestroy(pMsg);
@@ -50,7 +51,7 @@ void test3() {
   uint32_t           len;
   char *             serialized = syncClientRequestSerialize2(pMsg, &len);
   SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len);
-  syncClientRequestPrint2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2);
+  syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2);
 
   taosMemoryFree(serialized);
   syncClientRequestDestroy(pMsg);
@@ -63,8 +64,9 @@ void test4() {
   syncClientRequest2RpcMsg(pMsg, &rpcMsg);
   SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen);
   syncClientRequestFromRpcMsg(&rpcMsg, pMsg2);
-  syncClientRequestPrint2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2);
+  syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncClientRequestDestroy(pMsg);
   syncClientRequestDestroy(pMsg2);
 }
@@ -74,16 +76,16 @@ void test5() {
   SRpcMsg            rpcMsg;
   syncClientRequest2RpcMsg(pMsg, &rpcMsg);
   SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg);
-  syncClientRequestPrint2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2);
+  syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncClientRequestDestroy(pMsg);
   syncClientRequestDestroy(pMsg2);
 }
 
 int main() {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();
 
   test1();

source/libs/sync/test/syncElectTest.cpp

@@ -3,9 +3,8 @@
 #include "syncEnv.h"
 #include "syncIO.h"
 #include "syncInt.h"
-#include "syncRaftLog.h"
-#include "syncRaftStore.h"
 #include "syncUtil.h"
+#include "wal.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -16,42 +15,41 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+uint16_t    gPorts[] = {7010, 7110, 7210, 7310, 7410};
-int32_t  replicaNum = 3;
+const char* gDir = "./syncElectTest";
-int32_t  myIndex = 0;
+int32_t     gVgId = 1234;
-
-SRaftId    ids[TSDB_MAX_REPLICA];
-SSyncInfo  syncInfo;
-SSyncFSM*  pFsm;
-SWal*      pWal;
-SSyncNode* gSyncNode;
-
-SSyncNode* syncNodeInit() {
-  syncInfo.vgId = 1234;
-  syncInfo.rpcClient = gSyncIO->clientRpc;
-  syncInfo.FpSendMsg = syncIOSendMsg;
-  syncInfo.queue = gSyncIO->pMsgQ;
-  syncInfo.FpEqMsg = syncIOEqMsg;
-  syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex);
 
+void init() {
   int code = walInit();
   assert(code == 0);
+}
+
+void cleanup() { walCleanUp(); }
+
+SWal* createWal(char* path, int32_t vgId) {
   SWalCfg walCfg;
   memset(&walCfg, 0, sizeof(SWalCfg));
-  walCfg.vgId = syncInfo.vgId;
+  walCfg.vgId = vgId;
   walCfg.fsyncPeriod = 1000;
   walCfg.retentionPeriod = 1000;
   walCfg.rollPeriod = 1000;
   walCfg.retentionSize = 1000;
   walCfg.segSize = 1000;
   walCfg.level = TAOS_WAL_FSYNC;
-
+  SWal* pWal = walOpen(path, &walCfg);
-  char tmpdir[128];
-  snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex);
-  pWal = walOpen(tmpdir, &walCfg);
   assert(pWal != NULL);
+  return pWal;
+}
 
+SSyncNode* createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = vgId;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = NULL;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex);
   syncInfo.pWal = pWal;
 
   SSyncCfg* pCfg = &syncInfo.syncCfg;
@@ -59,9 +57,9 @@ SSyncNode* syncNodeInit() {
   pCfg->replicaNum = replicaNum;
 
   for (int i = 0; i < replicaNum; ++i) {
-    pCfg->nodeInfo[i].nodePort = ports[i];
+    pCfg->nodeInfo[i].nodePort = gPorts[i];
-    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn);
-    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+    // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
   }
 
   SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
@@ -78,50 +76,51 @@ SSyncNode* syncNodeInit() {
   gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
   gSyncIO->pSyncNode = pSyncNode;
 
+  syncNodeStart(pSyncNode);
+
   return pSyncNode;
 }
 
-SSyncNode* syncInitTest() { return syncNodeInit(); }
+void usage(char* exe) { printf("usage: %s replicaNum myIndex \n", exe); }
-
-void initRaftId(SSyncNode* pSyncNode) {
-  for (int i = 0; i < replicaNum; ++i) {
-    ids[i] = pSyncNode->replicasId[i];
-    char* s = syncUtilRaftId2Str(&ids[i]);
-    printf("raftId[%d] : %s\n", i, s);
-    taosMemoryFree(s);
-  }
-}
 
 int main(int argc, char** argv) {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
-
+  if (argc != 3) {
-  myIndex = 0;
+    usage(argv[0]);
-  if (argc >= 2) {
+    exit(-1);
-    myIndex = atoi(argv[1]);
   }
+  int32_t replicaNum = atoi(argv[1]);
+  int32_t myIndex = atoi(argv[2]);
 
-  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  assert(replicaNum >= 1 && replicaNum <= 5);
+  assert(myIndex >= 0 && myIndex < replicaNum);
+
+  init();
+  int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]);
   assert(ret == 0);
-
   ret = syncEnvStart();
   assert(ret == 0);
 
-  gSyncNode = syncInitTest();
+  char walPath[128];
-  assert(gSyncNode != NULL);
+  snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex);
-  syncNodePrint2((char*)"", gSyncNode);
+  SWal* pWal = createWal(walPath, gVgId);
 
-  initRaftId(gSyncNode);
+  SSyncNode* pSyncNode = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir);
+  assert(pSyncNode != NULL);
+  syncNodeLog2((char*)"==syncElectTest==", pSyncNode);
 
   //---------------------------
   while (1) {
-    sTrace(
+    char* s = syncNode2SimpleStr(pSyncNode);
-        "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
+    sTrace("%s", s);
-        gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm,
+    taosMemoryFree(s);
-        gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS);
     taosMsleep(1000);
   }
 
+  syncNodeClose(pSyncNode);
+  walClose(pWal);
+  syncIOStop();
+  cleanup();
   return 0;
 }

source/libs/sync/test/syncElectTest2.cpp

@@ -1,135 +0,0 @@
-#include <gtest/gtest.h>
-#include <stdio.h>
-#include "syncEnv.h"
-#include "syncIO.h"
-#include "syncInt.h"
-#include "syncRaftLog.h"
-#include "syncRaftStore.h"
-#include "syncUtil.h"
-#include "tref.h"
-
-void logTest() {
-  sTrace("--- sync log test: trace");
-  sDebug("--- sync log test: debug");
-  sInfo("--- sync log test: info");
-  sWarn("--- sync log test: warn");
-  sError("--- sync log test: error");
-  sFatal("--- sync log test: fatal");
-}
-
-uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
-int32_t  replicaNum = 3;
-int32_t  myIndex = 0;
-
-SRaftId   ids[TSDB_MAX_REPLICA];
-SSyncInfo syncInfo;
-SSyncFSM* pFsm;
-SWal*     pWal;
-
-int64_t syncNodeInit() {
-  syncInfo.vgId = 1234;
-  syncInfo.rpcClient = gSyncIO->clientRpc;
-  syncInfo.FpSendMsg = syncIOSendMsg;
-  syncInfo.queue = gSyncIO->pMsgQ;
-  syncInfo.FpEqMsg = syncIOEqMsg;
-  syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect2_test_%d", myIndex);
-
-  int code = walInit();
-  assert(code == 0);
-  SWalCfg walCfg;
-  memset(&walCfg, 0, sizeof(SWalCfg));
-  walCfg.vgId = syncInfo.vgId;
-  walCfg.fsyncPeriod = 1000;
-  walCfg.retentionPeriod = 1000;
-  walCfg.rollPeriod = 1000;
-  walCfg.retentionSize = 1000;
-  walCfg.segSize = 1000;
-  walCfg.level = TAOS_WAL_FSYNC;
-
-  char tmpdir[128];
-  snprintf(tmpdir, sizeof(tmpdir), "./elect2_test_wal_%d", myIndex);
-  pWal = walOpen(tmpdir, &walCfg);
-  assert(pWal != NULL);
-
-  syncInfo.pWal = pWal;
-
-  SSyncCfg* pCfg = &syncInfo.syncCfg;
-  pCfg->myIndex = myIndex;
-  pCfg->replicaNum = replicaNum;
-
-  for (int i = 0; i < replicaNum; ++i) {
-    pCfg->nodeInfo[i].nodePort = ports[i];
-    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
-    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
-  }
-
-  int64_t rid = syncStart(&syncInfo);
-  assert(rid > 0);
-
-  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
-  assert(pSyncNode != NULL);
-
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
-  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
-  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
-  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
-  gSyncIO->pSyncNode = pSyncNode;
-
-  syncNodeRelease(pSyncNode);
-
-  return rid;
-}
-
-void initRaftId(SSyncNode* pSyncNode) {
-  for (int i = 0; i < replicaNum; ++i) {
-    ids[i] = pSyncNode->replicasId[i];
-    char* s = syncUtilRaftId2Str(&ids[i]);
-    printf("raftId[%d] : %s\n", i, s);
-    taosMemoryFree(s);
-  }
-}
-
-int main(int argc, char** argv) {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
-  tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
-
-  myIndex = 0;
-  if (argc >= 2) {
-    myIndex = atoi(argv[1]);
-  }
-
-  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
-  assert(ret == 0);
-
-  ret = syncInit();
-  assert(ret == 0);
-
-  int64_t rid = syncNodeInit();
-  assert(rid > 0);
-
-  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
-  assert(pSyncNode != NULL);
-
-  syncNodePrint2((char*)"", pSyncNode);
-  initRaftId(pSyncNode);
-
-  //---------------------------
-  while (1) {
-    sTrace(
-        "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
-        pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm,
-        pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS);
-    taosMsleep(1000);
-  }
-
-  syncNodeRelease(pSyncNode);
-
-  return 0;
-}

source/libs/sync/test/syncElectTest3.cpp

@@ -1,138 +0,0 @@
-#include <gtest/gtest.h>
-#include <stdio.h>
-#include "syncEnv.h"
-#include "syncIO.h"
-#include "syncInt.h"
-#include "syncRaftLog.h"
-#include "syncRaftStore.h"
-#include "syncUtil.h"
-#include "tref.h"
-
-void logTest() {
-  sTrace("--- sync log test: trace");
-  sDebug("--- sync log test: debug");
-  sInfo("--- sync log test: info");
-  sWarn("--- sync log test: warn");
-  sError("--- sync log test: error");
-  sFatal("--- sync log test: fatal");
-}
-
-uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
-int32_t  replicaNum = 3;
-int32_t  myIndex = 0;
-
-SRaftId   ids[TSDB_MAX_REPLICA];
-SSyncInfo syncInfo;
-SSyncFSM* pFsm;
-SWal*     pWal;
-
-int64_t syncNodeInit() {
-  syncInfo.vgId = 1234;
-  syncInfo.rpcClient = gSyncIO->clientRpc;
-  syncInfo.FpSendMsg = syncIOSendMsg;
-  syncInfo.queue = gSyncIO->pMsgQ;
-  syncInfo.FpEqMsg = syncIOEqMsg;
-  syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect3_test_%d", myIndex);
-
-  int code = walInit();
-  assert(code == 0);
-  SWalCfg walCfg;
-  memset(&walCfg, 0, sizeof(SWalCfg));
-  walCfg.vgId = syncInfo.vgId;
-  walCfg.fsyncPeriod = 1000;
-  walCfg.retentionPeriod = 1000;
-  walCfg.rollPeriod = 1000;
-  walCfg.retentionSize = 1000;
-  walCfg.segSize = 1000;
-  walCfg.level = TAOS_WAL_FSYNC;
-
-  char tmpdir[128];
-  snprintf(tmpdir, sizeof(tmpdir), "./elect3_test_wal_%d", myIndex);
-  pWal = walOpen(tmpdir, &walCfg);
-  assert(pWal != NULL);
-
-  syncInfo.pWal = pWal;
-
-  SSyncCfg* pCfg = &syncInfo.syncCfg;
-  pCfg->myIndex = myIndex;
-  pCfg->replicaNum = replicaNum;
-
-  for (int i = 0; i < replicaNum; ++i) {
-    pCfg->nodeInfo[i].nodePort = ports[i];
-    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
-    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
-  }
-
-  int64_t rid = syncStart(&syncInfo);
-  assert(rid > 0);
-
-  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
-  assert(pSyncNode != NULL);
-
-  pSyncNode->hbBaseLine = 500;
-  pSyncNode->electBaseLine = 1500;
-
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
-  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
-  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
-  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
-  gSyncIO->pSyncNode = pSyncNode;
-
-  syncNodeRelease(pSyncNode);
-
-  return rid;
-}
-
-void initRaftId(SSyncNode* pSyncNode) {
-  for (int i = 0; i < replicaNum; ++i) {
-    ids[i] = pSyncNode->replicasId[i];
-    char* s = syncUtilRaftId2Str(&ids[i]);
-    printf("raftId[%d] : %s\n", i, s);
-    taosMemoryFree(s);
-  }
-}
-
-int main(int argc, char** argv) {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
-  tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
-
-  myIndex = 0;
-  if (argc >= 2) {
-    myIndex = atoi(argv[1]);
-  }
-
-  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
-  assert(ret == 0);
-
-  ret = syncInit();
-  assert(ret == 0);
-
-  int64_t rid = syncNodeInit();
-  assert(rid > 0);
-
-  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
-  assert(pSyncNode != NULL);
-
-  syncNodePrint2((char*)"", pSyncNode);
-  initRaftId(pSyncNode);
-
-  //---------------------------
-  while (1) {
-    sTrace(
-        "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d",
-        pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm,
-        pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS);
-    taosMsleep(1000);
-  }
-
-  syncNodeRelease(pSyncNode);
-
-  return 0;
-}

source/libs/sync/test/syncEncodeTest.cpp

@@ -8,6 +8,7 @@
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
+#include "wal.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -102,7 +103,7 @@ SRpcMsg *step0() {
 }
 
 SyncClientRequest *step1(const SRpcMsg *pMsg) {
-  SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true);
+  SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true, 1000);
   return pRetMsg;
 }
 
@@ -159,23 +160,23 @@ int main(int argc, char **argv) {
 
   // step0
   SRpcMsg *pMsg0 = step0();
-  syncRpcMsgPrint2((char *)"==step0==", pMsg0);
+  syncRpcMsgLog2((char *)"==step0==", pMsg0);
 
   // step1
   SyncClientRequest *pMsg1 = step1(pMsg0);
-  syncClientRequestPrint2((char *)"==step1==", pMsg1);
+  syncClientRequestLog2((char *)"==step1==", pMsg1);
 
   // step2
   SRpcMsg *pMsg2 = step2(pMsg1);
-  syncRpcMsgPrint2((char *)"==step2==", pMsg2);
+  syncRpcMsgLog2((char *)"==step2==", pMsg2);
 
   // step3
   SyncClientRequest *pMsg3 = step3(pMsg2);
-  syncClientRequestPrint2((char *)"==step3==", pMsg3);
+  syncClientRequestLog2((char *)"==step3==", pMsg3);
 
   // step4
   SSyncRaftEntry *pMsg4 = step4(pMsg3);
-  syncEntryPrint2((char *)"==step4==", pMsg4);
+  syncEntryLog2((char *)"==step4==", pMsg4);
 
   // log, relog
   SSyncNode *pSyncNode = syncNodeInit();
@@ -183,7 +184,7 @@ int main(int argc, char **argv) {
   SSyncRaftEntry *pEntry = pMsg4;
   pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry);
   SSyncRaftEntry *pEntry2 = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, pEntry->index);
-  syncEntryPrint2((char *)"==pEntry2==", pEntry2);
+  syncEntryLog2((char *)"==pEntry2==", pEntry2);
 
   // step5
   uint32_t len;
@@ -194,11 +195,11 @@ int main(int argc, char **argv) {
 
   // step6
   SSyncRaftEntry *pMsg6 = step6(pMsg5, len);
-  syncEntryPrint2((char *)"==step6==", pMsg6);
+  syncEntryLog2((char *)"==step6==", pMsg6);
 
   // step7
   SRpcMsg *pMsg7 = step7(pMsg6);
-  syncRpcMsgPrint2((char *)"==step7==", pMsg7);
+  syncRpcMsgLog2((char *)"==step7==", pMsg7);
 
   return 0;
 }

source/libs/sync/test/syncEnqTest.cpp

@@ -89,14 +89,14 @@ int main(int argc, char** argv) {
   SSyncNode* pSyncNode = syncInitTest();
   assert(pSyncNode != NULL);
 
-  syncNodePrint2((char*)"syncInitTest", pSyncNode);
+  syncNodeLog2((char*)"syncInitTest", pSyncNode);
 
   initRaftId(pSyncNode);
 
   //--------------------------------------------------------------
 
   for (int i = 0; i < 10; ++i) {
-    SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncEnqTest");
+    SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, 1000, "syncEnqTest");
     SRpcMsg        rpcMsg;
     syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);

source/libs/sync/test/syncEntryTest.cpp

@@ -52,7 +52,7 @@ void test3() {
   pSyncMsg->isWeak = 1;
   strcpy(pSyncMsg->data, "test3");
 
-  SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NOOP);
+  SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200);
   syncEntryPrint(pEntry);
 
   syncClientRequestDestroy(pSyncMsg);
@@ -68,7 +68,6 @@ void test4() {
   pEntry->isWeak = true;
   pEntry->term = 44;
   pEntry->index = 55;
-  pEntry->entryType = SYNC_RAFT_ENTRY_CONFIG;
   strcpy(pEntry->data, "test4");
   syncEntryPrint(pEntry);
 
@@ -84,9 +83,8 @@ void test4() {
 }
 
 int main(int argc, char** argv) {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
 
   test1();
   test2();