Merge branch '3.0' into test/abs_tag_compute

docs-cn/20-third-party/01-grafana.mdx

@@ -3,6 +3,9 @@ sidebar_label: Grafana
 title: Grafana
 ---
 
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
 TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/) 快速集成搭建数据监测报警系统，整个过程无需任何代码开发，TDengine 中数据表的内容可以在仪表盘(DashBoard)上进行可视化展现。关于 TDengine 插件的使用您可以在[GitHub](https://github.com/taosdata/grafanaplugin/blob/master/README.md)中了解更多。
 
 ## 前置条件
@@ -23,9 +26,12 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/
 
 ## 配置 Grafana
 
-### 安装方式一：安装脚本
+### 安装 Grafana Plugin 并配置数据源
 
-将集群信息设置为环境变量（也可以使用 [`.env`(dotenv) 文件](https://hexdocs.pm/dotenvy/dotenv-file-format.html) ）：
+<Tabs defaultValue="script">
+<TabItem value="script" label="使用安装脚本">
+
+将集群信息设置为环境变量；也可以使用 `.env` 文件，请参考 [dotenv](https://hexdocs.pm/dotenvy/dotenv-file-format.html)：
 
 ```sh
 export TDENGINE_API=http://tdengine.local:6041
@@ -42,7 +48,8 @@ bash -c "$(curl -fsSL https://raw.githubusercontent.com/taosdata/grafanaplugin/m
 
 该脚本将自动安装 Grafana 插件并配置数据源。安装完毕后，需要重启 Grafana 服务后生效。
 
-### 安装方式二：手动安装 TDengine 数据源插件
+</TabItem>
+<TabItem value="manual" label="手动安装并配置">
 
 使用 [`grafana-cli` 命令行工具](https://grafana.com/docs/grafana/latest/administration/cli/) 进行插件[安装](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation)。
 
@@ -94,6 +101,9 @@ GF_INSTALL_PLUGINS=tdengine-datasource
 
 ![TDengine Database Grafana plugin add data source](./add_datasource4.webp)
 
+</TabItem>
+</Tabs>
+
 ### 创建 Dashboard
 
 回到主界面创建 Dashboard，点击 Add Query 进入面板查询页面：

docs-en/20-third-party/01-grafana.mdx

@@ -3,6 +3,9 @@ sidebar_label: Grafana
 title: Grafana
 ---
 
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+
 TDengine can be quickly integrated with the open-source data visualization system [Grafana](https://www.grafana.com/) to build a data monitoring and alerting system. The whole process does not require any code development. And you can visualize the contents of the data tables in TDengine on a dashboard.
 
 You can learn more about using the TDengine plugin on [GitHub](https://github.com/taosdata/grafanaplugin/blob/master/README.md).
@@ -25,7 +28,10 @@ TDengine currently supports Grafana versions 7.5 and above. Users can go to the
 
 ## Configuring Grafana
 
-### Option 1: Install with `install.sh`
+### Install Grafana Plugin and Configure Data Source
+
+<Tabs defaultValue="script">
+<TabItem value="script" label="Using Script">
 
 Set the url and authorization environment variables by `export` or a [`.env`(dotenv) file](https://hexdocs.pm/dotenvy/dotenv-file-format.html):
 
@@ -46,7 +52,8 @@ With this script, TDengine data source plugin and the Grafana data source will b
 
 And then, restart Grafana service and open Grafana in web-browser, usually <http://localhost:3000>.
 
-### Option 2: Install Plugin Manually
+</TabItem>
+<TabItem value="manual" label="Install & Configure Manually">
 
 Follow the installation steps in [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) with the [``grafana-cli`` command-line tool](https://grafana.com/docs/grafana/latest/administration/cli/) for plugin installation.
 
@@ -98,6 +105,9 @@ Click `Save & Test` to test. You should see a success message if the test worked
 
 ![TDengine Database TDinsight plugin add database 4](./grafana/add_datasource4.webp)
 
+</TabItem>
+</Tabs>
+
 ### Create Dashboard
 
 Go back to the main interface to create a dashboard and click Add Query to enter the panel query page:

include/common/tmsg.h

@@ -1205,6 +1205,7 @@ typedef struct {
   int8_t  completed;  // all results are returned to client
   int8_t  precision;
   int8_t  compressed;
+  int8_t  streamBlockType;
   int32_t compLen;
   int32_t numOfRows;
   int32_t numOfCols;
@@ -2510,7 +2511,7 @@ typedef struct {
 
 int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp);
 int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp);
-
+void tFreeSTableIndexInfo(void *pInfo);
 
 typedef struct {
   int8_t  mqMsgType;
@@ -2752,8 +2753,8 @@ typedef struct {
   char*    msg;
 } SVDeleteReq;
 
-int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq);
+int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
-int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq);
+int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
 
 typedef struct {
   int64_t affectedRows;

include/common/tmsgdef.h

@@ -194,6 +194,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_QND_MSG)
@@ -233,6 +234,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_COMMON_RESPONSE, "sync-common-response", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPLY_MSG, "sync-apply-msg", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_CONFIG_CHANGE, "sync-config-change", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_SEND, "sync-snapshot-send", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_RSP, "sync-snapshot-rsp", NULL, NULL)
   
 #if defined(TD_MSG_NUMBER_)
   TDMT_MAX

include/libs/catalog/catalog.h

@@ -67,6 +67,7 @@ typedef struct SCatalogReq {
   SArray *pUdf;           // element is udf name
   SArray *pIndex;         // element is index name
   SArray *pUser;          // element is SUserAuthInfo
+  SArray *pTableIndex;    // element is SNAME
   bool    qNodeRequired;  // valid qnode
   bool    forceUpdate;    
 } SCatalogReq;
@@ -82,6 +83,7 @@ typedef struct SMetaData {
   SArray    *pDbInfo;     // pRes = SDbInfo*
   SArray    *pTableMeta;  // pRes = STableMeta*
   SArray    *pTableHash;  // pRes = SVgroupInfo*
+  SArray    *pTableIndex; // pRes = SArray<STableIndexInfo>*
   SArray    *pUdfList;    // pRes = SFuncInfo*
   SArray    *pIndex;      // pRes = SIndexInfo*
   SArray    *pUser;       // pRes = bool*
@@ -277,7 +279,7 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
 int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
 
-int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes);
+int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes);
 
 int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo* pInfo);
 

include/libs/function/function.h

@@ -146,6 +146,7 @@ typedef struct SqlFunctionCtx {
   struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   int32_t                curBufPage;
+  bool                   increase;
 
   char                   udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;

include/libs/function/functionMgt.h

@@ -129,6 +129,10 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_SPREAD_MERGE,
   FUNCTION_TYPE_HISTOGRAM_PARTIAL,
   FUNCTION_TYPE_HISTOGRAM_MERGE,
+  FUNCTION_TYPE_HYPERLOGLOG_PARTIAL,
+  FUNCTION_TYPE_HYPERLOGLOG_MERGE,
+  FUNCTION_TYPE_ELAPSED_PARTIAL,
+  FUNCTION_TYPE_ELAPSED_MERGE,
 
   // user defined funcion
   FUNCTION_TYPE_UDF = 10000

include/libs/nodes/plannodes.h

@@ -304,6 +304,7 @@ typedef struct SDownstreamSourceNode {
 typedef struct SExchangePhysiNode {
   SPhysiNode node;
   int32_t    srcGroupId;     // group id of datasource suplans
+  bool       singleChannel;
   SNodeList* pSrcEndPoints;  // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
 } SExchangePhysiNode;
 

include/libs/stream/tstream.h

@@ -308,9 +308,11 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc
   if (pTask->sinkType == TASK_SINK__TABLE) {
     ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
     pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
+    taosFreeQitem(pBlock);
   } else if (pTask->sinkType == TASK_SINK__SMA) {
     ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
     pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
+    taosFreeQitem(pBlock);
   } else {
     ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);
     taosWriteQitem(pTask->outputQueue->queue, pBlock);

include/libs/sync/sync.h

@@ -88,11 +88,16 @@ typedef struct SReConfigCbMeta {
 } SReConfigCbMeta;
 
 typedef struct SSnapshot {
-  void *data;
+  void*     data;
   SyncIndex lastApplyIndex;
   SyncTerm  lastApplyTerm;
+  SyncIndex lastConfigIndex;
 } SSnapshot;
 
+typedef struct SSnapshotMeta {
+  SyncIndex lastConfigIndex;
+} SSnapshotMeta;
+
 typedef struct SSyncFSM {
   void* data;
 
@@ -141,10 +146,28 @@ typedef struct SSyncLogStore {
   // return commit index of log
   SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
 
+  // refactor, log[0 .. n] ==> log[m .. n]
+  int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
+  int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore);
+  bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore);
+  int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore);
+  bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index);
+
+  SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
+  SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
+
+  int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
+  int32_t (*syncLogGetEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry);
+  int32_t (*syncLogTruncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
+
 } SSyncLogStore;
 
 typedef struct SSyncInfo {
   bool        isStandBy;
+  bool        snapshotEnable;
   SyncGroupId vgId;
   SSyncCfg    syncCfg;
   char        path[TSDB_FILENAME_LEN];
@@ -172,6 +195,8 @@ bool        syncEnvIsStart();
 const char* syncStr(ESyncState state);
 bool        syncIsRestoreFinish(int64_t rid);
 
+int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta);
+
 // to be moved to static
 void syncStartNormal(int64_t rid);
 void syncStartStandBy(int64_t rid);

include/libs/sync/syncTools.h

@@ -301,6 +301,7 @@ typedef struct SyncAppendEntries {
   SyncIndex prevLogIndex;
   SyncTerm  prevLogTerm;
   SyncIndex commitIndex;
+  SyncTerm  privateTerm;
   uint32_t  dataLen;
   char      data[];
 } SyncAppendEntries;
@@ -332,6 +333,7 @@ typedef struct SyncAppendEntriesReply {
   SRaftId  destId;
   // private data
   SyncTerm  term;
+  SyncTerm  privateTerm;
   bool      success;
   SyncIndex matchIndex;
 } SyncAppendEntriesReply;
@@ -385,6 +387,75 @@ void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
 void syncApplyMsgLog(const SyncApplyMsg* pMsg);
 void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncSnapshotSend {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  SyncTerm  term;
+  SyncIndex lastIndex;  // lastIndex of snapshot
+  SyncTerm  lastTerm;   // lastTerm of snapshot
+  SyncTerm  privateTerm;
+  int32_t   seq;
+  uint32_t  dataLen;
+  char      data[];
+} SyncSnapshotSend;
+
+SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId);
+void              syncSnapshotSendDestroy(SyncSnapshotSend* pMsg);
+void              syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen);
+void              syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg);
+char*             syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len);
+SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len);
+void              syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg);
+void              syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg);
+SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*            syncSnapshotSend2Json(const SyncSnapshotSend* pMsg);
+char*             syncSnapshotSend2Str(const SyncSnapshotSend* pMsg);
+
+// for debug ----------------------
+void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg);
+void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg);
+void syncSnapshotSendLog(const SyncSnapshotSend* pMsg);
+void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncSnapshotRsp {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  SyncTerm  term;
+  SyncIndex lastIndex;
+  SyncTerm  lastTerm;
+  SyncTerm  privateTerm;
+  int32_t   ack;
+  int32_t   code;
+} SyncSnapshotRsp;
+
+SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId);
+void             syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg);
+void             syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen);
+void             syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg);
+char*            syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len);
+SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len);
+void             syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg);
+void             syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg);
+SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg);
+char*            syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg);
+
+// for debug ----------------------
+void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg);
+void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg);
+void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg);
+void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg);
+
 // on message ----------------------
 int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
 int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
@@ -395,6 +466,29 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg)
 int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
 int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
 
+int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+int32_t syncNodeOnSnapshotSendCb(SSyncNode* ths, SyncSnapshotSend* pMsg);
+int32_t syncNodeOnSnapshotRspCb(SSyncNode* ths, SyncSnapshotRsp* pMsg);
+
+// -----------------------------------------
+typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg);
+typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg);
+typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg);
+typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg);
+typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg);
+typedef int32_t (*FpOnAppendEntriesReplyCb)(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+typedef int32_t (*FpOnTimeoutCb)(SSyncNode* pSyncNode, SyncTimeout* pMsg);
+typedef int32_t (*FpOnSnapshotSendCb)(SSyncNode* ths, SyncSnapshotSend* pMsg);
+typedef int32_t (*FpOnSnapshotRspCb)(SSyncNode* ths, SyncSnapshotRsp* pMsg);
+
+// option ----------------------------------
+bool syncNodeSnapshotEnable(SSyncNode* pSyncNode);
+
 // ---------------------------------------------
 
 #ifdef __cplusplus

include/libs/wal/wal.h

@@ -141,6 +141,8 @@ typedef struct SWal {
   // ctl
   int64_t       refId;
   TdThreadMutex mutex;
+  // ref
+  SHashObj *pRefHash;  // ref -> SWalRef
   // path
   char path[WAL_PATH_LEN];
   // reusable write head
@@ -184,7 +186,7 @@ int32_t walRollback(SWal *, int64_t ver);
 // notify that previous logs can be pruned safely
 int32_t walBeginSnapshot(SWal *, int64_t ver);
 int32_t walEndSnapshot(SWal *);
-void    walRestoreFromSnapshot(SWal *, int64_t ver);
+int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
 // int32_t  walDataCorrupted(SWal*);
 
 // read
@@ -199,6 +201,16 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
 int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
 int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
 
+typedef struct {
+  int64_t refId;
+  int64_t ver;
+} SWalRef;
+
+SWalRef *walOpenRef(SWal *);
+void     walCloseRef(SWalRef *);
+int32_t  walRefVer(SWalRef *, int64_t ver);
+int32_t  walUnrefVer(SWal *);
+
 // deprecated
 #if 0
 int32_t walRead(SWal *, SWalHead **, int64_t ver);

source/client/src/clientSml.c

@@ -303,7 +303,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
           uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
         }
         taos_free_result(res2);
-        taosMsleep(500);
+        taosMsleep(10);
       }
       break;
     }
@@ -327,7 +327,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
           uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
         }
         taos_free_result(res2);
-        taosMsleep(500);
+        taosMsleep(10);
       }
       break;
     }
@@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
           uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
         }
         taos_free_result(res2);
-        taosMsleep(500);
+        taosMsleep(10);
       }
       break;
     }
@@ -373,7 +373,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
           uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
         }
         taos_free_result(res2);
-        taosMsleep(500);
+        taosMsleep(10);
       }
       break;
     }
@@ -424,7 +424,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) {
           uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
         }
         taos_free_result(res2);
-        taosMsleep(500);
+        taosMsleep(10);
       }
       break;
     }
@@ -461,18 +461,18 @@ static int32_t smlProcessSchemaAction(SSmlHandle* info, SSchema* schemaField, SH
 
 static int32_t smlModifyDBSchemas(SSmlHandle* info) {
   int32_t code = 0;
+  SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
+  SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
+  strcpy(pName.dbname, info->pRequest->pDb);
 
   SSmlSTableMeta** tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, NULL);
   while (tableMetaSml) {
     SSmlSTableMeta* sTableData = *tableMetaSml;
-
     STableMeta *pTableMeta = NULL;
-    SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
 
     size_t superTableLen = 0;
     void *superTable = taosHashGetKey(tableMetaSml, &superTableLen);
-    SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
+    memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
-    strcpy(pName.dbname, info->pRequest->pDb);
     memcpy(pName.tname, superTable, superTableLen);
 
     code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta);
@@ -487,7 +487,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
       code = smlApplySchemaAction(info, &schemaAction);
       if (code != TSDB_CODE_SUCCESS) {
         uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName);
-        return code;
+        goto end;
       }
       info->cost.numOfCreateSTables++;
     }else if (code == TSDB_CODE_SUCCESS) {
@@ -502,7 +502,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
       code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &schemaAction, true);
       if (code != TSDB_CODE_SUCCESS) {
         taosHashCleanup(hashTmp);
-        return code;
+        goto end;
       }
 
       taosHashClear(hashTmp);
@@ -512,29 +512,33 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
       code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false);
       taosHashCleanup(hashTmp);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto end;
       }
 
-      code = catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, -1);
+      code = catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, 1);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto end;
       }
     } else {
       uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code));
-      return code;
+      goto end;
     }
     if(pTableMeta) taosMemoryFree(pTableMeta);
 
     code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, (char*)superTable);
-      return code;
+      goto end;
     }
     sTableData->tableMeta = pTableMeta;
 
     tableMetaSml = (SSmlSTableMeta**)taosHashIterate(info->superTables, tableMetaSml);
   }
   return 0;
+
+end:
+  catalogRefreshTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, 1);
+  return code;
 }
 
 //=========================================================================
@@ -1544,7 +1548,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
   }
 
   size_t typeLen = strlen(type->valuestring);
-  if (typeLen == 1 && type->valuestring[0] == 's') {
+  if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) {
     //seconds
     timeDouble = timeDouble * 1e9;
     if(smlDoubleToInt64OverFlow(timeDouble)){
@@ -1552,9 +1556,10 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
     *tsVal = timeDouble;
-  } else if (typeLen == 2 && type->valuestring[1] == 's') {
+  } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) {
     switch (type->valuestring[0]) {
       case 'm':
+      case 'M':
         //milliseconds
         timeDouble = timeDouble * 1e6;
         if(smlDoubleToInt64OverFlow(timeDouble)){
@@ -1564,6 +1569,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
         *tsVal = timeDouble;
         break;
       case 'u':
+      case 'U':
         //microseconds
         timeDouble = timeDouble * 1e3;
         if(smlDoubleToInt64OverFlow(timeDouble)){
@@ -1573,6 +1579,7 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
         *tsVal = timeDouble;
         break;
       case 'n':
+      case 'N':
         //nanoseconds
         *tsVal = timeDouble;
         break;
@@ -2285,6 +2292,8 @@ static int32_t smlParseLine(SSmlHandle *info, char* lines[], int numLines){
 
 static int smlProcess(SSmlHandle *info, char* lines[], int numLines) {
   int32_t code = TSDB_CODE_SUCCESS;
+  int32_t retryNum = 0;
+
   info->cost.parseTime = taosGetTimestampUs();
 
   code = smlParseLine(info, lines, numLines);
@@ -2298,7 +2307,12 @@ static int smlProcess(SSmlHandle *info, char* lines[], int numLines) {
   info->cost.numOfCTables = taosHashGetSize(info->childTables);
 
   info->cost.schemaTime = taosGetTimestampUs();
-  code = smlModifyDBSchemas(info);
+
+  do{
+    code = smlModifyDBSchemas(info);
+    if (code == 0) break;
+  } while (retryNum++ < taosHashGetSize(info->superTables));
+
   if (code != 0) {
     uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code));
     goto cleanup;
@@ -2409,6 +2423,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
   info->pRequest->code = smlProcess(info, lines, numLines);
 
 end:
+  info->taos->schemalessType = 0;
   uDebug("result:%s", info->msgBuf.buf);
   smlDestroyInfo(info);
   return (TAOS_RES*)request;

source/client/test/smlTest.cpp

@@ -1272,14 +1272,40 @@ TEST(testCase, sml_params_Test) {
   };
   TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
   ASSERT_EQ(taos_errno(res), TSDB_CODE_PAR_DB_NOT_SPECIFIED);
-  taos_free_result(pRes);
+  taos_free_result(res);
 
   pRes = taos_query(taos, "use param");
-  taos_free_result(pRes);
+  taos_free_result(res);
 
   res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
   ASSERT_EQ(taos_errno(res), TSDB_CODE_SML_INVALID_DB_CONF);
+  taos_free_result(res);
+}
+
+TEST(testCase, sml_16384_Test) {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(taos, nullptr);
+
+  TAOS_RES* pRes = taos_query(taos, "create database if not exists d16384 schemaless 1");
   taos_free_result(pRes);
+
+  const char *sql[] = {
+      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=t,c1=127i8 1626006833639000000",
+  };
+
+  pRes = taos_query(taos, "use d16384");
+  taos_free_result(pRes);
+
+  TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, 0);
+  ASSERT_EQ(taos_errno(res), 0);
+  taos_free_result(res);
+
+  const char *sql1[] = {
+      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000",
+  };
+  TAOS_RES* res1 = taos_schemaless_insert(taos, (char**)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0);
+  ASSERT_EQ(taos_errno(res1), 0);
+  taos_free_result(res1);
 }
 
 TEST(testCase, sml_oom_Test) {
@@ -1303,3 +1329,29 @@ TEST(testCase, sml_oom_Test) {
   ASSERT_EQ(taos_errno(res), 0);
   taos_free_result(pRes);
 }
+
+TEST(testCase, sml_16368_Test) {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(taos, nullptr);
+
+  TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "use d16368");
+  taos_free_result(pRes);
+
+  const char *sql[] = {
+      "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, \"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": \"binary\"}}},\n"
+      "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n"
+      "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, \"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}},\n"
+      "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n"
+      "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": {\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}},\n"
+      "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": {\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": \"nchar\"}}},\n"
+      "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": {\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": {\"value\": \"ste2\", \"type\": \"nchar\"}}},\n"
+      "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n"
+      "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"
+  };
+  pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS);
+  ASSERT_EQ(taos_errno(pRes), 0);
+  taos_free_result(pRes);
+}

source/common/src/tdatablock.c

@@ -1752,7 +1752,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       taosArrayClear(tagArray);
       taosArrayPush(tagArray, &tagVal);
       tTagNew(tagArray, 1, false, &pTag);
-      if (!pTag) {
+      if (pTag == NULL) {
         tdDestroySVCreateTbReq(&createTbReq);
         taosArrayDestroy(tagArray);
         return NULL;
@@ -1763,9 +1763,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
 
       tdDestroySVCreateTbReq(&createTbReq);
-
       if (code < 0) {
-        tdDestroySVCreateTbReq(&createTbReq);
         taosArrayDestroy(tagArray);
         return NULL;
       }
@@ -1775,6 +1773,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
   }
 
   // assign data
+  // TODO
   ret = taosMemoryCalloc(1, cap + 46);
   ret = POINTER_SHIFT(ret, 46);
   ret->header.vgId = vgId;
@@ -1804,8 +1803,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
     int32_t schemaLen = 0;
     if (createTb) {
       SVCreateTbReq createTbReq = {0};
-      char*         cname = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
+      char*         cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
-      snprintf(cname, TSDB_TABLE_FNAME_LEN, "%s:%ld", stbFullName, pDataBlock->info.groupId);
       createTbReq.name = cname;
       createTbReq.flags = 0;
       createTbReq.type = TSDB_CHILD_TABLE;
@@ -1819,7 +1817,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       taosArrayPush(tagArray, &tagVal);
       STag* pTag = NULL;
       tTagNew(tagArray, 1, false, &pTag);
-      if (!pTag) {
+      if (pTag == NULL) {
         tdDestroySVCreateTbReq(&createTbReq);
         taosArrayDestroy(tagArray);
         taosMemoryFreeClear(ret);
@@ -1945,7 +1943,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
 
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
   blockDataEnsureCapacity(pBlock, numOfRows);
-  pBlock->info.rows = numOfRows;
 
   const char* pStart = pData;
 
@@ -2019,6 +2016,7 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
     pStart += colLen[i];
   }
 
+  pBlock->info.rows = numOfRows;
   ASSERT(pStart - pData == dataLen);
   return pStart;
 }

source/common/src/tdataformat.c

@@ -244,7 +244,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     }
   }
 
-  // ASSERT(flags); // only 1 column(ts)
+  ASSERT(flags);
 
   // decide
   uint32_t nData = 0;
@@ -268,8 +268,8 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
         nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv;
         break;
       default:
-        break; // only ts column
+        break;
-        // ASSERT(0);
+        ASSERT(0);
     }
 
     uint8_t tflags = 0;
@@ -374,7 +374,7 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
           ptv = pf + pTSchema->flen;
           break;
         default:
-          // ASSERT(0);
+          ASSERT(0);
           break;
       }
     } else {
@@ -421,12 +421,26 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     _set_none:
       if ((flags & 0xf0) == 0) {
         setBitMap(pb, 0, iColumn - 1, flags);
+        if (flags & TSROW_HAS_VAL) { // set 0
+          if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+            *(VarDataOffsetT *)(pf + pTColumn->offset) = 0;
+          } else {
+            tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type);
+          }
+        }
       }
       continue;
 
     _set_null:
       if ((flags & 0xf0) == 0) {
         setBitMap(pb, 1, iColumn - 1, flags);
+        if (flags & TSROW_HAS_VAL) { // set 0
+          if (IS_VAR_DATA_TYPE(pTColumn->type)) {
+            *(VarDataOffsetT *)(pf + pTColumn->offset) = 0;
+          } else {
+            tPutValue(pf + pTColumn->offset, &((SValue){0}), pTColumn->type);
+          }
+        }
       } else {
         SET_IDX(pidx, pTSKVRow->nCols, nkv, flags);
         pTSKVRow->nCols++;
@@ -497,7 +511,7 @@ void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal
   SValue    value;
 
   ASSERT(iCol < pTSchema->numOfCols);
-  // ASSERT(flags); // only 1 ts column
+  ASSERT(flags);
   ASSERT(pRow->sver == pTSchema->version);
 
   if (iCol == 0) {

source/common/src/tmsg.c

@@ -2491,6 +2491,15 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR
   return 0;
 }
 
+void tFreeSTableIndexInfo(void* info) {
+  if (NULL == info) {
+    return;
+  }
+
+  STableIndexInfo *pInfo = (STableIndexInfo*)info;
+
+  taosMemoryFree(pInfo->expr);
+}
 
 int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   SEncoder encoder = {0};

source/common/test/dataformatTest.cpp

@@ -52,61 +52,61 @@ STSchema *genSTSchema(int16_t nCols) {
 
     switch (i) {
       case 0: {
-        pSchema[0].type = TSDB_DATA_TYPE_TIMESTAMP;
+        pSchema[i].type = TSDB_DATA_TYPE_TIMESTAMP;
-        pSchema[0].bytes = TYPE_BYTES[pSchema[0].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 1: {
-        pSchema[1].type = TSDB_DATA_TYPE_INT;
+        pSchema[i].type = TSDB_DATA_TYPE_INT;
-        pSchema[1].bytes = TYPE_BYTES[pSchema[1].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
         ;
       } break;
       case 2: {
-        pSchema[2].type = TSDB_DATA_TYPE_BIGINT;
+        pSchema[i].type = TSDB_DATA_TYPE_BIGINT;
-        pSchema[2].bytes = TYPE_BYTES[pSchema[2].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 3: {
-        pSchema[3].type = TSDB_DATA_TYPE_FLOAT;
+        pSchema[i].type = TSDB_DATA_TYPE_FLOAT;
-        pSchema[3].bytes = TYPE_BYTES[pSchema[3].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 4: {
-        pSchema[4].type = TSDB_DATA_TYPE_DOUBLE;
+        pSchema[i].type = TSDB_DATA_TYPE_DOUBLE;
-        pSchema[4].bytes = TYPE_BYTES[pSchema[4].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 5: {
-        pSchema[5].type = TSDB_DATA_TYPE_BINARY;
+        pSchema[i].type = TSDB_DATA_TYPE_BINARY;
-        pSchema[5].bytes = 12;
+        pSchema[i].bytes = 12;
       } break;
       case 6: {
-        pSchema[6].type = TSDB_DATA_TYPE_NCHAR;
+        pSchema[i].type = TSDB_DATA_TYPE_NCHAR;
-        pSchema[6].bytes = 42;
+        pSchema[i].bytes = 42;
       } break;
       case 7: {
-        pSchema[7].type = TSDB_DATA_TYPE_TINYINT;
+        pSchema[i].type = TSDB_DATA_TYPE_TINYINT;
-        pSchema[7].bytes = TYPE_BYTES[pSchema[7].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 8: {
-        pSchema[8].type = TSDB_DATA_TYPE_SMALLINT;
+        pSchema[i].type = TSDB_DATA_TYPE_SMALLINT;
-        pSchema[8].bytes = TYPE_BYTES[pSchema[8].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 9: {
-        pSchema[9].type = TSDB_DATA_TYPE_BOOL;
+        pSchema[i].type = TSDB_DATA_TYPE_BOOL;
-        pSchema[9].bytes = TYPE_BYTES[pSchema[9].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 10: {
-        pSchema[10].type = TSDB_DATA_TYPE_UTINYINT;
+        pSchema[i].type = TSDB_DATA_TYPE_UTINYINT;
-        pSchema[10].bytes = TYPE_BYTES[pSchema[10].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 11: {
-        pSchema[11].type = TSDB_DATA_TYPE_USMALLINT;
+        pSchema[i].type = TSDB_DATA_TYPE_USMALLINT;
-        pSchema[11].bytes = TYPE_BYTES[pSchema[11].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 12: {
-        pSchema[12].type = TSDB_DATA_TYPE_UINT;
+        pSchema[i].type = TSDB_DATA_TYPE_UINT;
-        pSchema[12].bytes = TYPE_BYTES[pSchema[12].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
       case 13: {
-        pSchema[13].type = TSDB_DATA_TYPE_UBIGINT;
+        pSchema[i].type = TSDB_DATA_TYPE_UBIGINT;
-        pSchema[13].bytes = TYPE_BYTES[pSchema[13].type];
+        pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
       } break;
 
       default:
@@ -146,9 +146,9 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
       case 0:
         sscanf(data[i], "%" PRIi64, &colVal.value.ts);
         break;
-      case 1: {
+      case 1:
         sscanf(data[i], "%" PRIi32, &colVal.value.i32);
-      } break;
+        break;
       case 2:
         sscanf(data[i], "%" PRIi64, &colVal.value.i64);
         break;
@@ -274,9 +274,6 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
     case TSDB_DATA_TYPE_MEDIUMBLOB:
       printf("MedBLOB ");
       break;
-    // case TSDB_DATA_TYPE_BINARY:
-    //   printf("BINARY ");
-    //   break;
     case TSDB_DATA_TYPE_MAX:
       printf("UNDEF ");
       break;
@@ -404,9 +401,10 @@ static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
 }
 
 TEST(testCase, AllNormTest) {
-  int16_t  nCols = 1;
+  int16_t       nCols = 14;
-  STSRow2 *row = nullptr;
+  STSRowBuilder rb = {0};
-  SArray  *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  STSRow2      *row = nullptr;
+  SArray       *pArray = taosArrayInit(nCols, sizeof(SColVal));
   EXPECT_NE(pArray, nullptr);
 
   STSchema *pTSchema = genSTSchema(nCols);
@@ -414,15 +412,16 @@ TEST(testCase, AllNormTest) {
 
   // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
   // c9 bool
-  char *data[10] = {"1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "1"};
+  char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"};
 
   genTestData((const char **)&data, nCols, &pArray);
 
-  tTSRowNew(NULL, pArray, pTSchema, &row);
+  tTSRowNew(&rb, pArray, pTSchema, &row);
 
   debugPrintTSRow(row, pTSchema, __func__, __LINE__);
   checkTSRow((const char **)&data, row, pTSchema);
 
+  tsRowBuilderClear(&rb);
   taosArrayDestroy(pArray);
   taosMemoryFree(pTSchema);
 }
@@ -443,12 +442,12 @@ TEST(testCase, NoneTest) {
   const char *data[nRows][nCols] = {
       {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
       {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220002", "no", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "nu"},
+      {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220003", "nu", "no", "no", "no", "no", "nu", "no", "no", "no", "no", "no", "no", "no"},
+      {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
       {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
       {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
       {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
-      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
       {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
       {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
       {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
@@ -465,13 +464,12 @@ TEST(testCase, NoneTest) {
       {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
        "no"}};
 
-
   for (int r = 0; r < nRows; ++r) {
     genTestData((const char **)&data[r], nCols, &pArray);
     tTSRowNew(NULL, pArray, pTSchema, &row);
     debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
     checkTSRow((const char **)&data[r], row, pTSchema);  // check
-    taosMemoryFreeClear(row);
+    tTSRowFree(row);
     taosArrayClear(pArray);
   }
 

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -215,8 +215,12 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER;
 
+  if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
+
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
@@ -226,9 +230,8 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
-
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
 
   code = 0;
 

source/dnode/mgmt/mgmt_mnode/src/mmWorker.c

@@ -71,7 +71,7 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
 }
 
 static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
-  dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));
+  dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));
   taosWriteQitem(pWorker->queue, pMsg);
   return 0;
 }

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -359,6 +359,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

source/dnode/mnode/impl/inc/mndInt.h

@@ -19,6 +19,7 @@
 #include "mndDef.h"
 
 #include "sdb.h"
+#include "sync.h"
 #include "syncTools.h"
 #include "tcache.h"
 #include "tdatablock.h"
@@ -75,7 +76,6 @@ typedef struct {
 } STelemMgmt;
 
 typedef struct {
-  SWal   *pWal;
   sem_t   syncSem;
   int64_t sync;
   bool    standby;
@@ -108,6 +108,7 @@ typedef struct SMnode {
   SQHandle      *pQuery;
   SHashObj      *infosMeta;
   SHashObj      *perfsMeta;
+  SWal          *pWal;
   SShowMgmt      showMgmt;
   SProfileMgmt   profileMgmt;
   STelemMgmt     telemMgmt;

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -41,6 +41,7 @@ int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pV
 int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
 int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray);
 int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId);
+int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray);
 
 void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby);
 void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndDb.c

@@ -636,57 +636,6 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
-  if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
-    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) {
-      return -1;
-    }
-  } else {
-    SVgObj newVgroup = {0};
-    memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
-    mndTransSetSerial(pTrans);
-
-    if (newVgroup.replica < pDb->cfg.replications) {
-      mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
-            pVgroup->vnodeGid[0].dnodeId);
-
-      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-
-      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
-      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-    } else {
-      mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
-
-      SVnodeGid del1 = {0};
-      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-
-      SVnodeGid del2 = {0};
-      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
-      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
-      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
-      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
-    }
-
-    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
-    if (pVgRaw == NULL) return -1;
-    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
-      sdbFreeRaw(pVgRaw);
-      return -1;
-    }
-    sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
-  }
-
-  return 0;
-}
-
 static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;

source/dnode/mnode/impl/src/mndDnode.c

@@ -566,9 +566,11 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
   pRaw = NULL;
 
   if (pMObj != NULL) {
+    mDebug("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
     if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER;
   }
   if (numOfVnodes > 0) {
+    mDebug("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
     if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
   }
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndMain.c

@@ -139,10 +139,40 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
   return 0;
 }
 
+static int32_t mndInitWal(SMnode *pMnode) {
+  char path[PATH_MAX + 20] = {0};
+  snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
+  SWalCfg cfg = {
+      .vgId = 1,
+      .fsyncPeriod = 0,
+      .rollPeriod = -1,
+      .segSize = -1,
+      .retentionPeriod = -1,
+      .retentionSize = -1,
+      .level = TAOS_WAL_FSYNC,
+  };
+
+  pMnode->pWal = walOpen(path, &cfg);
+  if (pMnode->pWal == NULL) {
+    mError("failed to open wal since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void mndCloseWal(SMnode *pMnode) {
+  if (pMnode->pWal != NULL) {
+    walClose(pMnode->pWal);
+    pMnode->pWal = NULL;
+  }
+}
+
 static int32_t mndInitSdb(SMnode *pMnode) {
   SSdbOpt opt = {0};
   opt.path = pMnode->path;
   opt.pMnode = pMnode;
+  opt.pWal = pMnode->pWal;
 
   pMnode->pSdb = sdbInit(&opt);
   if (pMnode->pSdb == NULL) {
@@ -156,7 +186,6 @@ static int32_t mndOpenSdb(SMnode *pMnode) {
   if (!pMnode->deploy) {
     return sdbReadFile(pMnode->pSdb);
   } else {
-    // return sdbDeploy(pMnode->pSdb);;
     return 0;
   }
 }
@@ -182,6 +211,7 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
 }
 
 static int32_t mndInitSteps(SMnode *pMnode) {
+  if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1;
@@ -201,7 +231,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
-  if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
@@ -376,41 +406,93 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
   syncRpcMsgLog2(logBuf, pMsg);
   taosMemoryFree(syncNodeStr);
 
-  if (pMsg->msgType == TDMT_SYNC_TIMEOUT) {
+  // ToDo: ugly! use function pointer
-    SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg);
+  if (syncNodeSnapshotEnable(pSyncNode)) {
-    code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+    if (pMsg->msgType == TDMT_SYNC_TIMEOUT) {
-    syncTimeoutDestroy(pSyncMsg);
+      SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg);
-  } else if (pMsg->msgType == TDMT_SYNC_PING) {
+      code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
-    SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
+      syncTimeoutDestroy(pSyncMsg);
-    code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_PING) {
-    syncPingDestroy(pSyncMsg);
+      SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
-  } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
+      code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
-    SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
+      syncPingDestroy(pSyncMsg);
-    code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
-    syncPingReplyDestroy(pSyncMsg);
+      SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
-  } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
+      code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
-    SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
+      syncPingReplyDestroy(pSyncMsg);
-    code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
-    syncClientRequestDestroy(pSyncMsg);
+      SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
-  } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
+      code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
-    SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
+      syncClientRequestDestroy(pSyncMsg);
-    code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+
-    syncRequestVoteDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
-  } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
+      SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
-    SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnRequestVoteSnapshotCb(pSyncNode, pSyncMsg);
-    code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+      syncRequestVoteDestroy(pSyncMsg);
-    syncRequestVoteReplyDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
-  } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) {
+      SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg);
-    SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg);
+      code = syncNodeOnRequestVoteReplySnapshotCb(pSyncNode, pSyncMsg);
-    code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+      syncRequestVoteReplyDestroy(pSyncMsg);
-    syncAppendEntriesDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) {
-  } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
+      SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg);
-    SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnAppendEntriesSnapshotCb(pSyncNode, pSyncMsg);
-    code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+      syncAppendEntriesDestroy(pSyncMsg);
-    syncAppendEntriesReplyDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
+      SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnAppendEntriesReplySnapshotCb(pSyncNode, pSyncMsg);
+      syncAppendEntriesReplyDestroy(pSyncMsg);
+
+    } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) {
+      SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg);
+      code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg);
+      syncSnapshotSendDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) {
+      SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg);
+      code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg);
+      syncSnapshotRspDestroy(pSyncMsg);
+
+    } else {
+      mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType));
+      code = TAOS_SYNC_PROPOSE_OTHER_ERROR;
+    }
+
   } else {
-    mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType));
+    if (pMsg->msgType == TDMT_SYNC_TIMEOUT) {
-    code = TAOS_SYNC_PROPOSE_OTHER_ERROR;
+      SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg);
+      code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+      syncTimeoutDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_PING) {
+      SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg);
+      code = syncNodeOnPingCb(pSyncNode, pSyncMsg);
+      syncPingDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) {
+      SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+      syncPingReplyDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) {
+      SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg);
+      code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+      syncClientRequestDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) {
+      SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg);
+      code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+      syncRequestVoteDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) {
+      SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+      syncRequestVoteReplyDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) {
+      SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg);
+      code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+      syncAppendEntriesDestroy(pSyncMsg);
+    } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) {
+      SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg);
+      code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+      syncAppendEntriesReplyDestroy(pSyncMsg);
+    } else {
+      mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType));
+      code = TAOS_SYNC_PROPOSE_OTHER_ERROR;
+    }
   }
 
   mndReleaseSyncRef(pMnode);

source/dnode/mnode/impl/src/mndScheduler.c

@@ -35,6 +35,13 @@
 
 extern bool tsStreamSchedV;
 
+static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
+  int32_t childId = taosArrayGetSize(pArray);
+  pTask->childId = childId;
+  taosArrayPush(pArray, &pTask);
+  return 0;
+}
+
 int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr,
                            int32_t* pLen, double filesFactor) {
   SNode*      pAst = NULL;
@@ -186,7 +193,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
     SVgObj* pVgroup;
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
     if (pIter == NULL) break;
-    if (pVgroup->dbUid != pStream->dbUid) {
+    if (strcmp(pVgroup->dbName, pStream->targetDb) != 0) {
       sdbRelease(pSdb, pVgroup);
       continue;
     }
@@ -195,7 +202,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    taosArrayPush(tasks, &pTask);
+    mndAddTaskToTaskSet(tasks, pTask);
 
     pTask->nodeId = pVgroup->vgId;
     pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
@@ -235,7 +242,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-  taosArrayPush(tasks, &pTask);
+  mndAddTaskToTaskSet(tasks, pTask);
 
   pTask->nodeId = pStream->fixedSinkVgId;
 #if 0
@@ -286,7 +293,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   pStream->tasks = taosArrayInit(totLevel, sizeof(void*));
 
   bool hasExtraSink = false;
-  if (totLevel == 2 || strcmp(pStream->sourceDb, pStream->targetDb) != 0) {
+  bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
+  if (totLevel == 2 || externalTargetDB) {
     SArray* taskOneLevel = taosArrayInit(0, sizeof(void*));
     taosArrayPush(pStream->tasks, &taskOneLevel);
     // add extra sink
@@ -377,7 +385,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
           return -1;
         }
         sdbRelease(pSdb, pVgroup);
-        taosArrayPush(taskOneLevel, &pTask);
+        mndAddTaskToTaskSet(taskOneLevel, pTask);
       }
     } else {
       // merge plan
@@ -405,7 +413,6 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         if (pStream->fixedSinkVgId == 0) {
           pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
 
-          /*pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;*/
           pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
           SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb);
           ASSERT(pDb);
@@ -426,10 +433,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
             SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
             for (int32_t j = 0; j < sinkLvSize; j++) {
               SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
-              /*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
               if (pLastLevelTask->nodeId == pVgInfo->vgId) {
                 pVgInfo->taskId = pLastLevelTask->taskId;
-                /*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
                 break;
               }
             }

source/dnode/mnode/impl/src/mndSync.c

@@ -17,15 +17,27 @@
 #include "mndSync.h"
 #include "mndTrans.h"
 
-int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
+static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   SMsgHead *pHead = pMsg->pCont;
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = htonl(pHead->vgId);
 
-  return tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
+  int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
 }
 
-int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return tmsgSendReq(pEpSet, pMsg); }
+static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  int32_t code = tmsgSendReq(pEpSet, pMsg);
+  if (code != 0) {
+    rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
+  }
+  return code;
+}
 
 void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   SMnode    *pMnode = pFsm->data;
@@ -34,7 +46,7 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
 
   int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw);
   pMgmt->errCode = cbMeta.code;
-  mTrace("trans:%d, is proposed, savedTransId:%d code:0x%x, ver:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId,
+  mDebug("trans:%d, is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64 " role:%s raw:%p", transId,
          pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term, syncStr(cbMeta.state), pRaw);
 
   if (pMgmt->errCode == 0) {
@@ -50,6 +62,10 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
     tsem_post(&pMgmt->syncSem);
   } else {
     if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) {
+      SSnapshotMeta sMeta = {0};
+      if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) {
+        sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex);
+      }
       sdbWriteFile(pMnode->pSdb);
     }
   }
@@ -57,13 +73,20 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
 
 int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   SMnode *pMnode = pFsm->data;
-  pSnapshot->lastApplyIndex = sdbGetApplyIndex(pMnode->pSdb);
+  pSnapshot->lastApplyIndex = sdbGetCommitIndex(pMnode->pSdb);
-  pSnapshot->lastApplyTerm = sdbGetApplyTerm(pMnode->pSdb);
+  pSnapshot->lastApplyTerm = sdbGetCommitTerm(pMnode->pSdb);
+  pSnapshot->lastConfigIndex = sdbGetCurConfig(pMnode->pSdb);
   return 0;
 }
 
 void mndRestoreFinish(struct SSyncFSM *pFsm) {
   SMnode *pMnode = pFsm->data;
+
+  SSnapshotMeta sMeta = {0};
+  if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) {
+    sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex);
+  }
+
   if (!pMnode->deploy) {
     mInfo("mnode sync restore finished, and will handle outstanding transactions");
     mndTransPullup(pMnode);
@@ -78,8 +101,8 @@ void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta)
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
 
   pMgmt->errCode = cbMeta.code;
-  mInfo("trans:-1, sync reconfig is proposed, savedTransId:%d code:0x%x, curTerm:%" PRId64 " term:%" PRId64,
+  mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId,
-        pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term);
+        cbMeta.code, cbMeta.index, cbMeta.term);
 
   if (pMgmt->transId == -1) {
     if (pMgmt->errCode != 0) {
@@ -144,29 +167,12 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
 int32_t mndInitSync(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
 
-  char path[PATH_MAX + 20] = {0};
-  snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
-  SWalCfg cfg = {
-      .vgId = 1,
-      .fsyncPeriod = 0,
-      .rollPeriod = -1,
-      .segSize = -1,
-      .retentionPeriod = -1,
-      .retentionSize = -1,
-      .level = TAOS_WAL_FSYNC,
-  };
-
-  pMgmt->pWal = walOpen(path, &cfg);
-  if (pMgmt->pWal == NULL) {
-    mError("failed to open wal since %s", terrstr());
-    return -1;
-  }
-
   SSyncInfo syncInfo = {.vgId = 1, .FpSendMsg = mndSyncSendMsg, .FpEqMsg = mndSyncEqMsg};
   snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP);
-  syncInfo.pWal = pMgmt->pWal;
+  syncInfo.pWal = pMnode->pWal;
   syncInfo.pFsm = mndSyncMakeFsm(pMnode);
   syncInfo.isStandBy = pMgmt->standby;
+  syncInfo.snapshotEnable = true;
 
   SSyncCfg *pCfg = &syncInfo.syncCfg;
   pCfg->replicaNum = pMnode->replica;
@@ -196,10 +202,6 @@ void mndCleanupSync(SMnode *pMnode) {
   mDebug("mnode sync is stopped, id:%" PRId64, pMgmt->sync);
 
   tsem_destroy(&pMgmt->syncSem);
-  if (pMgmt->pWal != NULL) {
-    walClose(pMgmt->pWal);
-  }
-
   memset(pMgmt, 0, sizeof(SSyncMgmt));
 }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -922,7 +922,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
   char    detail[1024] = {0};
   int32_t len = snprintf(detail, sizeof(detail), "msgType:%s numOfEps:%d inUse:%d", TMSG_INFO(pAction->msgType),
                          pAction->epSet.numOfEps, pAction->epSet.inUse);
-  for (int32_t i = 0; i < pTrans->lastErrorEpset.numOfEps; ++i) {
+  for (int32_t i = 0; i < pAction->epSet.numOfEps; ++i) {
     len += snprintf(detail + len, sizeof(detail) - len, " ep:%d-%s:%u", i, pAction->epSet.eps[i].fqdn,
                     pAction->epSet.eps[i].port);
   }
@@ -1085,6 +1085,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
     }
 
     if (code == 0) {
+      if (!pMnode->deploy && !mndIsMaster(pMnode)) break;
+
       pTrans->code = 0;
       pTrans->redoActionPos++;
       mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage),
@@ -1386,6 +1388,10 @@ void mndTransPullup(SMnode *pMnode) {
     mndReleaseTrans(pMnode, pTrans);
   }
 
+  SSnapshotMeta sMeta = {0};
+  if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) {
+    sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex);
+  }
   sdbWriteFile(pMnode->pSdb);
   taosArrayDestroy(pArray);
 }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -55,6 +55,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_HASHRANGE_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp);
 
@@ -1166,7 +1167,155 @@ _OVER:
   return code;
 }
 
-static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; }
+int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
+  if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_CONFIG) != 0) {
+      return -1;
+    }
+  } else {
+    SVgObj newVgroup = {0};
+    memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
+    mndTransSetSerial(pTrans);
+
+    if (newVgroup.replica < pDb->cfg.replications) {
+      mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId,
+            pVgroup->vnodeGid[0].dnodeId);
+
+      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+
+      if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
+      if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+    } else if (newVgroup.replica > pDb->cfg.replications) {
+      mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
+
+      SVnodeGid del1 = {0};
+      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
+      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+
+      SVnodeGid del2 = {0};
+      if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
+      if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+      if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
+      if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
+    } else {
+    }
+
+    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+    if (pVgRaw == NULL) return -1;
+    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
+      sdbFreeRaw(pVgRaw);
+      return -1;
+    }
+    sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+  }
+
+  return 0;
+}
+
+static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
+  return 0;
+}
+
+static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
+  int32_t  code = -1;
+  SSdbRaw *pRaw = NULL;
+  STrans  *pTrans = NULL;
+  SArray  *pArray = mndBuildDnodesArray(pMnode, 0);
+
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
+  if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
+  mDebug("trans:%d, used to split vgroup, vgId:%d", pTrans->id, pVgroup->vgId);
+
+  SVgObj newVg1 = {0};
+  memcpy(&newVg1, pVgroup, sizeof(SVgObj));
+  mInfo("vgId:%d, vgroup info before split, replica:%d hashBegin:%u hashEnd:%u", newVg1.vgId, newVg1.replica,
+        newVg1.hashBegin, newVg1.hashEnd);
+  for (int32_t i = 0; i < newVg1.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
+  }
+
+  if (newVg1.replica == 1) {
+    if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER;
+    if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1], true) != 0) goto _OVER;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
+  } else if (newVg1.replica == 3) {
+    SVnodeGid del1 = {0};
+    if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER;
+    if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER;
+    if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
+    if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
+  } else {
+    goto _OVER;
+  }
+
+  SVgObj newVg2 = {0};
+  memcpy(&newVg1, &newVg2, sizeof(SVgObj));
+  newVg1.replica = 1;
+  newVg1.hashEnd = (newVg1.hashBegin + newVg1.hashEnd) / 2;
+  memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));
+
+  newVg2.replica = 1;
+  newVg2.hashBegin = newVg1.hashEnd + 1;
+  memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid));
+  memset(&newVg1.vnodeGid[1], 0, sizeof(SVnodeGid));
+
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER;
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg2, TDMT_VND_ALTER_HASHRANGE) != 0) goto _OVER;
+
+  // adjust vgroup
+  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg1, pArray) != 0) goto _OVER;
+  if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, &newVg2, pArray) != 0) goto _OVER;
+
+_OVER:
+  mndTransDrop(pTrans);
+  sdbFreeRaw(pRaw);
+  return code;
+}
+
+static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) {
+  SMnode   *pMnode = pReq->info.node;
+  int32_t   code = -1;
+  int32_t   vgId = 2;
+  SUserObj *pUser = NULL;
+  SVgObj   *pVgroup = NULL;
+  SDbObj   *pDb = NULL;
+
+  mDebug("vgId:%d, start to split", vgId);
+
+  pVgroup = mndAcquireVgroup(pMnode, vgId);
+  if (pVgroup == NULL) goto _OVER;
+
+  pDb = mndAcquireDb(pMnode, pVgroup->dbName);
+  if (pDb == NULL) goto _OVER;
+
+  pUser = mndAcquireUser(pMnode, pReq->conn.user);
+  if (pUser == NULL) {
+    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+    goto _OVER;
+  }
+
+  if (mndCheckNodeAuth(pUser) != 0) {
+    goto _OVER;
+  }
+
+  code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup);
+  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
+
+_OVER:
+  mndReleaseUser(pMnode, pUser);
+  mndReleaseVgroup(pMnode, pVgroup);
+  mndReleaseDb(pMnode, pDb);
+  return code;
+}
 
 static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
                                               SDnodeObj *pSrc, SDnodeObj *pDst) {

source/dnode/mnode/sdb/inc/sdb.h

@@ -22,6 +22,7 @@
 #include "tlockfree.h"
 #include "tlog.h"
 #include "tmsg.h"
+#include "wal.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -165,12 +166,14 @@ typedef struct SSdbRow {
 
 typedef struct SSdb {
   SMnode        *pMnode;
+  SWal          *pWal;
   char          *currDir;
   char          *tmpDir;
   int64_t        lastCommitVer;
   int64_t        lastCommitTerm;
   int64_t        curVer;
   int64_t        curTerm;
+  int64_t        curConfig;
   int64_t        tableVer[SDB_MAX];
   int64_t        maxId[SDB_MAX];
   EKeyType       keyTypes[SDB_MAX];
@@ -205,6 +208,7 @@ typedef struct {
 typedef struct SSdbOpt {
   const char *path;
   SMnode     *pMnode;
+  SWal       *pWal;
 } SSdbOpt;
 
 /**
@@ -358,9 +362,13 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type);
  * @return int32_t The current index of sdb
  */
 void    sdbSetApplyIndex(SSdb *pSdb, int64_t index);
-int64_t sdbGetApplyIndex(SSdb *pSdb);
 void    sdbSetApplyTerm(SSdb *pSdb, int64_t term);
+void    sdbSetCurConfig(SSdb *pSdb, int64_t config);
+int64_t sdbGetApplyIndex(SSdb *pSdb);
 int64_t sdbGetApplyTerm(SSdb *pSdb);
+int64_t sdbGetCommitIndex(SSdb *pSdb);
+int64_t sdbGetCommitTerm(SSdb *pSdb);
+int64_t sdbGetCurConfig(SSdb *pSdb);
 
 SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen);
 void     sdbFreeRaw(SSdbRaw *pRaw);

source/dnode/mnode/sdb/src/sdb.c

@@ -52,10 +52,12 @@ SSdb *sdbInit(SSdbOpt *pOption) {
     pSdb->keyTypes[i] = SDB_KEY_INT32;
   }
 
+  pSdb->pWal = pOption->pWal;
   pSdb->curVer = -1;
   pSdb->curTerm = -1;
   pSdb->lastCommitVer = -1;
   pSdb->lastCommitTerm = -1;
+  pSdb->curConfig = -1;
   pSdb->pMnode = pOption->pMnode;
   taosThreadMutexInit(&pSdb->filelock, NULL);
   mDebug("sdb init successfully");
@@ -159,8 +161,16 @@ static int32_t sdbCreateDir(SSdb *pSdb) {
 
 void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; }
 
-int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; }
-
 void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; }
 
+void sdbSetCurConfig(SSdb *pSdb, int64_t config) { pSdb->curConfig = config; }
+
+int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; }
+
 int64_t sdbGetApplyTerm(SSdb *pSdb) { return pSdb->curTerm; }
+
+int64_t sdbGetCommitIndex(SSdb *pSdb) { return pSdb->lastCommitVer; }
+
+int64_t sdbGetCommitTerm(SSdb *pSdb) { return pSdb->lastCommitTerm; }
+
+int64_t sdbGetCurConfig(SSdb *pSdb) { return pSdb->curConfig; }

source/dnode/mnode/sdb/src/sdbFile.c

@@ -110,6 +110,16 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
     return -1;
   }
 
+  ret = taosReadFile(pFile, &pSdb->curConfig, sizeof(int64_t));
+  if (ret < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+  if (ret != sizeof(int64_t)) {
+    terrno = TSDB_CODE_FILE_CORRUPTED;
+    return -1;
+  }
+
   for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
     int64_t maxId = 0;
     ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
@@ -173,6 +183,11 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
     return -1;
   }
 
+  if (taosWriteFile(pFile, &pSdb->curConfig, sizeof(int64_t)) != sizeof(int64_t)) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
     int64_t maxId = 0;
     if (i < SDB_MAX) {
@@ -288,8 +303,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
   pSdb->lastCommitVer = pSdb->curVer;
   pSdb->lastCommitTerm = pSdb->curTerm;
   memcpy(pSdb->tableVer, tableVer, sizeof(tableVer));
-  mDebug("read sdb file:%s successfully, ver:%" PRId64 " term:%" PRId64, file, pSdb->lastCommitVer,
+  mDebug("read sdb file:%s successfully, index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, pSdb->lastCommitVer,
-         pSdb->lastCommitTerm);
+         pSdb->lastCommitTerm, pSdb->curConfig);
 
 _OVER:
   taosCloseFile(&pFile);
@@ -426,12 +441,23 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
 }
 
 int32_t sdbWriteFile(SSdb *pSdb) {
+  int32_t code = 0;
   if (pSdb->curVer == pSdb->lastCommitVer) {
     return 0;
   }
 
   taosThreadMutexLock(&pSdb->filelock);
-  int32_t code = sdbWriteFileImp(pSdb);
+  if (pSdb->pWal != NULL) {
+    code = walBeginSnapshot(pSdb->pWal, pSdb->curVer);
+  }
+  if (code == 0) {
+    code = sdbWriteFileImp(pSdb);
+  }
+  if (code == 0) {
+    if (pSdb->pWal != NULL) {
+      code = walEndSnapshot(pSdb->pWal);
+    }
+  }
   if (code != 0) {
     mError("failed to write sdb file since %s", terrstr());
   }
@@ -496,6 +522,9 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) {
   snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
 
   taosThreadMutexLock(&pSdb->filelock);
+  int64_t commitIndex = pSdb->lastCommitVer;
+  int64_t commitTerm = pSdb->lastCommitTerm;
+  int64_t curConfig = pSdb->curConfig;
   if (taosCopyFile(datafile, pIter->name) < 0) {
     taosThreadMutexUnlock(&pSdb->filelock);
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -514,7 +543,8 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) {
   }
 
   *ppIter = pIter;
-  mInfo("sdbiter:%p, is created to read snapshot, file:%s", pIter, pIter->name);
+  mInfo("sdbiter:%p, is created to read snapshot, index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pIter,
+        commitIndex, commitTerm, curConfig, pIter->name);
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -24,6 +24,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
 static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRpcMsg *pRsp);
 
 int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
@@ -163,6 +164,9 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
     case TDMT_VND_ALTER_CONFIRM:
       vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp);
       break;
+    case TDMT_VND_ALTER_HASHRANGE:
+      vnodeProcessAlterHasnRangeReq(pVnode, version, pReq, len, pRsp);
+      break;
     case TDMT_VND_ALTER_CONFIG:
       break;
     default:
@@ -889,3 +893,13 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void
 
   return 0;
 }
+
+static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode));
+
+  // todo 
+  // 1. stop work 
+  // 2. adjust hash range / compact / remove wals / rename vgroups
+  // 3. reload sync
+  return 0;
+}

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -252,6 +252,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
 
 int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
   SSyncInfo syncInfo = {
+      .isStandBy = false,
+      .snapshotEnable = false,
       .vgId = pVnode->config.vgId,
       .isStandBy = pVnode->config.standby,
       .syncCfg = pVnode->config.syncCfg,

source/libs/catalog/inc/catalogInt.h

@@ -67,6 +67,7 @@ typedef enum {
   CTG_TASK_GET_DB_INFO,
   CTG_TASK_GET_TB_META,
   CTG_TASK_GET_TB_HASH,
+  CTG_TASK_GET_TB_INDEX,
   CTG_TASK_GET_INDEX,
   CTG_TASK_GET_UDF,
   CTG_TASK_GET_USER,
@@ -93,6 +94,10 @@ typedef struct SCtgTbMetaCtx {
   int32_t flag;
 } SCtgTbMetaCtx;
 
+typedef struct SCtgTbIndexCtx {
+  SName* pName;
+} SCtgTbIndexCtx;
+
 typedef struct SCtgDbVgCtx {
   char dbFName[TSDB_DB_FNAME_LEN];
 } SCtgDbVgCtx;
@@ -189,6 +194,7 @@ typedef struct SCtgJob {
   int32_t          indexNum;
   int32_t          userNum;
   int32_t          dbInfoNum;
+  int32_t          tbIndexNum;
 } SCtgJob;
 
 typedef struct SCtgMsgCtx {
@@ -490,7 +496,7 @@ int32_t ctgGetDBVgInfoFromMnode(CTG_PARAMS, SBuildUseDBInput *input, SUseDbOutpu
 int32_t ctgGetQnodeListFromMnode(CTG_PARAMS, SArray *out, SCtgTask* pTask);
 int32_t ctgGetDBCfgFromMnode(CTG_PARAMS, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask);
 int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *out, SCtgTask* pTask);
-int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, const char *tbFName, SArray** out, SCtgTask* pTask);
+int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, SName* name, SArray** out, SCtgTask* pTask);
 int32_t ctgGetUdfInfoFromMnode(CTG_PARAMS, const char *funcName, SFuncInfo *out, SCtgTask* pTask);
 int32_t ctgGetUserDbAuthFromMnode(CTG_PARAMS, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask);
 int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask);

source/libs/catalog/src/catalog.c

@@ -1136,14 +1136,14 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps
   CTG_API_LEAVE(ctgGetIndexInfoFromMnode(CTG_PARAMS_LIST(), indexName, pInfo, NULL));
 }
 
-int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* tbFName, SArray** pRes) {
+int32_t catalogGetTableIndex(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pRes) {
   CTG_API_ENTER();
   
-  if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == tbFName || NULL == pRes) {
+  if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == pRes) {
     CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
-  CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), tbFName, pRes, NULL));
+  CTG_API_LEAVE(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), (SName*)pTableName, pRes, NULL));
 }
 
 

source/libs/catalog/src/ctgAsync.c

@@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -232,6 +232,35 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user)
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
+  SCtgTask task = {0};
+
+  task.type = CTG_TASK_GET_TB_INDEX;
+  task.taskId = taskIdx;
+  task.pJob = pJob;
+
+  task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbIndexCtx));
+  if (NULL == task.taskCtx) {
+    CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+  }
+
+  SCtgTbIndexCtx* ctx = task.taskCtx;
+  ctx->pName = taosMemoryMalloc(sizeof(*name));
+  if (NULL == ctx->pName) {
+    taosMemoryFree(task.taskCtx);
+    CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+  }
+  
+  memcpy(ctx->pName, name, sizeof(*name));
+
+  taosArrayPush(pJob->pTasks, &task);
+
+  qDebug("QID:%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) {
   int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum;
   if (dbNum > 0) {
@@ -329,8 +358,9 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
   int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex);
   int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser);
   int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo);
+  int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
 
-  *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum;
+  *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum;
   if (*taskNum <= 0) {
     ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId);
     return TSDB_CODE_SUCCESS;
@@ -360,6 +390,7 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
   pJob->indexNum = indexNum;
   pJob->userNum = userNum;
   pJob->dbInfoNum = dbInfoNum;
+  pJob->tbIndexNum = tbIndexNum;
 
   pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask));
 
@@ -398,6 +429,11 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
     CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name));
   }
 
+  for (int32_t i = 0; i < tbIndexNum; ++i) {
+    SName* name = taosArrayGet(pReq->pTableIndex, i);
+    CTG_ERR_JRET(ctgInitGetTbIndexTask(pJob, taskIdx++, name));
+  }
+
   for (int32_t i = 0; i < indexNum; ++i) {
     char* indexName = taosArrayGet(pReq->pIndex, i);
     CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName));
@@ -479,6 +515,21 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgDumpTbIndexRes(SCtgTask* pTask) {
+  SCtgJob* pJob = pTask->pJob;
+  if (NULL == pJob->jobRes.pTableIndex) {
+    pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes));
+    if (NULL == pJob->jobRes.pTableIndex) {
+      CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
+
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableHash, &res);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t ctgDumpIndexRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pIndex) {
@@ -817,6 +868,20 @@ _return:
   CTG_RET(code);
 }
 
+int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
+  int32_t code = 0;
+  CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
+
+  TSWAP(pTask->res, pTask->msgCtx.out);
+  
+_return:
+
+  ctgHandleTaskEnd(pTask, code);
+
+  CTG_RET(code);
+}
+
+
 int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) {
   int32_t code = 0;
   CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
@@ -1056,13 +1121,24 @@ _return:
   CTG_RET(code);
 }
 
+int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) {
+  int32_t code = 0;
+  SCatalog* pCtg = pTask->pJob->pCtg; 
+  void *pTrans = pTask->pJob->pTrans; 
+  const SEpSet* pMgmtEps = &pTask->pJob->pMgmtEps;
+  SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx;
+  
+  CTG_ERR_RET(ctgGetTbIndexFromMnode(CTG_PARAMS_LIST(), pCtx->pName, NULL, pTask));
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) {
   SCatalog* pCtg = pTask->pJob->pCtg; 
   void *pTrans = pTask->pJob->pTrans; 
   const SEpSet* pMgmtEps = &pTask->pJob->pMgmtEps;
 
   CTG_ERR_RET(ctgGetQnodeListFromMnode(CTG_PARAMS_LIST(), NULL, pTask));
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1168,15 +1244,16 @@ int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) {
 }
 
 SCtgAsyncFps gCtgAsyncFps[] = {
-  {ctgLaunchGetQnodeTask,  ctgHandleGetQnodeRsp,  ctgDumpQnodeRes},
+  {ctgLaunchGetQnodeTask,   ctgHandleGetQnodeRsp,   ctgDumpQnodeRes},
-  {ctgLaunchGetDbVgTask,   ctgHandleGetDbVgRsp,   ctgDumpDbVgRes},
+  {ctgLaunchGetDbVgTask,    ctgHandleGetDbVgRsp,    ctgDumpDbVgRes},
-  {ctgLaunchGetDbCfgTask,  ctgHandleGetDbCfgRsp,  ctgDumpDbCfgRes},
+  {ctgLaunchGetDbCfgTask,   ctgHandleGetDbCfgRsp,   ctgDumpDbCfgRes},
-  {ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes},
+  {ctgLaunchGetDbInfoTask,  ctgHandleGetDbInfoRsp,  ctgDumpDbInfoRes},
-  {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes},
+  {ctgLaunchGetTbMetaTask,  ctgHandleGetTbMetaRsp,  ctgDumpTbMetaRes},
-  {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes},
+  {ctgLaunchGetTbHashTask,  ctgHandleGetTbHashRsp,  ctgDumpTbHashRes},
-  {ctgLaunchGetIndexTask,  ctgHandleGetIndexRsp,  ctgDumpIndexRes},
+  {ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes},
-  {ctgLaunchGetUdfTask,    ctgHandleGetUdfRsp,    ctgDumpUdfRes},
+  {ctgLaunchGetIndexTask,   ctgHandleGetIndexRsp,   ctgDumpIndexRes},
-  {ctgLaunchGetUserTask,   ctgHandleGetUserRsp,   ctgDumpUserRes},
+  {ctgLaunchGetUdfTask,     ctgHandleGetUdfRsp,     ctgDumpUdfRes},
+  {ctgLaunchGetUserTask,    ctgHandleGetUserRsp,    ctgDumpUserRes},
 };
 
 int32_t ctgMakeAsyncRes(SCtgJob *pJob) {

source/libs/catalog/src/ctgRemote.c

@@ -427,11 +427,13 @@ int32_t ctgGetIndexInfoFromMnode(CTG_PARAMS, const char *indexName, SIndexInfo *
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, const char *tbFName, SArray** out, SCtgTask* pTask) {
+int32_t ctgGetTbIndexFromMnode(CTG_PARAMS, SName *name, SArray** out, SCtgTask* pTask) {
   char *msg = NULL;
   int32_t msgLen = 0;
   int32_t reqType = TDMT_MND_GET_TABLE_INDEX;
   void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont;
+  char tbFName[TSDB_TABLE_FNAME_LEN];
+  tNameExtractFullName(name, tbFName);
 
   ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName);
 

source/libs/catalog/src/ctgUtil.c

@@ -59,6 +59,9 @@ void ctgFreeSMetaData(SMetaData* pData) {
   
   taosArrayDestroy(pData->pTableHash);
   pData->pTableHash = NULL;
+
+  taosArrayDestroy(pData->pTableIndex);
+  pData->pTableIndex = NULL;
   
   taosArrayDestroy(pData->pUdfList);
   pData->pUdfList = NULL;
@@ -248,6 +251,14 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
       taosMemoryFreeClear(pCtx->out);
       break;
     }
+    case TDMT_MND_GET_TABLE_INDEX: {
+      SArray** pOut = (SArray**)pCtx->out;
+      if (pOut) {
+        taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo);
+        taosMemoryFreeClear(pCtx->out);
+      }
+      break;
+    }
     case TDMT_MND_RETRIEVE_FUNC: {
       SFuncInfo* pOut = (SFuncInfo*)pCtx->out;
       taosMemoryFree(pOut->pCode);
@@ -344,6 +355,13 @@ void ctgFreeTask(SCtgTask* pTask) {
       taosMemoryFreeClear(pTask->res);
       break;
     }
+    case CTG_TASK_GET_TB_INDEX: {
+      SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx;
+      taosMemoryFreeClear(taskCtx->pName);
+      taosMemoryFreeClear(pTask->taskCtx);
+      taosArrayDestroyEx(pTask->res, tFreeSTableIndexInfo);
+      break;
+    }
     case CTG_TASK_GET_INDEX: {
       taosMemoryFreeClear(pTask->taskCtx);
       taosMemoryFreeClear(pTask->res);

source/libs/command/inc/commandInt.h

@@ -32,15 +32,17 @@ extern "C" {
 #define EXPLAIN_PROJECTION_FORMAT "Projection"
 #define EXPLAIN_JOIN_FORMAT "%s"
 #define EXPLAIN_AGG_FORMAT "Aggragate"
+#define EXPLAIN_INDEF_ROWS_FORMAT "Indefinite Rows Function"
 #define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1"
 #define EXPLAIN_SORT_FORMAT "Sort"
 #define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s"
+#define EXPLAIN_FILL_FORMAT "Fill"
 #define EXPLAIN_SESSION_FORMAT "Session"
 #define EXPLAIN_STATE_WINDOW_FORMAT "StateWindow on Column %s"
 #define EXPLAIN_PARITION_FORMAT "Partition on Column %s"
 #define EXPLAIN_ORDER_FORMAT "Order: %s"
 #define EXPLAIN_FILTER_FORMAT "Filter: "
-#define EXPLAIN_FILL_FORMAT "Fill: %s"
+#define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: "
 #define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
 #define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
 #define EXPLAIN_OUTPUT_FORMAT "Output: "
@@ -66,6 +68,8 @@ extern "C" {
 #define EXPLAIN_WIDTH_FORMAT "width=%d"
 #define EXPLAIN_FUNCTIONS_FORMAT "functions=%d"
 #define EXPLAIN_EXECINFO_FORMAT "cost=%.3f..%.3f rows=%" PRIu64
+#define EXPLAIN_MODE_FORMAT "mode=%s"
+#define EXPLAIN_STRING_TYPE_FORMAT "%s"
 
 typedef struct SExplainGroup {
   int32_t   nodeNum;

source/libs/command/src/explain.c

@@ -179,6 +179,21 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
       pPhysiChildren = mergePhysiNode->node.pChildren;
       break;
     }
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: {
+      SIndefRowsFuncPhysiNode *indefPhysiNode = (SIndefRowsFuncPhysiNode *)pNode;
+      pPhysiChildren = indefPhysiNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: {
+      SMergeIntervalPhysiNode *intPhysiNode = (SMergeIntervalPhysiNode *)pNode;
+      pPhysiChildren = intPhysiNode->window.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_FILL: {
+      SFillPhysiNode *fillPhysiNode = (SFillPhysiNode *)pNode;
+      pPhysiChildren = fillPhysiNode->node.pChildren;
+      break;
+    }
     default:
       qError("not supported physical node type %d", pNode->type);
       QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
@@ -212,12 +227,15 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group
   SExplainRsp *rsp = NULL;
   for (int32_t i = 0; i < group->nodeNum; ++i) {
     rsp = taosArrayGet(group->nodeExecInfo, i);
+/*
     if (group->physiPlanExecIdx >= rsp->numOfPlans) {
       qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
       return TSDB_CODE_QRY_APP_ERROR;
     }
 
     taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx);
+*/
+    taosArrayPush(*pExecInfo, rsp->subplanInfo);
   }
 
   ++group->physiPlanExecIdx;
@@ -599,6 +617,42 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       }
       break;
     }
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: {
+      SIndefRowsFuncPhysiNode *pIndefNode = (SIndefRowsFuncPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INDEF_ROWS_FORMAT);
+      EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      }
+      if (pIndefNode->pVectorFuncs) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIndefNode->pVectorFuncs->length);
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIndefNode->node.pOutputDataBlockDesc->totalRowSize);
+      EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
+                           nodesGetOutputNumFromSlotList(pIndefNode->node.pOutputDataBlockDesc->pSlots));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIndefNode->node.pOutputDataBlockDesc->outputRowSize);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+        if (pIndefNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pIndefNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
+                                     TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
     case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: {
       SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
       SExplainGroup      *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId));
@@ -607,7 +661,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
       }
 
-      EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum);
+      EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, pExchNode->singleChannel ? 1 : group->nodeNum);
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
       if (pResNode->pExecInfo) {
         QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@@ -750,6 +804,106 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       }
       break;
     }
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: {
+      SMergeIntervalPhysiNode *pIntNode = (SMergeIntervalPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk));
+      EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length);
+      EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize);
+      EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
+                           nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        uint8_t precision = getIntervalPrecision(pIntNode);
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
+                        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision),
+                        pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
+                        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision),
+                        pIntNode->slidingUnit);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+        if (pIntNode->window.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE,
+                                     TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_FILL: {
+      SFillPhysiNode *pFillNode = (SFillPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_FILL_FORMAT);
+      EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_MODE_FORMAT, nodesGetFillModeString(pFillNode->mode));
+      EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pFillNode->node.pOutputDataBlockDesc->totalRowSize);
+      EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
+                           nodesGetOutputNumFromSlotList(pFillNode->node.pOutputDataBlockDesc->pSlots));
+        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+        EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pFillNode->node.pOutputDataBlockDesc->outputRowSize);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        if (pFillNode->pValues) {
+          SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues;
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT);
+          SNode* tNode = NULL;
+          int32_t i = 0;
+          FOREACH(tNode, pValues->pNodeList) {
+            if (i) {
+              EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+            }
+            SValueNode* tValue = (SValueNode*)tNode;
+            char *value = nodesGetStrValueFromNode(tValue);
+            EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value);
+            taosMemoryFree(value);
+            ++i;
+          }
+
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey,
+                        pFillNode->timeRange.ekey);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+        if (pFillNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pFillNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
+                                     TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
     case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: {
       SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode;
       EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT);

source/libs/executor/inc/executorimpl.h

@@ -804,7 +804,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t
 
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                           SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                          STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo);
+                                          STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream);
 SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
     SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
 SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
@@ -892,8 +892,8 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI
 int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
     __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order);
 int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
-int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey);
+int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey,
-int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey);
+    SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t size);
 SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize);
 SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex);
 int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows,

source/libs/executor/src/executorimpl.c

@@ -1130,6 +1130,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     pCtx->start.key = INT64_MIN;
     pCtx->end.key = INT64_MIN;
     pCtx->numOfParams = pExpr->base.numOfParams;
+    pCtx->increase = false;
 
     pCtx->param = pFunct->pParam;
     //    for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
@@ -2008,8 +2009,16 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
         // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
         SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
         char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
-        for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+        if (pCtx[j].increase) {
-          colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+          int64_t ts = *(int64_t*) in;
+          for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+            colDataAppend(pColInfoData, pBlock->info.rows + k, (const char *)&ts, pCtx[j].resultInfo->isNullRes);
+            ts++;
+          }
+        } else {
+          for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+            colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+          }
         }
       }
     }
@@ -4676,7 +4685,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     }
 
     int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
-    pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo);
+    bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type);
+    pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream);
 
   } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) {
     int32_t children = 8;
@@ -5339,7 +5349,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) {
+int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t size) {
+  pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pSup->keySize = sizeof(int64_t) + sizeof(TSKEY);
   pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize);
   pSup->pResultRows = taosArrayInit(1024, size);
@@ -5358,15 +5369,11 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size
   if (bufSize <= pageSize) {
     bufSize = pageSize * 4;
   }
-  return createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH);
+  int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH);
-}
+  for(int32_t i = 0; i < numOfOutput; ++i) {
-
+    pCtx[i].pBuf = pSup->pResultBuf;
-int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey) {
+  }
-  return initStreamAggSupporter(pSup, pKey, sizeof(SResultWindowInfo));
+  return code;
-}
-
-int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey) {
-  return initStreamAggSupporter(pSup, pKey, sizeof(SStateWindowInfo));
 }
 
 int64_t getSmaWaterMark(int64_t interval, double filesFactor) {

source/libs/executor/src/scanoperator.c

@@ -1756,6 +1756,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
   ;
   pInfo->readHandle = *pReadHandle;
   pInfo->curPos = 0;
+  pInfo->pFilterNode = pPhyNode->node.pConditions;
   pOperator->name = "TagScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
   pOperator->blocking = false;

source/libs/executor/src/timewindowoperator.c

@@ -1438,9 +1438,15 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
   return needed;
 }
 
+void increaseTs(SqlFunctionCtx* pCtx) {
+  if (pCtx[0].pExpr->pExpr->_function.pFunctNode->funcType == FUNCTION_TYPE_WSTARTTS) {
+    pCtx[0].increase = true;
+  }
+}
+
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                           SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                          STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
+                                          STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo, bool isStream) {
   SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
   SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -1460,6 +1466,11 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
 
   int32_t code =
       initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str);
+  
+  if (isStream) {
+    ASSERT(numOfCols > 0);
+    increaseTs(pInfo->binfo.pCtx);
+  }
 
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
 
@@ -2128,6 +2139,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
   int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols,
       pResBlock, keyBufSize, pTaskInfo->id.str);
+  ASSERT(numOfCols > 0);
+  increaseTs(pInfo->binfo.pCtx);
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -2212,6 +2225,8 @@ int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo,
   for (int32_t i = 0; i < numOfCols; ++i) {
     pBasicInfo->pCtx[i].pBuf = NULL;
   }
+  ASSERT(numOfCols > 0);
+  increaseTs(pBasicInfo->pCtx);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2228,6 +2243,10 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int
   pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
 }
 
+int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
+  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo));
+}
+
 SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
                                                   SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId,
                                                   STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
@@ -2244,8 +2263,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
-  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  
-  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
+  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -3097,6 +3116,10 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
+int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
+  return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo));
+}
+
 SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                 SExecTaskInfo* pTaskInfo) {
   SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;
@@ -3130,8 +3153,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
     goto _error;
   }
 
-  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pInfo->binfo.pCtx, numOfCols);
-  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/function/inc/builtinsimpl.h

@@ -102,6 +102,8 @@ bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 int32_t topFunction(SqlFunctionCtx *pCtx);
 int32_t bottomFunction(SqlFunctionCtx *pCtx);
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
+int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 
 bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -114,7 +116,10 @@ int32_t getSpreadInfoSize();
 bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t elapsedFunction(SqlFunctionCtx* pCtx);
+int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getElapsedInfoSize();
 
 bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -126,7 +131,10 @@ int32_t getHistogramInfoSize();
 
 bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t hllFunction(SqlFunctionCtx* pCtx);
+int32_t hllFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getHLLInfoSize();
 
 bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);

source/libs/function/src/builtins.c

@@ -313,6 +313,7 @@ static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int
 static int32_t translateApercentilePartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateApercentileImpl(pFunc, pErrBuf, len, true);
 }
+
 static int32_t translateApercentileMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateApercentileImpl(pFunc, pErrBuf, len, false);
 }
@@ -401,6 +402,7 @@ static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
 static int32_t translateSpreadPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateSpreadImpl(pFunc, pErrBuf, len, true);
 }
+
 static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateSpreadImpl(pFunc, pErrBuf, len, false);
 }
@@ -442,6 +444,64 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+
+  if (isPartial) {
+    if (1 != numOfParams && 2 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    // param1
+    if (2 == numOfParams) {
+      SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
+      if (QUERY_NODE_VALUE != nodeType(pParamNode1)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      SValueNode* pValue = (SValueNode*)pParamNode1;
+
+      pValue->notReserved = true;
+
+      paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
+      if (!IS_INTEGER_TYPE(paraType)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      if (pValue->datum.i == 0) {
+        return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                               "ELAPSED function time unit parameter should be greater than db precision");
+      }
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = getElapsedInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (1 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (TSDB_DATA_TYPE_BINARY != paraType) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateElapsedPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateElapsedImpl(pFunc, pErrBuf, len, true);
+}
+
+static int32_t translateElapsedMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateElapsedImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
   if (3 != numOfParams) {
@@ -551,6 +611,7 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
 static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateHistogramImpl(pFunc, pErrBuf, len, true);
 }
+
 static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return translateHistogramImpl(pFunc, pErrBuf, len, false);
 }
@@ -564,6 +625,28 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHLLImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  if (isPartial) {
+    pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateHLLPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHLLImpl(pFunc, pErrBuf, len, true);
+}
+
+static int32_t translateHLLMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHLLImpl(pFunc, pErrBuf, len, false);
+}
+
 static bool validateStateOper(const SValueNode* pVal) {
   if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
     return false;
@@ -1020,6 +1103,22 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
  *
  */
 
+static bool validateHourRange(int8_t hour) {
+  if (hour < 0 || hour > 12) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
+  if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '-')) {
+    return true;
+  }
+
+  return false;
+}
+
 static bool validateTimezoneFormat(const SValueNode* pVal) {
   if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
     return false;
@@ -1028,6 +1127,8 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
   char*   tz = varDataVal(pVal->datum.p);
   int32_t len = varDataLen(pVal->datum.p);
 
+  char buf[3] = {0};
+  int8_t hour = -1, minute = -1;
   if (len == 0) {
     return false;
   } else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
@@ -1040,6 +1141,20 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
           if (!isdigit(tz[i])) {
             return false;
           }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 4) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
         }
         break;
       }
@@ -1051,9 +1166,24 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
             }
             continue;
           }
+
           if (!isdigit(tz[i])) {
             return false;
           }
+
+          if (i == 2) {
+            memcpy(buf, &tz[i - 1], 2);
+            hour = taosStr2Int8(buf, NULL, 10);
+            if (!validateHourRange(hour)) {
+              return false;
+            }
+          } else if (i == 5) {
+            memcpy(buf, &tz[i - 1], 2);
+            minute = taosStr2Int8(buf, NULL, 10);
+            if (!validateMinuteRange(hour, minute, tz[0])) {
+              return false;
+            }
+          }
         }
         break;
       }
@@ -1339,6 +1469,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = functionSetup,
     .processFunc  = topFunction,
     .finalizeFunc = topBotFinalize,
+    .combineFunc  = topCombine,
   },
   {
     .name = "bottom",
@@ -1348,7 +1479,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = bottomFunction,
-    .finalizeFunc = topBotFinalize
+    .finalizeFunc = topBotFinalize,
+    .combineFunc  = bottomCombine,
   },
   {
     .name = "spread",
@@ -1394,6 +1526,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getElapsedFuncEnv,
     .initFunc     = elapsedFunctionSetup,
     .processFunc  = elapsedFunction,
+    .finalizeFunc = elapsedFinalize,
+    .pPartialFunc = "_elapsed_partial",
+    .pMergeFunc   = "_elapsed_merge"
+  },
+  {
+    .name = "_elapsed_partial",
+    .type = FUNCTION_TYPE_ELAPSED,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .dataRequiredFunc = statisDataRequired,
+    .translateFunc = translateElapsedPartial,
+    .getEnvFunc   = getElapsedFuncEnv,
+    .initFunc     = elapsedFunctionSetup,
+    .processFunc  = elapsedFunction,
+    .finalizeFunc = elapsedPartialFinalize
+  },
+  {
+    .name = "_elapsed_merge",
+    .type = FUNCTION_TYPE_ELAPSED,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .dataRequiredFunc = statisDataRequired,
+    .translateFunc = translateElapsedMerge,
+    .getEnvFunc   = getElapsedFuncEnv,
+    .initFunc     = elapsedFunctionSetup,
+    .processFunc  = elapsedFunctionMerge,
     .finalizeFunc = elapsedFinalize
   },
   {
@@ -1478,6 +1634,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHLLFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = hllFunction,
+    .finalizeFunc = hllFinalize,
+    .pPartialFunc = "_hyperloglog_partial",
+    .pMergeFunc   = "_hyperloglog_merge"
+  },
+  {
+    .name = "_hyperloglog_partial",
+    .type = FUNCTION_TYPE_HYPERLOGLOG_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHLLPartial,
+    .getEnvFunc   = getHLLFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = hllFunction,
+    .finalizeFunc = hllPartialFinalize
+  },
+  {
+    .name = "_hyperloglog_merge",
+    .type = FUNCTION_TYPE_HYPERLOGLOG_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHLLMerge,
+    .getEnvFunc   = getHLLFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = hllFunctionMerge,
     .finalizeFunc = hllFinalize
   },
   {

source/libs/function/src/builtinsimpl.c

@@ -1389,6 +1389,18 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple
   }
 }
 
+void releaseSource(STuplePos* pPos) {
+  if (pPos->pageId == -1) {
+    return ;
+  }
+  // Todo(liuyao) relase row
+}
+
+void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) {
+  releaseSource(pDestPos);
+  *pDestPos = *pSourcePos;
+}
+
 int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) {
   SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
   SMinmaxResInfo*      pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
@@ -1400,10 +1412,12 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
     if (pSBuf->assign && 
         ( (((*(double*)&pDBuf->v) < (*(double*)&pSBuf->v)) ^ isMinFunc) || !pDBuf->assign ) ) {
       *(double*) &pDBuf->v = *(double*) &pSBuf->v;
+      replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos);
     }
   } else {
     if ( pSBuf->assign && ( ((pDBuf->v < pSBuf->v) ^ isMinFunc) || !pDBuf->assign ) ) {
       pDBuf->v = pSBuf->v;
+      replaceTupleData(&pDBuf->tuplePos, &pSBuf->tuplePos);
     }
   }
   pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
@@ -2856,7 +2870,6 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
   STopBotRes*          pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
-  pEntryInfo->complete = true;
 
   int32_t type = pCtx->input.pData[0]->info.type;
   int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
@@ -2881,6 +2894,67 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pEntryInfo->numOfRes;
 }
 
+void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type,
+    bool isTopQuery) {
+  SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
+  STopBotRes* pRes = getTopBotOutputInfo(pCtx);
+  int32_t     maxSize = pCtx->param[1].param.i;
+  STopBotResItem* pItems = pRes->pItems;
+  assert(pItems != NULL);
+
+  // not full yet
+  if (pEntryInfo->numOfRes < maxSize) {
+    STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes];
+    pItem->v = pSourceItem->v;
+    pItem->uid = pSourceItem->uid;
+    pItem->tuplePos.pageId = -1;
+    replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos);
+    pEntryInfo->numOfRes++;
+    taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn,
+                 !isTopQuery);
+  } else {  // replace the minimum value in the result
+    if ((isTopQuery && (
+        (IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i > pItems[0].v.i) ||
+        (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u > pItems[0].v.u) ||
+        (IS_FLOAT_TYPE(type) && pSourceItem->v.d > pItems[0].v.d)))
+        || (!isTopQuery && (
+        (IS_SIGNED_NUMERIC_TYPE(type) && pSourceItem->v.i < pItems[0].v.i) ||
+        (IS_UNSIGNED_NUMERIC_TYPE(type) && pSourceItem->v.u < pItems[0].v.u) ||
+        (IS_FLOAT_TYPE(type) && pSourceItem->v.d < pItems[0].v.d))
+        )) {
+      // replace the old data and the coresponding tuple data
+      STopBotResItem* pItem = &pItems[0];
+      pItem->v = pSourceItem->v;
+      pItem->uid = pSourceItem->uid;
+
+      // save the data of this tuple by over writing the old data
+      replaceTupleData(&pItem->tuplePos, &pSourceItem->tuplePos);
+      taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type,
+                     topBotResComparFn, NULL, !isTopQuery);
+    }
+  }
+}
+
+int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+  int32_t type = pDestCtx->input.pData[0]->info.type;
+  SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+  STopBotRes*          pSBuf = getTopBotOutputInfo(pSourceCtx);
+  for (int32_t i = 0; i < pSResInfo->numOfRes; i++) {
+    addResult(pDestCtx, pSBuf->pItems + i, type, true);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
+  int32_t type = pDestCtx->input.pData[0]->info.type;
+  SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
+  STopBotRes*          pSBuf = getTopBotOutputInfo(pSourceCtx);
+  for (int32_t i = 0; i < pSResInfo->numOfRes; i++) {
+    addResult(pDestCtx, pSBuf->pItems + i, type, false);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 bool getSpreadFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SSpreadInfo);
   return true;
@@ -3027,6 +3101,10 @@ int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return pResInfo->numOfRes;
 }
 
+int32_t getElapsedInfoSize() {
+  return (int32_t)sizeof(SElapsedInfo);
+}
+
 bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SElapsedInfo);
   return true;
@@ -3128,6 +3206,30 @@ _elapsed_over:
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t elapsedFunctionMerge(SqlFunctionCtx *pCtx) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData* pCol = pInput->pData[0];
+  ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
+
+  SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  int32_t start = pInput->startRowIndex;
+  char* data = colDataGetData(pCol, start);
+  SElapsedInfo* pInputInfo = (SElapsedInfo *)varDataVal(data);
+
+  pInfo->timeUnit = pInputInfo->timeUnit;
+  if (pInfo->min > pInputInfo->min) {
+    pInfo->min = pInputInfo->min;
+  }
+
+  if (pInfo->max < pInputInfo->max) {
+    pInfo->max = pInputInfo->max;
+  }
+
+  SET_VAL(GET_RES_INFO(pCtx), 1, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
   double result = (double)pInfo->max - (double)pInfo->min;
@@ -3136,6 +3238,24 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return functionFinalize(pCtx, pBlock);
 }
 
+int32_t elapsedPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  int32_t resultBytes = getElapsedInfoSize();
+  char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
+
+  memcpy(varDataVal(res), pInfo, resultBytes);
+  varDataSetLen(res, resultBytes);
+
+  int32_t          slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  colDataAppend(pCol, pBlock->info.rows, res, false);
+
+  taosMemoryFree(res);
+  return pResInfo->numOfRes;
+}
+
 int32_t getHistogramInfoSize() {
   return (int32_t)sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
 }
@@ -3411,7 +3531,6 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 }
 
 int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
-  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
   SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
   int32_t resultBytes = getHistogramInfoSize();
   char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
@@ -3428,6 +3547,10 @@ int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return 1;
 }
 
+int32_t getHLLInfoSize() {
+  return (int32_t)sizeof(SHLLInfo);
+}
+
 bool getHLLFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHLLInfo);
   return true;
@@ -3553,6 +3676,27 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t hllFunctionMerge(SqlFunctionCtx *pCtx) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData* pCol = pInput->pData[0];
+  ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
+
+  SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  int32_t start = pInput->startRowIndex;
+  char* data = colDataGetData(pCol, start);
+  SHLLInfo* pInputInfo = (SHLLInfo *)varDataVal(data);
+
+  for (int32_t k = 0; k < HLL_BUCKETS; ++k) {
+    if (pInfo->buckets[k] < pInputInfo->buckets[k]) {
+      pInfo->buckets[k] = pInputInfo->buckets[k];
+    }
+  }
+
+  SET_VAL(GET_RES_INFO(pCtx), 1, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx);
 
@@ -3565,6 +3709,24 @@ int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return functionFinalize(pCtx, pBlock);
 }
 
+int32_t hllPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  int32_t resultBytes = getHLLInfoSize();
+  char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
+
+  memcpy(varDataVal(res), pInfo, resultBytes);
+  varDataSetLen(res, resultBytes);
+
+  int32_t          slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  colDataAppend(pCol, pBlock->info.rows, res, false);
+
+  taosMemoryFree(res);
+  return pResInfo->numOfRes;
+}
+
 bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SStateInfo);
   return true;

source/libs/index/inc/indexCache.h

@@ -62,25 +62,25 @@ typedef struct CacheTerm {
 } CacheTerm;
 //
 
-IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type);
+IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type);
 
-void indexCacheForceToMerge(void* cache);
+void idxCacheForceToMerge(void* cache);
-void indexCacheDestroy(void* cache);
+void idxCacheDestroy(void* cache);
-void indexCacheBroadcast(void* cache);
+void idxCacheBroadcast(void* cache);
-void indexCacheWait(void* cache);
+void idxCacheWait(void* cache);
 
-Iterate* indexCacheIteratorCreate(IndexCache* cache);
+Iterate* idxCacheIteratorCreate(IndexCache* cache);
 void     idxCacheIteratorDestroy(Iterate* iiter);
 
-int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
+int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid);
 
 // int indexCacheGet(void *cache, uint64_t *rst);
-int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s);
+int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s);
 
-void indexCacheRef(IndexCache* cache);
+void idxCacheRef(IndexCache* cache);
-void indexCacheUnRef(IndexCache* cache);
+void idxCacheUnRef(IndexCache* cache);
 
-void indexCacheDebug(IndexCache* cache);
+void idxCacheDebug(IndexCache* cache);
 
 void idxCacheDestroyImm(IndexCache* cache);
 #ifdef __cplusplus

source/libs/index/inc/indexComm.h

@@ -34,11 +34,11 @@ typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
 typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
 
-__compar_fn_t indexGetCompar(int8_t type);
+__compar_fn_t idxGetCompar(int8_t type);
 TExeCond      tCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b, int8_t dType);
 TExeCond      tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
-_cache_range_compare indexGetCompare(RangeType ty);
+_cache_range_compare idxGetCompare(RangeType ty);
 
 int32_t idxConvertData(void* src, int8_t type, void** dst);
 int32_t idxConvertDataToStr(void* src, int8_t type, void** dst);

source/libs/index/inc/indexInt.h

@@ -133,24 +133,24 @@ typedef struct TFileCacheKey {
 } ICacheKey;
 int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit);
 
-int64_t indexAddRef(void* p);
+int64_t idxAddRef(void* p);
-int32_t indexRemoveRef(int64_t ref);
+int32_t idxRemoveRef(int64_t ref);
-void    indexAcquireRef(int64_t ref);
+void    idxAcquireRef(int64_t ref);
-void    indexReleaseRef(int64_t ref);
+void    idxReleaseRef(int64_t ref);
 
-int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
+int32_t idxSerialCacheKey(ICacheKey* key, char* buf);
 // int32_t indexSerialKey(ICacheKey* key, char* buf);
 // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 
-#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
+#define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
 
-#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
+#define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F)
 
-#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
+#define IDX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
-  do {                                       \
+  do {                                     \
-    uint8_t oldTy = ty;                      \
+    uint8_t oldTy = ty;                    \
-    ty = (ty >> 4) | exTy;                   \
+    ty = (ty >> 4) | exTy;                 \
-    ty = (ty << 4) | oldTy;                  \
+    ty = (ty << 4) | oldTy;                \
   } while (0)
 
 #ifdef __cplusplus

source/libs/index/inc/indexTfile.h

@@ -117,10 +117,10 @@ int          tfileWriterPut(TFileWriter* tw, void* data, bool order);
 int          tfileWriterFinish(TFileWriter* tw);
 
 //
-IndexTFile* indexTFileCreate(const char* path);
+IndexTFile* idxTFileCreate(const char* path);
-void        indexTFileDestroy(IndexTFile* tfile);
+void        idxTFileDestroy(IndexTFile* tfile);
-int         indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid);
+int         idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid);
-int         indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr);
+int         idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr);
 
 Iterate* tfileIteratorCreate(TFileReader* reader);
 void     tfileIteratorDestroy(Iterate* iterator);

source/libs/index/src/index.c

@@ -90,7 +90,7 @@ static void idxMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateV
 // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 // int32_t        indexSerialKey(ICacheKey* key, char* buf);
 
-static void indexPost(void* idx) {
+static void idxPost(void* idx) {
   SIndex* pIdx = idx;
   tsem_post(&pIdx->sem);
 }
@@ -106,8 +106,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
     return -1;
   }
 
-  // sIdx->cache = (void*)indexCacheCreate(sIdx);
+  // sIdx->cache = (void*)idxCacheCreate(sIdx);
-  sIdx->tindex = indexTFileCreate(path);
+  sIdx->tindex = idxTFileCreate(path);
   if (sIdx->tindex == NULL) {
     goto END;
   }
@@ -118,8 +118,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
   taosThreadMutexInit(&sIdx->mtx, NULL);
   tsem_init(&sIdx->sem, 0, 0);
 
-  sIdx->refId = indexAddRef(sIdx);
+  sIdx->refId = idxAddRef(sIdx);
-  indexAcquireRef(sIdx->refId);
+  idxAcquireRef(sIdx->refId);
 
   *index = sIdx;
   return 0;
@@ -136,7 +136,7 @@ void indexDestroy(void* handle) {
   SIndex* sIdx = handle;
   taosThreadMutexDestroy(&sIdx->mtx);
   tsem_destroy(&sIdx->sem);
-  indexTFileDestroy(sIdx->tindex);
+  idxTFileDestroy(sIdx->tindex);
   taosMemoryFree(sIdx->path);
   taosMemoryFree(sIdx);
   return;
@@ -147,33 +147,33 @@ void indexClose(SIndex* sIdx) {
     void* iter = taosHashIterate(sIdx->colObj, NULL);
     while (iter) {
       IndexCache** pCache = iter;
-      indexCacheForceToMerge((void*)(*pCache));
+      idxCacheForceToMerge((void*)(*pCache));
       indexInfo("%s wait to merge", (*pCache)->colName);
       indexWait((void*)(sIdx));
       indexInfo("%s finish to wait", (*pCache)->colName);
       iter = taosHashIterate(sIdx->colObj, iter);
-      indexCacheUnRef(*pCache);
+      idxCacheUnRef(*pCache);
     }
     taosHashCleanup(sIdx->colObj);
     sIdx->colObj = NULL;
   }
-  indexReleaseRef(sIdx->refId);
+  idxReleaseRef(sIdx->refId);
-  indexRemoveRef(sIdx->refId);
+  idxRemoveRef(sIdx->refId);
 }
-int64_t indexAddRef(void* p) {
+int64_t idxAddRef(void* p) {
   // impl
   return taosAddRef(indexRefMgt, p);
 }
-int32_t indexRemoveRef(int64_t ref) {
+int32_t idxRemoveRef(int64_t ref) {
   // impl later
   return taosRemoveRef(indexRefMgt, ref);
 }
 
-void indexAcquireRef(int64_t ref) {
+void idxAcquireRef(int64_t ref) {
   // impl
   taosAcquireRef(indexRefMgt, ref);
 }
-void indexReleaseRef(int64_t ref) {
+void idxReleaseRef(int64_t ref) {
   // impl
   taosReleaseRef(indexRefMgt, ref);
 }
@@ -186,11 +186,11 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
 
     char      buf[128] = {0};
     ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
-    int32_t   sz = indexSerialCacheKey(&key, buf);
+    int32_t   sz = idxSerialCacheKey(&key, buf);
 
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     if (cache == NULL) {
-      IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType);
+      IndexCache* pCache = idxCacheCreate(index, p->suid, p->colName, p->colType);
       taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*));
     }
   }
@@ -201,12 +201,12 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
 
     char      buf[128] = {0};
     ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
-    int32_t   sz = indexSerialCacheKey(&key, buf);
+    int32_t   sz = idxSerialCacheKey(&key, buf);
     indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
 
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
-    int ret = indexCachePut(*cache, p, uid);
+    int ret = idxCachePut(*cache, p, uid);
     if (ret != 0) {
       return ret;
     }
@@ -289,7 +289,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
   tm->nColName = nColName;
 
   char*   buf = NULL;
-  int32_t len = idxConvertDataToStr((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
+  int32_t len = idxConvertDataToStr((void*)colVal, IDX_TYPE_GET_TYPE(colType), (void**)&buf);
   assert(len != -1);
 
   tm->colVal = buf;
@@ -331,7 +331,7 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result)
   ICacheKey key = {
       .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
   indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
-  int32_t sz = indexSerialCacheKey(&key, buf);
+  int32_t sz = idxSerialCacheKey(&key, buf);
 
   taosThreadMutexLock(&sIdx->mtx);
   IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz);
@@ -345,14 +345,14 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result)
   int64_t st = taosGetTimestampUs();
 
   SIdxTRslt* tr = idxTRsltCreate();
-  if (0 == indexCacheSearch(cache, query, tr, &s)) {
+  if (0 == idxCacheSearch(cache, query, tr, &s)) {
     if (s == kTypeDeletion) {
       indexInfo("col: %s already drop by", term->colName);
       // coloum already drop by other oper, no need to query tindex
       return 0;
     } else {
       st = taosGetTimestampUs();
-      if (0 != indexTFileSearch(sIdx->tindex, query, tr)) {
+      if (0 != idxTFileSearch(sIdx->tindex, query, tr)) {
         indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal);
         goto END;
       }
@@ -465,23 +465,23 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
 
   IndexCache* pCache = (IndexCache*)cache;
 
-  while (quit && atomic_load_32(&pCache->merging) == 1) {
+  while (quit && atomic_load_32(&pCache->merging) == 1)
-  }
+    ;
   TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
   if (pReader == NULL) {
     indexWarn("empty tfile reader found");
   }
   // handle flush
-  Iterate* cacheIter = indexCacheIteratorCreate(pCache);
+  Iterate* cacheIter = idxCacheIteratorCreate(pCache);
   if (cacheIter == NULL) {
     indexError("%p immtable is empty, ignore merge opera", pCache);
     idxCacheDestroyImm(pCache);
     tfileReaderUnRef(pReader);
     atomic_store_32(&pCache->merging, 0);
     if (quit) {
-      indexPost(sIdx);
+      idxPost(sIdx);
     }
-    indexReleaseRef(sIdx->refId);
+    idxReleaseRef(sIdx->refId);
     return 0;
   }
 
@@ -532,7 +532,7 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
   tfileIteratorDestroy(tfileIter);
 
   tfileReaderUnRef(pReader);
-  indexCacheUnRef(pCache);
+  idxCacheUnRef(pCache);
 
   int64_t cost = taosGetTimestampUs() - st;
   if (ret != 0) {
@@ -542,9 +542,9 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) {
   }
   atomic_store_32(&pCache->merging, 0);
   if (quit) {
-    indexPost(sIdx);
+    idxPost(sIdx);
   }
-  indexReleaseRef(sIdx->refId);
+  idxReleaseRef(sIdx->refId);
 
   return ret;
 }
@@ -561,7 +561,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) {
   value->colVal = NULL;
 }
 
-static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) {
+static int64_t idxGetAvailableVer(SIndex* sIdx, IndexCache* cache) {
   ICacheKey key = {.suid = cache->suid, .colName = cache->colName, .nColName = strlen(cache->colName)};
   int64_t   ver = CACHE_VERSION(cache);
 
@@ -579,7 +579,7 @@ static int64_t indexGetAvaialbleVer(SIndex* sIdx, IndexCache* cache) {
   return ver;
 }
 static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
-  int64_t version = indexGetAvaialbleVer(sIdx, cache);
+  int64_t version = idxGetAvailableVer(sIdx, cache);
   indexInfo("file name version: %" PRId64 "", version);
   uint8_t colType = cache->type;
 
@@ -620,8 +620,8 @@ END:
   return -1;
 }
 
-int32_t indexSerialCacheKey(ICacheKey* key, char* buf) {
+int32_t idxSerialCacheKey(ICacheKey* key, char* buf) {
-  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON);
+  bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON);
 
   char* p = buf;
   char  tbuf[65] = {0};

source/libs/index/src/indexCache.c

@@ -68,7 +68,7 @@ static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRsl
      cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON,
      cacheSearchRange_JSON}};
 
-static void doMergeWork(SSchedMsg* msg);
+static void idxDoMergeWork(SSchedMsg* msg);
 static bool idxCacheIteratorNext(Iterate* itera);
 
 static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
@@ -127,7 +127,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt*
   MemTable*   mem = cache;
   IndexCache* pCache = mem->pCache;
 
-  _cache_range_compare cmpFn = indexGetCompare(type);
+  _cache_range_compare cmpFn = idxGetCompare(type);
 
   CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
   pCt->colVal = term->colVal;
@@ -187,7 +187,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
   pCt->version = atomic_load_64(&pCache->version);
 
   char* exBuf = NULL;
-  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+  if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
     exBuf = idxPackJsonData(term);
     pCt->colVal = exBuf;
   }
@@ -257,7 +257,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
   if (cache == NULL) {
     return 0;
   }
-  _cache_range_compare cmpFn = indexGetCompare(type);
+  _cache_range_compare cmpFn = idxGetCompare(type);
 
   MemTable*   mem = cache;
   IndexCache* pCache = mem->pCache;
@@ -266,7 +266,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTR
   pCt->colVal = term->colVal;
   pCt->version = atomic_load_64(&pCache->version);
 
-  int8_t dType = INDEX_TYPE_GET_TYPE(term->colType);
+  int8_t dType = IDX_TYPE_GET_TYPE(term->colType);
   int    skip = 0;
   char*  exBuf = NULL;
   if (type == CONTAINS) {
@@ -331,9 +331,9 @@ static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTRslt* tr, ST
   // impl later
   return 0;
 }
-static IterateValue* indexCacheIteratorGetValue(Iterate* iter);
+static IterateValue* idxCacheIteratorGetValue(Iterate* iter);
 
-IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) {
+IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) {
   IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache));
   if (cache == NULL) {
     indexError("failed to create index cache");
@@ -342,7 +342,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
 
   cache->mem = idxInternalCacheCreate(type);
   cache->mem->pCache = cache;
-  cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
+  cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
   cache->type = type;
   cache->index = idx;
   cache->version = 0;
@@ -352,13 +352,13 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
   taosThreadMutexInit(&cache->mtx, NULL);
   taosThreadCondInit(&cache->finished, NULL);
 
-  indexCacheRef(cache);
+  idxCacheRef(cache);
   if (idx != NULL) {
-    indexAcquireRef(idx->refId);
+    idxAcquireRef(idx->refId);
   }
   return cache;
 }
-void indexCacheDebug(IndexCache* cache) {
+void idxCacheDebug(IndexCache* cache) {
   MemTable* tbl = NULL;
 
   taosThreadMutexLock(&cache->mtx);
@@ -405,7 +405,7 @@ void indexCacheDebug(IndexCache* cache) {
   }
 }
 
-void indexCacheDestroySkiplist(SSkipList* slt) {
+void idxCacheDestroySkiplist(SSkipList* slt) {
   SSkipListIterator* iter = tSkipListCreateIter(slt);
   while (iter != NULL && tSkipListIterNext(iter)) {
     SSkipListNode* node = tSkipListIterGet(iter);
@@ -418,11 +418,11 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
   tSkipListDestroyIter(iter);
   tSkipListDestroy(slt);
 }
-void indexCacheBroadcast(void* cache) {
+void idxCacheBroadcast(void* cache) {
   IndexCache* pCache = cache;
   taosThreadCondBroadcast(&pCache->finished);
 }
-void indexCacheWait(void* cache) {
+void idxCacheWait(void* cache) {
   IndexCache* pCache = cache;
   taosThreadCondWait(&pCache->finished, &pCache->mtx);
 }
@@ -435,14 +435,14 @@ void idxCacheDestroyImm(IndexCache* cache) {
 
   tbl = cache->imm;
   cache->imm = NULL;  // or throw int bg thread
-  indexCacheBroadcast(cache);
+  idxCacheBroadcast(cache);
 
   taosThreadMutexUnlock(&cache->mtx);
 
   idxMemUnRef(tbl);
   idxMemUnRef(tbl);
 }
-void indexCacheDestroy(void* cache) {
+void idxCacheDestroy(void* cache) {
   IndexCache* pCache = cache;
   if (pCache == NULL) {
     return;
@@ -455,12 +455,12 @@ void indexCacheDestroy(void* cache) {
   taosThreadMutexDestroy(&pCache->mtx);
   taosThreadCondDestroy(&pCache->finished);
   if (pCache->index != NULL) {
-    indexReleaseRef(((SIndex*)pCache->index)->refId);
+    idxReleaseRef(((SIndex*)pCache->index)->refId);
   }
   taosMemoryFree(pCache);
 }
 
-Iterate* indexCacheIteratorCreate(IndexCache* cache) {
+Iterate* idxCacheIteratorCreate(IndexCache* cache) {
   if (cache->imm == NULL) {
     return NULL;
   }
@@ -477,7 +477,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   iiter->val.colVal = NULL;
   iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL;
   iiter->next = idxCacheIteratorNext;
-  iiter->getValue = indexCacheIteratorGetValue;
+  iiter->getValue = idxCacheIteratorGetValue;
 
   taosThreadMutexUnlock(&cache->mtx);
 
@@ -492,30 +492,30 @@ void idxCacheIteratorDestroy(Iterate* iter) {
   taosMemoryFree(iter);
 }
 
-int indexCacheSchedToMerge(IndexCache* pCache, bool notify) {
+int idxCacheSchedToMerge(IndexCache* pCache, bool notify) {
   SSchedMsg schedMsg = {0};
-  schedMsg.fp = doMergeWork;
+  schedMsg.fp = idxDoMergeWork;
   schedMsg.ahandle = pCache;
   if (notify) {
     schedMsg.thandle = taosMemoryMalloc(1);
   }
   schedMsg.msg = NULL;
-  indexAcquireRef(pCache->index->refId);
+  idxAcquireRef(pCache->index->refId);
   taosScheduleTask(indexQhandle, &schedMsg);
   return 0;
 }
 
-static void indexCacheMakeRoomForWrite(IndexCache* cache) {
+static void idxCacheMakeRoomForWrite(IndexCache* cache) {
   while (true) {
     if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) {
       break;
     } else if (cache->imm != NULL) {
       // TODO: wake up by condition variable
-      indexCacheWait(cache);
+      idxCacheWait(cache);
     } else {
       bool quit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false;
 
-      indexCacheRef(cache);
+      idxCacheRef(cache);
       cache->imm = cache->mem;
       cache->mem = idxInternalCacheCreate(cache->type);
       cache->mem->pCache = cache;
@@ -525,18 +525,18 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
       }
       // sched to merge
       // unref cache in bgwork
-      indexCacheSchedToMerge(cache, quit);
+      idxCacheSchedToMerge(cache, quit);
     }
   }
 }
-int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
+int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
   if (cache == NULL) {
     return -1;
   }
-  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
+  bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
 
   IndexCache* pCache = cache;
-  indexCacheRef(pCache);
+  idxCacheRef(pCache);
   // encode data
   CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm));
   if (cache == NULL) {
@@ -559,7 +559,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
 
   taosThreadMutexLock(&pCache->mtx);
   pCache->occupiedMem += estimate;
-  indexCacheMakeRoomForWrite(pCache);
+  idxCacheMakeRoomForWrite(pCache);
   MemTable* tbl = pCache->mem;
   idxMemRef(tbl);
   tSkipListPut(tbl->mem, (char*)ct);
@@ -567,29 +567,29 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
 
   taosThreadMutexUnlock(&pCache->mtx);
 
-  indexCacheUnRef(pCache);
+  idxCacheUnRef(pCache);
   return 0;
   // encode end
 }
-void indexCacheForceToMerge(void* cache) {
+void idxCacheForceToMerge(void* cache) {
   IndexCache* pCache = cache;
-  indexCacheRef(pCache);
+  idxCacheRef(pCache);
   taosThreadMutexLock(&pCache->mtx);
 
   indexInfo("%p is forced to merge into tfile", pCache);
   pCache->occupiedMem += MEM_SIGNAL_QUIT;
-  indexCacheMakeRoomForWrite(pCache);
+  idxCacheMakeRoomForWrite(pCache);
 
   taosThreadMutexUnlock(&pCache->mtx);
-  indexCacheUnRef(pCache);
+  idxCacheUnRef(pCache);
   return;
 }
-int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
+int idxCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
   IndexCache* pCache = cache;
   return 0;
 }
 
-static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) {
+static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) {
   if (mem == NULL) {
     return 0;
   }
@@ -597,13 +597,13 @@ static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* t
   SIndexTerm*     term = query->term;
   EIndexQueryType qtype = query->qType;
 
-  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+  if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
     return cacheSearch[1][qtype](mem, term, tr, s);
   } else {
     return cacheSearch[0][qtype](mem, term, tr, s);
   }
 }
-int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) {
+int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) {
   int64_t st = taosGetTimestampUs();
   if (cache == NULL) {
     return 0;
@@ -618,10 +618,10 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
   idxMemRef(imm);
   taosThreadMutexUnlock(&pCache->mtx);
 
-  int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0;
+  int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0;
   if (ret == 0 && *s != kTypeDeletion) {
     // continue search in imm
-    ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0;
+    ret = (imm && imm->mem) ? idxQueryMem(imm, query, result, s) : 0;
   }
 
   idxMemUnRef(mem);
@@ -631,20 +631,20 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STe
   return ret;
 }
 
-void indexCacheRef(IndexCache* cache) {
+void idxCacheRef(IndexCache* cache) {
   if (cache == NULL) {
     return;
   }
   int ref = T_REF_INC(cache);
   UNUSED(ref);
 }
-void indexCacheUnRef(IndexCache* cache) {
+void idxCacheUnRef(IndexCache* cache) {
   if (cache == NULL) {
     return;
   }
   int ref = T_REF_DEC(cache);
   if (ref == 0) {
-    indexCacheDestroy(cache);
+    idxCacheDestroy(cache);
   }
 }
 
@@ -662,7 +662,7 @@ void idxMemUnRef(MemTable* tbl) {
   int ref = T_REF_DEC(tbl);
   if (ref == 0) {
     SSkipList* slt = tbl->mem;
-    indexCacheDestroySkiplist(slt);
+    idxCacheDestroySkiplist(slt);
     taosMemoryFree(tbl);
   }
 }
@@ -693,15 +693,15 @@ static int32_t idxCacheTermCompare(const void* l, const void* r) {
   return cmp;
 }
 
-static int indexFindCh(char* a, char c) {
+static int idxFindCh(char* a, char c) {
   char* p = a;
   while (*p != 0 && *p++ != c) {
   }
   return p - a;
 }
 static int idxCacheJsonTermCompareImpl(char* a, char* b) {
-  // int alen = indexFindCh(a, '&');
+  // int alen = idxFindCh(a, '&');
-  // int blen = indexFindCh(b, '&');
+  // int blen = idxFindCh(b, '&');
 
   // int cmp = strncmp(a, b, MIN(alen, blen));
   // if (cmp == 0) {
@@ -730,9 +730,9 @@ static int32_t idxCacheJsonTermCompare(const void* l, const void* r) {
   return cmp;
 }
 static MemTable* idxInternalCacheCreate(int8_t type) {
-  int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY;
+  int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY;
   int32_t (*cmpFn)(const void* l, const void* r) =
-      INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare;
+      IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare;
 
   MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable));
   idxMemRef(tbl);
@@ -742,7 +742,7 @@ static MemTable* idxInternalCacheCreate(int8_t type) {
   return tbl;
 }
 
-static void doMergeWork(SSchedMsg* msg) {
+static void idxDoMergeWork(SSchedMsg* msg) {
   IndexCache* pCache = msg->ahandle;
   SIndex*     sidx = (SIndex*)pCache->index;
 
@@ -771,7 +771,7 @@ static bool idxCacheIteratorNext(Iterate* itera) {
   return next;
 }
 
-static IterateValue* indexCacheIteratorGetValue(Iterate* iter) {
+static IterateValue* idxCacheIteratorGetValue(Iterate* iter) {
   // opt later
   return &iter->val;
 }

source/libs/index/src/indexComm.c

@@ -75,35 +75,35 @@ char* idxInt2str(int64_t val, char* dst, int radix) {
     ;
   return dst - 1;
 }
-__compar_fn_t indexGetCompar(int8_t type) {
+__compar_fn_t idxGetCompar(int8_t type) {
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
     return (__compar_fn_t)strcmp;
   }
   return getComparFunc(type, 0);
 }
 static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_LESS_THAN, a, b, type);
 }
 static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_LESS_EQUAL, a, b, type);
 }
 static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_GREATER_THAN, a, b, type);
 }
 static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
 
 static TExeCond tCompareContains(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_TERM, a, b, type);
 }
 static TExeCond tCompareEqual(void* a, void* b, int8_t type) {
-  __compar_fn_t func = indexGetCompar(type);
+  __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_TERM, a, b, type);
 }
 TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
@@ -205,14 +205,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
 static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {
     tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains, tCompareEqual};
 
-_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
+_cache_range_compare idxGetCompare(RangeType ty) { return rangeCompare[ty]; }
 
 char* idxPackJsonData(SIndexTerm* itm) {
   /*
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
    */
-  uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
+  uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType);
 
   int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
   char*   buf = (char*)taosMemoryCalloc(1, sz);
@@ -240,7 +240,7 @@ char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
    */
-  uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
+  uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType);
 
   int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
   char*   buf = (char*)taosMemoryCalloc(1, sz);
@@ -267,7 +267,7 @@ char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) {
    * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
    * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
    */
-  uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
+  uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType);
 
   int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
   char*   buf = (char*)taosMemoryCalloc(1, sz);

source/libs/index/src/indexFilter.c

@@ -318,7 +318,7 @@ int sifLessThan(void *a, void *b, int16_t dtype) {
 }
 int sifEqual(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
-  //__compar_fn_t func = indexGetCompar(dtype);
+  //__compar_fn_t func = idxGetCompar(dtype);
   return (int)tDoCompare(func, QUERY_TERM, a, b);
 }
 static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {

source/libs/index/src/indexJson.c

@@ -30,7 +30,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
     } else {
       p->colType = TSDB_DATA_TYPE_DOUBLE;
     }
-    INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
+    IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
   }
   // handle put
   return indexPut(index, terms, uid);
@@ -48,7 +48,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res
     } else {
       p->colType = TSDB_DATA_TYPE_DOUBLE;
     }
-    INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
+    IDX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
   }
   // handle search
   return indexSearch(index, tq, result);

source/libs/index/src/indexTfile.c

@@ -118,7 +118,7 @@ TFileCache* tfileCacheCreate(const char* path) {
     ICacheKey    key = {.suid = header->suid, .colName = header->colName, .nColName = (int32_t)strlen(header->colName)};
 
     char    buf[128] = {0};
-    int32_t sz = indexSerialCacheKey(&key, buf);
+    int32_t sz = idxSerialCacheKey(&key, buf);
     assert(sz < sizeof(buf));
     taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
     tfileReaderRef(reader);
@@ -149,7 +149,7 @@ void tfileCacheDestroy(TFileCache* tcache) {
 
 TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
   char    buf[128] = {0};
-  int32_t sz = indexSerialCacheKey(key, buf);
+  int32_t sz = idxSerialCacheKey(key, buf);
   assert(sz < sizeof(buf));
   TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
   if (reader == NULL || *reader == NULL) {
@@ -161,7 +161,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
 }
 void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
   char    buf[128] = {0};
-  int32_t sz = indexSerialCacheKey(key, buf);
+  int32_t sz = idxSerialCacheKey(key, buf);
   // remove last version index reader
   TFileReader** p = taosHashGet(tcache->tableCache, buf, sz);
   if (p != NULL && *p != NULL) {
@@ -281,7 +281,7 @@ static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return 0;
 }
 static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  bool hasJson = IDX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
 
   int      ret = 0;
   char*    p = tem->colVal;
@@ -305,7 +305,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
   int                  ret = 0;
   char*                p = tem->colVal;
   int                  skip = 0;
-  _cache_range_compare cmpFn = indexGetCompare(type);
+  _cache_range_compare cmpFn = idxGetCompare(type);
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
@@ -431,7 +431,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
     p = idxPackJsonDataPrefix(tem, &skip);
   }
 
-  _cache_range_compare cmpFn = indexGetCompare(ctype);
+  _cache_range_compare cmpFn = idxGetCompare(ctype);
 
   SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
 
@@ -457,7 +457,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
       } else if (0 != strncmp(ch, p, skip)) {
         continue;
       }
-      cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
+      cond = cmpFn(ch + skip, tem->colVal, IDX_TYPE_GET_TYPE(tem->colType));
     }
     if (MATCH == cond) {
       tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
@@ -476,7 +476,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr
   SIndexTerm*     term = query->term;
   EIndexQueryType qtype = query->qType;
   int             ret = 0;
-  if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+  if (IDX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
     ret = tfSearch[1][qtype](reader, term, tr);
   } else {
     ret = tfSearch[0][qtype](reader, term, tr);
@@ -536,7 +536,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
     __compar_fn_t fn;
 
     int8_t colType = tw->header.colType;
-    colType = INDEX_TYPE_GET_TYPE(colType);
+    colType = IDX_TYPE_GET_TYPE(colType);
     if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
       fn = tfileStrCompare;
     } else {
@@ -620,7 +620,7 @@ void tfileWriterDestroy(TFileWriter* tw) {
   taosMemoryFree(tw);
 }
 
-IndexTFile* indexTFileCreate(const char* path) {
+IndexTFile* idxTFileCreate(const char* path) {
   TFileCache* cache = tfileCacheCreate(path);
   if (cache == NULL) {
     return NULL;
@@ -635,7 +635,7 @@ IndexTFile* indexTFileCreate(const char* path) {
   tfile->cache = cache;
   return tfile;
 }
-void indexTFileDestroy(IndexTFile* tfile) {
+void idxTFileDestroy(IndexTFile* tfile) {
   if (tfile == NULL) {
     return;
   }
@@ -644,7 +644,7 @@ void indexTFileDestroy(IndexTFile* tfile) {
   taosMemoryFree(tfile);
 }
 
-int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) {
+int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) {
   int ret = -1;
   if (tfile == NULL) {
     return ret;
@@ -667,7 +667,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) {
 
   return tfileReaderSearch(reader, query, result);
 }
-int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
+int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
   // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName =
   // term->nColName, .version = 1};
 
@@ -845,7 +845,7 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
   TFileHeader* header = &write->header;
   uint8_t      colType = header->colType;
 
-  colType = INDEX_TYPE_GET_TYPE(colType);
+  colType = IDX_TYPE_GET_TYPE(colType);
   FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal));
   if (fstBuilderInsert(write->fb, key, tval->offset)) {
     fstSliceDestroy(&key);

source/libs/index/test/indexTests.cc

@@ -521,10 +521,10 @@ class CacheObj {
  public:
   CacheObj() {
     // TODO
-    cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY);
+    cache = idxCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY);
   }
   int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
-    int ret = indexCachePut(cache, term, uid);
+    int ret = idxCachePut(cache, term, uid);
     if (ret != 0) {
       //
       std::cout << "failed to put into cache: " << ret << std::endl;
@@ -533,12 +533,12 @@ class CacheObj {
   }
   void Debug() {
     //
-    indexCacheDebug(cache);
+    idxCacheDebug(cache);
   }
   int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
     SIdxTRslt* tr = idxTRsltCreate();
 
-    int ret = indexCacheSearch(cache, query, tr, s);
+    int ret = idxCacheSearch(cache, query, tr, s);
     idxTRsltMergeTo(tr, result);
     idxTRsltDestroy(tr);
 
@@ -549,7 +549,7 @@ class CacheObj {
   }
   ~CacheObj() {
     // TODO
-    indexCacheDestroy(cache);
+    idxCacheDestroy(cache);
   }
 
  private:

source/libs/planner/src/planPhysiCreater.c

@@ -1186,6 +1186,7 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   pExchange->srcGroupId = pMerge->srcGroupId;
+  pExchange->singleChannel = true;
   pExchange->node.pParent = (SPhysiNode*)pMerge;
   pExchange->node.pOutputDataBlockDesc = nodesCloneNode(pMerge->node.pOutputDataBlockDesc);
   if (NULL == pExchange->node.pOutputDataBlockDesc) {

source/libs/scalar/src/sclvector.c

@@ -348,7 +348,7 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn
   int32_t outputMaxLen = (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
 
   char* t = taosMemoryCalloc(1, outputMaxLen);
-  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), outputMaxLen, &len);
+  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len);
   varDataSetLen(t, len);
 
   colDataAppend(pOut->columnData, rowIndex, t, false);

source/libs/stream/src/streamData.c

@@ -53,6 +53,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock
     SSDataBlock*       pDataBlock = taosArrayGet(pArray, i);
     blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
     // TODO: refactor
+    pDataBlock->info.type = pRetrieve->streamBlockType;
     pDataBlock->info.childId = pReq->sourceChildId;
   }
   pData->blocks = pArray;

source/libs/stream/src/streamDispatch.c

@@ -72,6 +72,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
   pRetrieve->precision = TSDB_DEFAULT_PRECISION;
   pRetrieve->compressed = 0;
   pRetrieve->completed = 1;
+  pRetrieve->streamBlockType = pBlock->info.type;
   pRetrieve->numOfRows = htonl(pBlock->info.rows);
   pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
 

source/libs/sync/inc/syncAppendEntries.h

@@ -93,6 +93,7 @@ extern "C" {
 //       /\ UNCHANGED <<candidateVars, leaderVars>>
 //
 int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncAppendEntriesReply.h

@@ -41,6 +41,7 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, candidateVars, logVars, elections>>
 //
 int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncElection.h

@@ -39,6 +39,8 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode);
+int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode);
+
 int32_t syncNodeElect(SSyncNode* pSyncNode);
 int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg);
 

source/libs/sync/inc/syncIO.h

@@ -57,6 +57,9 @@ typedef struct SSyncIO {
   int32_t (*FpOnSyncAppendEntriesReply)(SSyncNode *pSyncNode, SyncAppendEntriesReply *pMsg);
   int32_t (*FpOnSyncTimeout)(SSyncNode *pSyncNode, SyncTimeout *pMsg);
 
+  int32_t (*FpOnSyncSnapshotSend)(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg);
+  int32_t (*FpOnSyncSnapshotRsp)(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg);
+
   int8_t isStart;
 
 } SSyncIO;

source/libs/sync/inc/syncIndexMgr.h

@@ -30,6 +30,7 @@ extern "C" {
 typedef struct SSyncIndexMgr {
   SRaftId (*replicas)[TSDB_MAX_REPLICA];
   SyncIndex  index[TSDB_MAX_REPLICA];
+  SyncTerm   privateTerm[TSDB_MAX_REPLICA];  // for advanced function
   int32_t    replicaNum;
   SSyncNode *pSyncNode;
 } SSyncIndexMgr;
@@ -43,6 +44,9 @@ SyncIndex      syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId
 cJSON *        syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr);
 char *         syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr);
 
+// void     syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term);
+// SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
+
 // for debug -------------------
 void syncIndexMgrPrint(SSyncIndexMgr *pObj);
 void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj);

source/libs/sync/inc/syncInt.h

@@ -58,6 +58,8 @@ typedef struct SSyncRespMgr           SSyncRespMgr;
 typedef struct SSyncSnapshotSender    SSyncSnapshotSender;
 typedef struct SSyncSnapshotReceiver  SSyncSnapshotReceiver;
 
+extern bool gRaftDetailLog;
+
 typedef struct SSyncNode {
   // init by SSyncInfo
   SyncGroupId vgId;
@@ -137,24 +139,27 @@ typedef struct SSyncNode {
   uint64_t          heartbeatTimerCounter;
 
   // callback
-  int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg);
+  FpOnPingCb               FpOnPing;
-  int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg);
+  FpOnPingReplyCb          FpOnPingReply;
-  int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg);
+  FpOnClientRequestCb      FpOnClientRequest;
-  int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg);
+  FpOnTimeoutCb            FpOnTimeout;
-  int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+  FpOnRequestVoteCb        FpOnRequestVote;
-  int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg);
+  FpOnRequestVoteReplyCb   FpOnRequestVoteReply;
-  int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+  FpOnAppendEntriesCb      FpOnAppendEntries;
-  int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg);
+  FpOnAppendEntriesReplyCb FpOnAppendEntriesReply;
+  FpOnSnapshotSendCb       FpOnSnapshotSend;
+  FpOnSnapshotRspCb        FpOnSnapshotRsp;
 
   // tools
   SSyncRespMgr* pSyncRespMgr;
 
   // restore state
-  // sem_t      restoreSem;
+  bool restoreFinish;
-  bool                   restoreFinish;
+  // SSnapshot*             pSnapshot;
-  SSnapshot*             pSnapshot;
+  SSyncSnapshotSender*   senders[TSDB_MAX_REPLICA];
-  SSyncSnapshotSender*   pSender;
+  SSyncSnapshotReceiver* pNewNodeReceiver;
-  SSyncSnapshotReceiver* pReceiver;
+
+  SSnapshotMeta sMeta;
 
 } SSyncNode;
 
@@ -164,6 +169,9 @@ void       syncNodeStart(SSyncNode* pSyncNode);
 void       syncNodeStartStandBy(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
+// option
+bool syncNodeSnapshotEnable(SSyncNode* pSyncNode);
+
 // ping --------------
 int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg);
 int32_t syncNodePingSelf(SSyncNode* pSyncNode);
@@ -205,6 +213,25 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
 void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId);
 void syncNodeVoteForSelf(SSyncNode* pSyncNode);
 
+// snapshot --------------
+bool syncNodeHasSnapshot(SSyncNode* pSyncNode);
+bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index);
+
+SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode);
+SyncTerm  syncNodeGetLastTerm(SSyncNode* pSyncNode);
+int32_t   syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm);
+
+SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode);
+
+SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index);
+SyncTerm  syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index);
+int32_t   syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm);
+
+int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag);
+
+bool                 syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId);
+SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId);
+
 // for debug --------------
 void syncNodePrint(SSyncNode* pObj);
 void syncNodePrint2(char* s, SSyncNode* pObj);

source/libs/sync/inc/syncRaftCfg.h

@@ -34,6 +34,7 @@ typedef struct SRaftCfg {
   TdFilePtr pFile;
   char      path[TSDB_FILENAME_LEN * 2];
   int8_t    isStandBy;
+  int8_t    snapshotEnable;
 } SRaftCfg;
 
 SRaftCfg *raftCfgOpen(const char *path);
@@ -50,7 +51,12 @@ char *  raftCfg2Str(SRaftCfg *pRaftCfg);
 int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
 int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);
 
-int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path);
+typedef struct SRaftCfgMeta {
+  int8_t isStandBy;
+  int8_t snapshotEnable;
+} SRaftCfgMeta;
+
+int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path);
 
 // for debug ----------------------
 void syncCfgPrint(SSyncCfg *pCfg);

source/libs/sync/inc/syncRaftLog.h

@@ -30,6 +30,7 @@ extern "C" {
 typedef struct SSyncLogStoreData {
   SSyncNode* pSyncNode;
   SWal*      pWal;
+  SyncIndex  beginIndex;  // valid begin index, default 0, may be set beginIndex > 0
 } SSyncLogStoreData;
 
 SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode);
@@ -39,14 +40,7 @@ char*          logStore2Str(SSyncLogStore* pLogStore);
 cJSON*         logStoreSimple2Json(SSyncLogStore* pLogStore);
 char*          logStoreSimple2Str(SSyncLogStore* pLogStore);
 
-// SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore);
+SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore);
-// SyncIndex       logStoreLastIndex(SSyncLogStore* pLogStore);
-// SyncTerm        logStoreLastTerm(SSyncLogStore* pLogStore);
-// SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index);
-// int32_t         logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
-// int32_t         logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex);
-// int32_t         logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index);
-// SyncIndex       logStoreGetCommitIndex(SSyncLogStore* pLogStore);
 
 // for debug
 void logStorePrint(SSyncLogStore* pLogStore);

source/libs/sync/inc/syncRaftStore.h

@@ -49,8 +49,8 @@ void    raftStoreClearVote(SRaftStore *pRaftStore);
 void    raftStoreNextTerm(SRaftStore *pRaftStore);
 void    raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term);
 int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson);
-cJSON * raftStore2Json(SRaftStore *pRaftStore);
+cJSON  *raftStore2Json(SRaftStore *pRaftStore);
-char *  raftStore2Str(SRaftStore *pRaftStore);
+char   *raftStore2Str(SRaftStore *pRaftStore);
 
 // for debug -------------------
 void raftStorePrint(SRaftStore *pObj);

source/libs/sync/inc/syncReplication.h

@@ -52,6 +52,7 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode);
+int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode);
 int32_t syncNodeReplicate(SSyncNode* pSyncNode);
 int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg);
 

source/libs/sync/inc/syncRequestVote.h

@@ -50,6 +50,7 @@ extern "C" {
 //       /\ UNCHANGED <<state, currentTerm, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncRequestVoteReply.h

@@ -45,6 +45,7 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, votedFor, leaderVars, logVars>>
 //
 int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncSnapshot.h

@@ -25,40 +25,64 @@ extern "C" {
 #include <stdlib.h>
 #include "cJSON.h"
 #include "syncInt.h"
+#include "syncMessage.h"
 #include "taosdef.h"
 
+#define SYNC_SNAPSHOT_SEQ_INVALID -1
+#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2
+#define SYNC_SNAPSHOT_SEQ_BEGIN 0
+#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF
+
+#define SYNC_SNAPSHOT_RETRY_MS 5000
+
 typedef struct SSyncSnapshotSender {
-  int32_t    sending;
+  bool       start;
-  int32_t    received;
+  int32_t    seq;
-  bool       finish;
+  int32_t    ack;
-  void *     pCurrentBlock;
+  void      *pReader;
+  void      *pCurrentBlock;
   int32_t    blockLen;
+  SSnapshot  snapshot;
   int64_t    sendingMS;
-  SSnapshot *pSnapshot;
   SSyncNode *pSyncNode;
+  int32_t    replicaIndex;
+  SyncTerm   term;
+  SyncTerm   privateTerm;
+  bool       finish;
 } SSyncSnapshotSender;
 
-SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode);
+SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex);
 void                 snapshotSenderDestroy(SSyncSnapshotSender *pSender);
+bool                 snapshotSenderIsStart(SSyncSnapshotSender *pSender);
+void                 snapshotSenderStart(SSyncSnapshotSender *pSender);
+void                 snapshotSenderStop(SSyncSnapshotSender *pSender);
 int32_t              snapshotSend(SSyncSnapshotSender *pSender);
-cJSON *              snapshotSender2Json(SSyncSnapshotSender *pSender);
+int32_t              snapshotReSend(SSyncSnapshotSender *pSender);
-char *               snapshotSender2Str(SSyncSnapshotSender *pSender);
+cJSON               *snapshotSender2Json(SSyncSnapshotSender *pSender);
+char                *snapshotSender2Str(SSyncSnapshotSender *pSender);
 
 typedef struct SSyncSnapshotReceiver {
-  bool       start;
+  bool start;
-  int32_t    received;
+
-  int32_t    progressIndex;
+  int32_t  ack;
-  void *     pCurrentBlock;
+  void    *pWriter;
-  int32_t    len;
+  SyncTerm term;
-  SSnapshot *pSnapshot;
+  SyncTerm privateTerm;
+
   SSyncNode *pSyncNode;
+  int32_t    replicaIndex;
 } SSyncSnapshotReceiver;
 
-SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode);
+SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex);
 void                   snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver);
-int32_t                snapshotReceive(SSyncSnapshotReceiver *pReceiver);
+void                   snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm);
-cJSON *                snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
+bool                   snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
-char *                 snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
+void                   snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
+cJSON                 *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
+char                  *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
+
+int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg);
+int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncUtil.h

@@ -61,6 +61,7 @@ bool        syncUtilIsData(tmsg_t msgType);
 bool        syncUtilUserPreCommit(tmsg_t msgType);
 bool        syncUtilUserCommit(tmsg_t msgType);
 bool        syncUtilUserRollback(tmsg_t msgType);
+void        syncUtilJson2Line(char* jsonStr);
 
 #ifdef __cplusplus
 }

source/libs/sync/src/syncAppendEntries.c

@@ -18,8 +18,10 @@
 #include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
+#include "syncSnapshot.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
+#include "wal.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -335,8 +337,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
                 cbMeta.currentTerm = ths->pRaftStore->currentTerm;
                 cbMeta.flag = 0x11;
 
+                SSnapshot snapshot;
+                ASSERT(ths->pFsm->FpGetSnapshot != NULL);
+                ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+
                 bool needExecute = true;
-                if (ths->pSnapshot != NULL && cbMeta.index <= ths->pSnapshot->lastApplyIndex) {
+                if (cbMeta.index <= snapshot.lastApplyIndex) {
                   needExecute = false;
                 }
 
@@ -427,3 +433,332 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
 
   return ret;
 }
+
+static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) {
+  int32_t code;
+
+  SyncIndex delBegin = pMsg->prevLogIndex + 1;
+  SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore);
+
+  // invert roll back!
+  for (SyncIndex index = delEnd; index >= delBegin; --index) {
+    if (ths->pFsm->FpRollBackCb != NULL) {
+      SSyncRaftEntry* pRollBackEntry;
+      code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, index, &pRollBackEntry);
+      ASSERT(code == 0);
+      ASSERT(pRollBackEntry != NULL);
+
+      if (syncUtilUserRollback(pRollBackEntry->msgType)) {
+        SRpcMsg rpcMsg;
+        syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
+
+        SFsmCbMeta cbMeta;
+        cbMeta.index = pRollBackEntry->index;
+        cbMeta.isWeak = pRollBackEntry->isWeak;
+        cbMeta.code = 0;
+        cbMeta.state = ths->state;
+        cbMeta.seqNum = pRollBackEntry->seqNum;
+        ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
+        rpcFreeCont(rpcMsg.pCont);
+      }
+
+      syncEntryDestory(pRollBackEntry);
+    }
+  }
+
+  // delete confict entries
+  code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin);
+  ASSERT(code == 0);
+  sInfo("sync event log truncate, from %ld to %ld", delBegin, delEnd);
+  logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore);
+
+  return code;
+}
+
+static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) {
+  SRpcMsg rpcMsg;
+  syncEntry2OriginalRpc(pEntry, &rpcMsg);
+  if (ths->pFsm != NULL) {
+    if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
+      SFsmCbMeta cbMeta;
+      cbMeta.index = pEntry->index;
+      cbMeta.isWeak = pEntry->isWeak;
+      cbMeta.code = 2;
+      cbMeta.state = ths->state;
+      cbMeta.seqNum = pEntry->seqNum;
+      ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
+    }
+  }
+  rpcFreeCont(rpcMsg.pCont);
+  return 0;
+}
+
+// really pre log match
+// prevLogIndex == -1
+static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) {
+  if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) {
+    if (gRaftDetailLog) {
+      sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex);
+    }
+    return true;
+  }
+
+  SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
+  if (pMsg->prevLogIndex > myLastIndex) {
+    if (gRaftDetailLog) {
+      sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex,
+             myLastIndex);
+    }
+    return false;
+  }
+
+  SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1);
+  if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) {
+    if (gRaftDetailLog) {
+      sTrace(
+          "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
+          "myPreLogTerm:%lu",
+          pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
+    }
+    return true;
+  }
+
+  if (gRaftDetailLog) {
+    sTrace(
+        "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
+        "myPreLogTerm:%lu",
+        pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
+  }
+
+  return false;
+}
+
+int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
+  int32_t ret = 0;
+  int32_t code = 0;
+
+  // print log
+  char logBuf[128] = {0};
+  snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId,
+           ths->pRaftStore->currentTerm);
+  syncAppendEntriesLog2(logBuf, pMsg);
+
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
+    sInfo("recv SyncAppendEntries maybe replica already dropped");
+    return ret;
+  }
+
+  // maybe update term
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    syncNodeUpdateTerm(ths, pMsg->term);
+  }
+  ASSERT(pMsg->term <= ths->pRaftStore->currentTerm);
+
+  // reset elect timer
+  if (pMsg->term == ths->pRaftStore->currentTerm) {
+    ths->leaderCache = pMsg->srcId;
+    syncNodeResetElectTimer(ths);
+  }
+  ASSERT(pMsg->dataLen >= 0);
+
+  // candidate to follower
+  //
+  // operation:
+  // to follower
+  do {
+    bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE;
+    if (condition) {
+      sTrace("recv SyncAppendEntries, candidate to follower");
+
+      syncNodeBecomeFollower(ths);
+      // do not reply?
+      return ret;
+    }
+  } while (0);
+
+  // fake match
+  //
+  // condition1:
+  // I have snapshot, no log, preIndex > myLastIndex
+  //
+  // condition2:
+  // I have snapshot, have log, log <= snapshot, preIndex > myLastIndex
+  //
+  // condition3:
+  // I have snapshot, preIndex < snapshot.lastApplyIndex
+  //
+  // condition4:
+  // I have snapshot, preIndex == snapshot.lastApplyIndex, no data
+  //
+  // operation:
+  // match snapshot.lastApplyIndex - 1;
+  // no operation on log
+  do {
+    SyncIndex myLastIndex = syncNodeGetLastIndex(ths);
+    SSnapshot snapshot;
+    ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+
+    bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
+                      syncNodeHasSnapshot(ths);
+    bool condition1 =
+        condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex);
+    bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) &&
+                      (pMsg->prevLogIndex > myLastIndex);
+    bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex);
+    bool condition4 = condition0 && (pMsg->prevLogIndex == snapshot.lastApplyIndex) && (pMsg->dataLen == 0);
+    bool condition = condition1 || condition2 || condition3 || condition4;
+
+    if (condition) {
+      sTrace(
+          "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, "
+          "condition1:%d, condition2:%d, condition3:%d, condition4:%d",
+          myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore),
+          ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4);
+
+      // prepare response msg
+      SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
+      pReply->srcId = ths->myRaftId;
+      pReply->destId = pMsg->srcId;
+      pReply->term = ths->pRaftStore->currentTerm;
+      pReply->privateTerm = ths->pNewNodeReceiver->privateTerm;
+      pReply->success = true;
+      pReply->matchIndex = snapshot.lastApplyIndex;
+
+      // send response
+      SRpcMsg rpcMsg;
+      syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+      syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+      syncAppendEntriesReplyDestroy(pReply);
+
+      return ret;
+    }
+  } while (0);
+
+  // calculate logOK here, before will coredump, due to fake match
+  bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg);
+
+  // not match
+  //
+  // condition1:
+  // term < myTerm
+  //
+  // condition2:
+  // !logOK
+  //
+  // operation:
+  // not match
+  // no operation on log
+  do {
+    bool condition1 = pMsg->term < ths->pRaftStore->currentTerm;
+    bool condition2 =
+        (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK;
+    bool condition = condition1 || condition2;
+
+    if (condition) {
+      sTrace(
+          "recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, "
+          "condition2:%d, logOK:%d",
+          ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore),
+          condition1, condition2, logOK);
+
+      // prepare response msg
+      SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
+      pReply->srcId = ths->myRaftId;
+      pReply->destId = pMsg->srcId;
+      pReply->term = ths->pRaftStore->currentTerm;
+      pReply->privateTerm = ths->pNewNodeReceiver->privateTerm;
+      pReply->success = false;
+      pReply->matchIndex = SYNC_INDEX_INVALID;
+
+      // send response
+      SRpcMsg rpcMsg;
+      syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+      syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+      syncAppendEntriesReplyDestroy(pReply);
+
+      return ret;
+    }
+  } while (0);
+
+  // really match
+  //
+  // condition:
+  // logOK
+  //
+  // operation:
+  // match
+  // make log same
+  do {
+    bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && logOK;
+    if (condition) {
+      // has extra entries (> preIndex) in local log
+      SyncIndex myLastIndex = syncNodeGetLastIndex(ths);
+      bool      hasExtraEntries = myLastIndex > pMsg->prevLogIndex;
+
+      // has entries in SyncAppendEntries msg
+      bool hasAppendEntries = pMsg->dataLen > 0;
+
+      sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex,
+             hasExtraEntries, hasAppendEntries);
+
+      if (hasExtraEntries) {
+        // make log same, rollback deleted entries
+        code = syncNodeMakeLogSame(ths, pMsg);
+        ASSERT(code == 0);
+      }
+
+      if (hasAppendEntries) {
+        // append entry
+        SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
+        ASSERT(pAppendEntry != NULL);
+
+        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
+        ASSERT(code == 0);
+
+        // pre commit
+        code = syncNodePreCommit(ths, pAppendEntry);
+        ASSERT(code == 0);
+
+        syncEntryDestory(pAppendEntry);
+      }
+
+      // prepare response msg
+      SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
+      pReply->srcId = ths->myRaftId;
+      pReply->destId = pMsg->srcId;
+      pReply->term = ths->pRaftStore->currentTerm;
+      pReply->privateTerm = ths->pNewNodeReceiver->privateTerm;
+      pReply->success = true;
+      pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex;
+
+      // send response
+      SRpcMsg rpcMsg;
+      syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+      syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+      syncAppendEntriesReplyDestroy(pReply);
+
+      // maybe update commit index, leader notice me
+      if (pMsg->commitIndex > ths->commitIndex) {
+        // has commit entry in local
+        if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) {
+          SyncIndex beginIndex = ths->commitIndex + 1;
+          SyncIndex endIndex = pMsg->commitIndex;
+
+          // update commit index
+          ths->commitIndex = pMsg->commitIndex;
+
+          // call back Wal
+          code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex);
+          ASSERT(code == 0);
+
+          code = syncNodeCommit(ths, beginIndex, endIndex, ths->state);
+          ASSERT(code == 0);
+        }
+      }
+      return ret;
+    }
+  } while (0);
+
+  return ret;
+}

source/libs/sync/src/syncAppendEntriesReply.c

@@ -17,8 +17,10 @@
 #include "syncCommit.h"
 #include "syncIndexMgr.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
+#include "syncSnapshot.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
 
@@ -94,3 +96,116 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p
 
   return ret;
 }
+
+int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
+  int32_t ret = 0;
+
+  // print log
+  char logBuf[128] = {0};
+  snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId,
+           ths->pRaftStore->currentTerm);
+  syncAppendEntriesReplyLog2(logBuf, pMsg);
+
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
+    sInfo("recv SyncAppendEntriesReply,  maybe replica already dropped");
+    return ret;
+  }
+
+  // drop stale response
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term,
+           ths->pRaftStore->currentTerm);
+    return ret;
+  }
+
+  syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex);
+  syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex);
+  {
+    SSnapshot snapshot;
+    ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+    sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
+           snapshot.lastApplyIndex, snapshot.lastApplyTerm);
+  }
+
+  // no need this code, because if I receive reply.term, then I must have sent for that term.
+  //  if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //    syncNodeUpdateTerm(ths, pMsg->term);
+  //  }
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128] = {0};
+    snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu",
+             pMsg->term, ths->pRaftStore->currentTerm);
+    syncNodeLog2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
+
+  ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
+
+  if (pMsg->success) {
+    // nextIndex'  = [nextIndex  EXCEPT ![i][j] = m.mmatchIndex + 1]
+    syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1);
+    sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success);
+
+    // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex]
+    syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
+
+    // maybe commit
+    if (ths->state == TAOS_SYNC_STATE_LEADER) {
+      syncMaybeAdvanceCommitIndex(ths);
+    }
+
+  } else {
+    SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
+    sTrace("update next not match, begin, index:%ld, success:%d", nextIndex, pMsg->success);
+
+    // notice! int64, uint64
+    if (nextIndex > SYNC_INDEX_BEGIN) {
+      --nextIndex;
+
+      // get sender
+      SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId));
+      ASSERT(pSender != NULL);
+      bool      hasSnapshot = syncNodeHasSnapshot(ths);
+      SSnapshot snapshot;
+      ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+
+      // start sending snapshot first time
+      // start here, stop by receiver
+      if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) &&
+          pMsg->privateTerm < pSender->privateTerm) {
+        snapshotSenderStart(pSender);
+
+        char* s = snapshotSender2Str(pSender);
+        sInfo("sync event snapshot send start sender first time, sender:%s", s);
+        taosMemoryFree(s);
+      }
+
+      SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1;
+
+      // update nextIndex to sentryIndex
+      if (nextIndex <= sentryIndex) {
+        nextIndex = sentryIndex;
+      }
+
+    } else {
+      nextIndex = SYNC_INDEX_BEGIN;
+    }
+
+    syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
+    sTrace("update next not match, end, index:%ld, success:%d", nextIndex, pMsg->success);
+  }
+
+  syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex);
+  syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex);
+  {
+    SSnapshot snapshot;
+    ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+    sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
+           snapshot.lastApplyIndex, snapshot.lastApplyTerm);
+  }
+
+  return ret;
+}

source/libs/sync/src/syncCommit.c

@@ -94,109 +94,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
 
     // execute fsm
     if (pSyncNode->pFsm != NULL) {
-      for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
+      int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state);
-        if (i != SYNC_INDEX_INVALID) {
+      ASSERT(code == 0);
-          SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i);
-          assert(pEntry != NULL);
-
-          SRpcMsg rpcMsg;
-          syncEntry2OriginalRpc(pEntry, &rpcMsg);
-
-          if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
-            SFsmCbMeta cbMeta;
-            cbMeta.index = pEntry->index;
-            cbMeta.isWeak = pEntry->isWeak;
-            cbMeta.code = 0;
-            cbMeta.state = pSyncNode->state;
-            cbMeta.seqNum = pEntry->seqNum;
-            cbMeta.term = pEntry->term;
-            cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm;
-            cbMeta.flag = 0x1;
-
-            bool needExecute = true;
-            if (pSyncNode->pSnapshot != NULL && cbMeta.index <= pSyncNode->pSnapshot->lastApplyIndex) {
-              needExecute = false;
-            }
-
-            if (needExecute) {
-              pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
-            }
-          }
-
-          // config change
-          if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) {
-            SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg;
-
-            SSyncCfg newSyncCfg;
-            int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
-            ASSERT(ret == 0);
-
-            // update new config myIndex
-            bool hit = false;
-            for (int i = 0; i < newSyncCfg.replicaNum; ++i) {
-              if (strcmp(pSyncNode->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 &&
-                  pSyncNode->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) {
-                newSyncCfg.myIndex = i;
-                hit = true;
-                break;
-              }
-            }
-
-            if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
-              ASSERT(hit == true);
-            }
-
-            bool isDrop;
-            syncNodeUpdateConfig(pSyncNode, &newSyncCfg, &isDrop);
-
-            // change isStandBy to normal
-            if (!isDrop) {
-              if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
-                syncNodeBecomeLeader(pSyncNode);
-              } else {
-                syncNodeBecomeFollower(pSyncNode);
-              }
-            }
-
-            char* sOld = syncCfg2Str(&oldSyncCfg);
-            char* sNew = syncCfg2Str(&newSyncCfg);
-            sInfo("==config change== 0x1 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop);
-            taosMemoryFree(sOld);
-            taosMemoryFree(sNew);
-
-            if (pSyncNode->pFsm->FpReConfigCb != NULL) {
-              SReConfigCbMeta cbMeta = {0};
-              cbMeta.code = 0;
-              cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm;
-              cbMeta.index = pEntry->index;
-              cbMeta.term = pEntry->term;
-              cbMeta.oldCfg = oldSyncCfg;
-              cbMeta.flag = 0x1;
-              cbMeta.isDrop = isDrop;
-              pSyncNode->pFsm->FpReConfigCb(pSyncNode->pFsm, newSyncCfg, cbMeta);
-            }
-          }
-
-          // restore finish
-          if (pEntry->index == pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore)) {
-            if (pSyncNode->restoreFinish == false) {
-              if (pSyncNode->pFsm->FpRestoreFinishCb != NULL) {
-                pSyncNode->pFsm->FpRestoreFinishCb(pSyncNode->pFsm);
-              }
-              pSyncNode->restoreFinish = true;
-              sInfo("==syncMaybeAdvanceCommitIndex== restoreFinish set true %p vgId:%d", pSyncNode, pSyncNode->vgId);
-
-              /*
-              tsem_post(&pSyncNode->restoreSem);
-              sInfo("==syncMaybeAdvanceCommitIndex== RestoreFinish tsem_post %p", pSyncNode);
-              */
-            }
-          }
-
-          rpcFreeCont(rpcMsg.pCont);
-          syncEntryDestory(pEntry);
-        }
-      }
     }
   }
 }

source/libs/sync/src/syncElection.c

@@ -15,6 +15,7 @@
 
 #include "syncElection.h"
 #include "syncMessage.h"
+#include "syncRaftCfg.h"
 #include "syncRaftStore.h"
 #include "syncVoteMgr.h"
 
@@ -49,6 +50,26 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
   return ret;
 }
 
+int32_t syncNodeRequestVotePeersSnapshot(SSyncNode* pSyncNode) {
+  ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+
+  int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId);
+    pMsg->srcId = pSyncNode->myRaftId;
+    pMsg->destId = pSyncNode->peersId[i];
+    pMsg->term = pSyncNode->pRaftStore->currentTerm;
+
+    ret = syncNodeGetLastIndexTerm(pSyncNode, &(pMsg->lastLogIndex), &(pMsg->lastLogTerm));
+    ASSERT(ret == 0);
+
+    ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg);
+    ASSERT(ret == 0);
+    syncRequestVoteDestroy(pMsg);
+  }
+  return ret;
+}
+
 int32_t syncNodeElect(SSyncNode* pSyncNode) {
   int32_t ret = 0;
   if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
@@ -71,7 +92,12 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
     return ret;
   }
 
-  ret = syncNodeRequestVotePeers(pSyncNode);
+  if (pSyncNode->pRaftCfg->snapshotEnable) {
+    ret = syncNodeRequestVotePeersSnapshot(pSyncNode);
+  } else {
+    ret = syncNodeRequestVotePeers(pSyncNode);
+  }
+
   assert(ret == 0);
   syncNodeResetElectTimer(pSyncNode);
 

source/libs/sync/src/syncIO.c

@@ -75,7 +75,8 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
     syncUtilMsgNtoH(pMsg->pCont);
 
     char logBuf[256] = {0};
-    snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
+    snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port,
+             pMsg->msgType);
     syncRpcMsgLog2(logBuf, pMsg);
 
     syncUtilMsgHtoN(pMsg->pCont);
@@ -89,8 +90,10 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
 
 int32_t syncIOEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   int32_t ret = 0;
-  char    logBuf[128] = {0};
+
-  syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg);
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==syncIOEqMsg== msgType:%d", pMsg->msgType);
+  syncRpcMsgLog2(logBuf, pMsg);
 
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM);
@@ -253,7 +256,9 @@ static void *syncIOConsumerFunc(void *param) {
 
     for (int i = 0; i < numOfMsgs; ++i) {
       taosGetQitem(qall, (void **)&pRpcMsg);
-      syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg);
+      char logBuf[128];
+      snprintf(logBuf, sizeof(logBuf), "==syncIOConsumMsg== msgType:%d", pRpcMsg->msgType);
+      syncRpcMsgLog2(logBuf, pRpcMsg);
 
       // use switch case instead of if else
       if (pRpcMsg->msgType == TDMT_SYNC_PING) {
@@ -319,6 +324,23 @@ static void *syncIOConsumerFunc(void *param) {
           io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg);
           syncTimeoutDestroy(pSyncMsg);
         }
+
+      } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) {
+        if (io->FpOnSyncSnapshotSend != NULL) {
+          SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pRpcMsg);
+          assert(pSyncMsg != NULL);
+          io->FpOnSyncSnapshotSend(io->pSyncNode, pSyncMsg);
+          syncSnapshotSendDestroy(pSyncMsg);
+        }
+
+      } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) {
+        if (io->FpOnSyncSnapshotRsp != NULL) {
+          SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pRpcMsg);
+          assert(pSyncMsg != NULL);
+          io->FpOnSyncSnapshotRsp(io->pSyncNode, pSyncMsg);
+          syncSnapshotRspDestroy(pSyncMsg);
+        }
+
       } else {
         sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType);
       }

source/libs/sync/src/syncIndexMgr.c

@@ -46,6 +46,7 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
 
 void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) {
   memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index));
+  memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm));
   /*
   for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
     pSyncIndexMgr->index[i] = 0;
@@ -62,7 +63,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId,
   }
 
   // maybe config change
-  // assert(0);
+  assert(0);
 }
 
 SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
@@ -86,14 +87,27 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
     for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
       cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i]));
     }
-    int  respondNum = 0;
+
-    int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum);
+    {
-    for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+      int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum);
-      arr[i] = pSyncIndexMgr->index[i];
+      for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+        arr[i] = pSyncIndexMgr->index[i];
+      }
+      cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
+      taosMemoryFree(arr);
+      cJSON_AddItemToObject(pRoot, "index", pIndex);
     }
-    cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
+
-    taosMemoryFree(arr);
+    {
-    cJSON_AddItemToObject(pRoot, "index", pIndex);
+      int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum);
+      for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+        arr[i] = pSyncIndexMgr->privateTerm[i];
+      }
+      cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
+      taosMemoryFree(arr);
+      cJSON_AddItemToObject(pRoot, "privateTerm", pIndex);
+    }
+
     snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
     cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
   }
@@ -105,7 +119,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
 
 char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
   cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
@@ -132,7 +146,31 @@ void syncIndexMgrLog(SSyncIndexMgr *pObj) {
 }
 
 void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) {
-  char *serialized = syncIndexMgr2Str(pObj);
+  if (gRaftDetailLog) {
-  sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char *serialized = syncIndexMgr2Str(pObj);
-  taosMemoryFree(serialized);
+    sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
+void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) {
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
+      (pSyncIndexMgr->privateTerm)[i] = term;
+      return;
+    }
+  }
+
+  // maybe config change
+  assert(0);
+}
+
+SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
+      SyncTerm term = (pSyncIndexMgr->privateTerm)[i];
+      return term;
+    }
+  }
+  assert(0);
 }

source/libs/sync/src/syncMain.c

@@ -29,11 +29,14 @@
 #include "syncRequestVote.h"
 #include "syncRequestVoteReply.h"
 #include "syncRespMgr.h"
+#include "syncSnapshot.h"
 #include "syncTimeout.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
 #include "tref.h"
 
+bool gRaftDetailLog = false;
+
 static int32_t tsNodeRefId = -1;
 
 // ------ local funciton ---------
@@ -213,6 +216,18 @@ bool syncIsRestoreFinish(int64_t rid) {
   return b;
 }
 
+int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) {
+  SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid);
+  if (pSyncNode == NULL) {
+    return -1;
+  }
+  assert(rid == pSyncNode->rid);
+  *sMeta = pSyncNode->sMeta;
+
+  taosReleaseRef(tsNodeRefId, pSyncNode->rid);
+  return 0;
+}
+
 const char* syncGetMyRoleStr(int64_t rid) {
   const char* s = syncUtilState2String(syncGetMyRole(rid));
   return s;
@@ -411,8 +426,11 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
 
   snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path);
   if (!taosCheckExistFile(pSyncNode->configPath)) {
-    // create raft config file
+    // create a new raft config file
-    ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), pSyncInfo->isStandBy, pSyncNode->configPath);
+    SRaftCfgMeta meta;
+    meta.isStandBy = pSyncInfo->isStandBy;
+    meta.snapshotEnable = pSyncInfo->snapshotEnable;
+    ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath);
     assert(ret == 0);
 
   } else {
@@ -552,35 +570,64 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
   pSyncNode->FpOnPing = syncNodeOnPingCb;
   pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb;
   pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb;
-  pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
-  pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb;
-  pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb;
-  pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb;
   pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb;
 
+  pSyncNode->FpOnSnapshotSend = syncNodeOnSnapshotSendCb;
+  pSyncNode->FpOnSnapshotRsp = syncNodeOnSnapshotRspCb;
+
+  if (pSyncNode->pRaftCfg->snapshotEnable) {
+    sInfo("sync node use snapshot");
+    pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteSnapshotCb;
+    pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplySnapshotCb;
+    pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesSnapshotCb;
+    pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplySnapshotCb;
+
+  } else {
+    sInfo("sync node do not use snapshot");
+    pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
+    pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb;
+    pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb;
+    pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb;
+  }
+
   // tools
   pSyncNode->pSyncRespMgr = syncRespMgrCreate(NULL, 0);
   assert(pSyncNode->pSyncRespMgr != NULL);
 
   // restore state
   pSyncNode->restoreFinish = false;
-  pSyncNode->pSnapshot = NULL;
+
-  if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
+  // pSyncNode->pSnapshot = NULL;
-    pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot));
+  // if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
-    pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot);
+  //   pSyncNode->pSnapshot = taosMemoryMalloc(sizeof(SSnapshot));
-  }
+  //   pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, pSyncNode->pSnapshot);
+  // }
   // tsem_init(&(pSyncNode->restoreSem), 0, 0);
 
+  // snapshot senders
+  for (int i = 0; i < TSDB_MAX_REPLICA; ++i) {
+    SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i);
+    // ASSERT(pSender != NULL);
+    (pSyncNode->senders)[i] = pSender;
+  }
+
+  // snapshot receivers
+  pSyncNode->pNewNodeReceiver = snapshotReceiverCreate(pSyncNode, 100);
+
   // start in syncNodeStart
   // start raft
   // syncNodeBecomeFollower(pSyncNode);
 
+  // snapshot meta
+  pSyncNode->sMeta.lastConfigIndex = -1;
+
   return pSyncNode;
 }
 
 void syncNodeStart(SSyncNode* pSyncNode) {
   // start raft
   if (pSyncNode->replicaNum == 1) {
+    raftStoreNextTerm(pSyncNode->pRaftStore);
     syncNodeBecomeLeader(pSyncNode);
 
     syncNodeLog2("==state change become leader immediately==", pSyncNode);
@@ -662,9 +709,23 @@ void syncNodeClose(SSyncNode* pSyncNode) {
     taosMemoryFree(pSyncNode->pFsm);
   }
 
+  for (int i = 0; i < TSDB_MAX_REPLICA; ++i) {
+    if ((pSyncNode->senders)[i] != NULL) {
+      snapshotSenderDestroy((pSyncNode->senders)[i]);
+      (pSyncNode->senders)[i] = NULL;
+    }
+  }
+
+  if (pSyncNode->pNewNodeReceiver != NULL) {
+    snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver);
+    pSyncNode->pNewNodeReceiver = NULL;
+  }
+
+  /*
   if (pSyncNode->pSnapshot != NULL) {
     taosMemoryFree(pSyncNode->pSnapshot);
   }
+  */
 
   // tsem_destroy(&pSyncNode->restoreSem);
 
@@ -672,6 +733,9 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   // taosMemoryFree(pSyncNode);
 }
 
+// option
+bool syncNodeSnapshotEnable(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotEnable; }
+
 // ping --------------
 int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) {
   syncPingLog2((char*)"==syncNodePing==", pMsg);
@@ -762,7 +826,13 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
 
 int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) {
   int32_t ret = 0;
-  int32_t electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
+  int32_t electMS;
+
+  if (pSyncNode->pRaftCfg->isStandBy) {
+    electMS = TIMER_MAX_MS;
+  } else {
+    electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
+  }
   ret = syncNodeRestartElectTimer(pSyncNode, electMS);
   return ret;
 }
@@ -788,6 +858,13 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp
   SEpSet epSet;
   syncUtilraftId2EpSet(destRaftId, &epSet);
   if (pSyncNode->FpSendMsg != NULL) {
+    if (gRaftDetailLog) {
+      char* JsonStr = syncRpcMsg2Str(pMsg);
+      syncUtilJson2Line(JsonStr);
+      sTrace("sync send msg, vgId:%d, type:%d, msg:%s", pSyncNode->vgId, pMsg->msgType, JsonStr);
+      taosMemoryFree(JsonStr);
+    }
+
     // htonl
     syncUtilMsgHtoN(pMsg->pCont);
 
@@ -952,6 +1029,20 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
     cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout);
     cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf);
+
+    // restoreFinish
+    cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish);
+
+    // snapshot senders
+    cJSON* pSenders = cJSON_CreateArray();
+    cJSON_AddItemToObject(pRoot, "senders", pSenders);
+    for (int i = 0; i < TSDB_MAX_REPLICA; ++i) {
+      cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i]));
+    }
+
+    // snapshot receivers
+    cJSON* pReceivers = cJSON_CreateArray();
+    cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver));
   }
 
   cJSON* pJson = cJSON_CreateObject();
@@ -973,10 +1064,10 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
            "syncNode2SimpleStr vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, "
            "electTimerLogicClock:%lu, "
            "electTimerLogicClockUser:%lu, "
-           "electTimerMS:%d",
+           "electTimerMS:%d, replicaNum:%d",
            pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state,
            syncUtilState2String(pSyncNode->state), pSyncNode->pRaftCfg->isStandBy, pSyncNode->electTimerLogicClock,
-           pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS);
+           pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, pSyncNode->replicaNum);
   return s;
 }
 
@@ -1013,6 +1104,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, bool* isDro
   voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode);
   votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode);
 
+  pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
+
   // isDrop
   *isDrop = true;
   bool IamInOld, IamInNew;
@@ -1103,7 +1196,15 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
   for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) {
     // maybe overwrite myself, no harm
     // just do it!
-    pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1;
+
+    // pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1;
+
+    // maybe wal is deleted
+    SyncIndex lastIndex;
+    SyncTerm  lastTerm;
+    int32_t   code = syncNodeGetLastIndexTerm(pSyncNode, &lastIndex, &lastTerm);
+    ASSERT(code == 0);
+    pSyncNode->pNextIndex->index[i] = lastIndex + 1;
   }
 
   for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) {
@@ -1112,6 +1213,17 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
     pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID;
   }
 
+  // update sender private term
+  SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId));
+  if (pMySender != NULL) {
+    for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) {
+      if ((pSyncNode->senders)[i]->privateTerm > pMySender->privateTerm) {
+        pMySender->privateTerm = (pSyncNode->senders)[i]->privateTerm;
+      }
+    }
+    (pMySender->privateTerm) += 100;
+  }
+
   // stop elect timer
   syncNodeStopElectTimer(pSyncNode);
 
@@ -1186,6 +1298,153 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
   syncRequestVoteReplyDestroy(pMsg);
 }
 
+// snapshot --------------
+bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
+  bool      ret = false;
+  SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+  if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
+    pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+    if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN) {
+      ret = true;
+    }
+  }
+  return ret;
+}
+
+bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) {
+  ASSERT(syncNodeHasSnapshot(pSyncNode));
+  ASSERT(pSyncNode->pFsm->FpGetSnapshot != NULL);
+  ASSERT(index >= SYNC_INDEX_BEGIN);
+
+  SSnapshot snapshot;
+  pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+  bool b = (index <= snapshot.lastApplyIndex);
+  return b;
+}
+
+SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
+  SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+  if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
+    pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+  }
+  SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+
+  SyncIndex lastIndex = logLastIndex > snapshot.lastApplyIndex ? logLastIndex : snapshot.lastApplyIndex;
+  return lastIndex;
+}
+
+SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) {
+  SyncTerm lastTerm = 0;
+  if (syncNodeHasSnapshot(pSyncNode)) {
+    // has snapshot
+    SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+    if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
+      pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+    }
+
+    SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+    if (logLastIndex > snapshot.lastApplyIndex) {
+      lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore);
+    } else {
+      lastTerm = snapshot.lastApplyTerm;
+    }
+
+  } else {
+    // no snapshot
+    lastTerm = pSyncNode->pLogStore->syncLogLastTerm(pSyncNode->pLogStore);
+  }
+
+  return lastTerm;
+}
+
+// get last index and term along with snapshot
+int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, SyncTerm* pLastTerm) {
+  *pLastIndex = syncNodeGetLastIndex(pSyncNode);
+  *pLastTerm = syncNodeGetLastTerm(pSyncNode);
+  return 0;
+}
+
+SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) {
+  SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1;
+  return syncStartIndex;
+}
+
+SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
+  ASSERT(index >= SYNC_INDEX_BEGIN);
+  SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
+  ASSERT(index <= syncStartIndex);
+
+  SyncIndex preIndex = index - 1;
+  return preIndex;
+}
+
+SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
+  ASSERT(index >= SYNC_INDEX_BEGIN);
+  SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
+  ASSERT(index <= syncStartIndex);
+
+  if (index == SYNC_INDEX_BEGIN) {
+    return 0;
+  }
+
+  SyncTerm preTerm = 0;
+  if (syncNodeHasSnapshot(pSyncNode)) {
+    // has snapshot
+    SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+    if (pSyncNode->pFsm->FpGetSnapshot != NULL) {
+      pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+    }
+
+    if (index > snapshot.lastApplyIndex + 1) {
+      // should be log preTerm
+      SSyncRaftEntry* pPreEntry = NULL;
+      int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
+      ASSERT(code == 0);
+      ASSERT(pPreEntry != NULL);
+
+      preTerm = pPreEntry->term;
+      taosMemoryFree(pPreEntry);
+
+    } else if (index == snapshot.lastApplyIndex + 1) {
+      preTerm = snapshot.lastApplyTerm;
+
+    } else {
+      // maybe snapshot change
+      sError("sync get pre term, bad scene. index:%ld", index);
+      logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore);
+
+      SSyncRaftEntry* pPreEntry = NULL;
+      int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
+      ASSERT(code == 0);
+      ASSERT(pPreEntry != NULL);
+
+      preTerm = pPreEntry->term;
+      taosMemoryFree(pPreEntry);
+    }
+
+  } else {
+    // no snapshot
+    ASSERT(index > SYNC_INDEX_BEGIN);
+
+    SSyncRaftEntry* pPreEntry = NULL;
+    int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
+    ASSERT(code == 0);
+    ASSERT(pPreEntry != NULL);
+
+    preTerm = pPreEntry->term;
+    taosMemoryFree(pPreEntry);
+  }
+
+  return preTerm;
+}
+
+// get pre index and term of "index"
+int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) {
+  *pPreIndex = syncNodeGetPreIndex(pSyncNode, index);
+  *pPreTerm = syncNodeGetPreTerm(pSyncNode, index);
+  return 0;
+}
+
 // for debug --------------
 void syncNodePrint(SSyncNode* pObj) {
   char* serialized = syncNode2Str(pObj);
@@ -1327,7 +1586,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
   assert(pEntry != NULL);
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
-    ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    // ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
     syncNodeReplicate(ths);
   }
 
@@ -1383,13 +1643,14 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
   int32_t ret = 0;
   syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
 
-  SyncIndex       index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1;
+  SyncIndex       index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore);
   SyncTerm        term = ths->pRaftStore->currentTerm;
   SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index);
   assert(pEntry != NULL);
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
-    ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    // ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
 
     // start replicate right now!
     syncNodeReplicate(ths);
@@ -1459,3 +1720,137 @@ const char* syncStr(ESyncState state) {
       return "error";
   }
 }
+
+int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) {
+  int32_t    code = 0;
+  ESyncState state = flag;
+  sInfo("sync event commit from index:%" PRId64 " to index:%" PRId64 ", %s", beginIndex, endIndex,
+        syncUtilState2String(state));
+
+  // maybe execute by leader, skip snapshot
+  SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
+  if (ths->pFsm->FpGetSnapshot != NULL) {
+    ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot);
+  }
+  if (beginIndex <= snapshot.lastApplyIndex) {
+    beginIndex = snapshot.lastApplyIndex + 1;
+  }
+
+  // execute fsm
+  if (ths->pFsm != NULL) {
+    for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
+      if (i != SYNC_INDEX_INVALID) {
+        SSyncRaftEntry* pEntry;
+        code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry);
+        ASSERT(code == 0);
+        ASSERT(pEntry != NULL);
+
+        SRpcMsg rpcMsg;
+        syncEntry2OriginalRpc(pEntry, &rpcMsg);
+
+        if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
+          SFsmCbMeta cbMeta;
+          cbMeta.index = pEntry->index;
+          cbMeta.isWeak = pEntry->isWeak;
+          cbMeta.code = 0;
+          cbMeta.state = ths->state;
+          cbMeta.seqNum = pEntry->seqNum;
+          cbMeta.term = pEntry->term;
+          cbMeta.currentTerm = ths->pRaftStore->currentTerm;
+          cbMeta.flag = flag;
+
+          ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
+        }
+
+        // config change
+        if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) {
+          SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg;
+
+          SSyncCfg newSyncCfg;
+          int32_t  ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+          ASSERT(ret == 0);
+
+          // update new config myIndex
+          bool hit = false;
+          for (int i = 0; i < newSyncCfg.replicaNum; ++i) {
+            if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 &&
+                ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) {
+              newSyncCfg.myIndex = i;
+              hit = true;
+              break;
+            }
+          }
+
+          SReConfigCbMeta cbMeta = {0};
+          bool            isDrop;
+
+          // I am in newConfig
+          if (hit) {
+            syncNodeUpdateConfig(ths, &newSyncCfg, &isDrop);
+
+            // change isStandBy to normal
+            if (!isDrop) {
+              if (ths->state == TAOS_SYNC_STATE_LEADER) {
+                syncNodeBecomeLeader(ths);
+              } else {
+                syncNodeBecomeFollower(ths);
+              }
+            }
+
+            char* sOld = syncCfg2Str(&oldSyncCfg);
+            char* sNew = syncCfg2Str(&newSyncCfg);
+            sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop);
+            taosMemoryFree(sOld);
+            taosMemoryFree(sNew);
+          }
+
+          // always call FpReConfigCb
+          if (ths->pFsm->FpReConfigCb != NULL) {
+            cbMeta.code = 0;
+            cbMeta.currentTerm = ths->pRaftStore->currentTerm;
+            cbMeta.index = pEntry->index;
+            cbMeta.term = pEntry->term;
+            cbMeta.oldCfg = oldSyncCfg;
+            cbMeta.flag = 0x11;
+            cbMeta.isDrop = isDrop;
+            ths->pFsm->FpReConfigCb(ths->pFsm, newSyncCfg, cbMeta);
+          }
+        }
+
+        // restore finish
+        if (pEntry->index == ths->pLogStore->syncLogLastIndex(ths->pLogStore)) {
+          if (ths->restoreFinish == false) {
+            if (ths->pFsm->FpRestoreFinishCb != NULL) {
+              ths->pFsm->FpRestoreFinishCb(ths->pFsm);
+            }
+            ths->restoreFinish = true;
+            sInfo("restore finish %p vgId:%d", ths, ths->vgId);
+          }
+        }
+
+        rpcFreeCont(rpcMsg.pCont);
+        syncEntryDestory(pEntry);
+      }
+    }
+  }
+  return 0;
+}
+
+bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) {
+  for (int i = 0; i < ths->replicaNum; ++i) {
+    if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) {
+  SSyncSnapshotSender* pSender = NULL;
+  for (int i = 0; i < ths->replicaNum; ++i) {
+    if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) {
+      pSender = (ths->senders)[i];
+    }
+  }
+  return pSender;
+}

source/libs/sync/src/syncMessage.c

@@ -65,6 +65,16 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
     pRoot = syncAppendEntriesReply2Json(pSyncMsg);
     syncAppendEntriesReplyDestroy(pSyncMsg);
 
+  } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) {
+    SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+    pRoot = syncSnapshotSend2Json(pSyncMsg);
+    syncSnapshotSendDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) {
+    SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+    pRoot = syncSnapshotRsp2Json(pSyncMsg);
+    syncSnapshotRspDestroy(pSyncMsg);
+
   } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) {
     pRoot = cJSON_CreateObject();
     char* s;
@@ -135,9 +145,11 @@ void syncRpcMsgLog(SRpcMsg* pMsg) {
 }
 
 void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) {
-  char* serialized = syncRpcMsg2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncRpcMsg2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncTimeout----
@@ -264,9 +276,11 @@ void syncTimeoutLog(const SyncTimeout* pMsg) {
 }
 
 void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) {
-  char* serialized = syncTimeout2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncTimeout2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncPing----
@@ -524,9 +538,11 @@ void syncPingLog(const SyncPing* pMsg) {
 }
 
 void syncPingLog2(char* s, const SyncPing* pMsg) {
-  char* serialized = syncPing2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncPing2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncPingReply----
@@ -784,9 +800,11 @@ void syncPingReplyLog(const SyncPingReply* pMsg) {
 }
 
 void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) {
-  char* serialized = syncPingReply2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncPingReply2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncClientRequest----
@@ -925,9 +943,11 @@ void syncClientRequestLog(const SyncClientRequest* pMsg) {
 }
 
 void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) {
-  char* serialized = syncClientRequest2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncClientRequest2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncRequestVote----
@@ -1074,9 +1094,11 @@ void syncRequestVoteLog(const SyncRequestVote* pMsg) {
 }
 
 void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) {
-  char* serialized = syncRequestVote2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncRequestVote2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncRequestVoteReply----
@@ -1220,9 +1242,11 @@ void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) {
 }
 
 void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) {
-  char* serialized = syncRequestVoteReply2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncRequestVoteReply2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncAppendEntries----
@@ -1333,6 +1357,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) {
     snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
 
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
     snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex);
     cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf);
 
@@ -1386,9 +1413,11 @@ void syncAppendEntriesLog(const SyncAppendEntries* pMsg) {
 }
 
 void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) {
-  char* serialized = syncAppendEntries2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncAppendEntries2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncAppendEntriesReply----
@@ -1494,6 +1523,9 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) {
     cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
     cJSON_AddItemToObject(pRoot, "destId", pDestId);
 
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
     snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
     cJSON_AddStringToObject(pRoot, "term", u64buf);
     cJSON_AddNumberToObject(pRoot, "success", pMsg->success);
@@ -1535,9 +1567,11 @@ void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) {
 }
 
 void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
-  char* serialized = syncAppendEntriesReply2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncAppendEntriesReply2Str(pMsg);
-  taosMemoryFree(serialized);
+    sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }
 
 // ---- message process SyncApplyMsg----
@@ -1686,7 +1720,339 @@ void syncApplyMsgLog(const SyncApplyMsg* pMsg) {
 }
 
 void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) {
-  char* serialized = syncApplyMsg2Str(pMsg);
+  if (gRaftDetailLog) {
-  sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = syncApplyMsg2Str(pMsg);
+    sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
+// ---------------------------------------------
+SyncSnapshotSend* syncSnapshotSendBuild(uint32_t dataLen, int32_t vgId) {
+  uint32_t          bytes = sizeof(SyncSnapshotSend) + dataLen;
+  SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_SNAPSHOT_SEND;
+  pMsg->dataLen = dataLen;
+  return pMsg;
+}
+
+void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen) {
+  assert(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg) {
+  memcpy(pMsg, buf, len);
+  assert(len == pMsg->bytes);
+  assert(pMsg->bytes == sizeof(SyncSnapshotSend) + pMsg->dataLen);
+}
+
+char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  assert(buf != NULL);
+  syncSnapshotSendSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len) {
+  uint32_t          bytes = *((uint32_t*)buf);
+  SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes);
+  assert(pMsg != NULL);
+  syncSnapshotSendDeserialize(buf, len, pMsg);
+  assert(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncSnapshotSend2RpcMsg(const SyncSnapshotSend* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncSnapshotSendSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg) {
+  syncSnapshotSendDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncSnapshotSend* pMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  assert(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex);
+    cJSON_AddStringToObject(pRoot, "lastIndex", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm);
+    cJSON_AddStringToObject(pRoot, "lastTerm", u64buf);
+
+    cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq);
+
+    cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen);
+    char* s;
+    s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen);
+    cJSON_AddStringToObject(pRoot, "data", s);
+    taosMemoryFree(s);
+    s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen);
+    cJSON_AddStringToObject(pRoot, "data2", s);
+    taosMemoryFree(s);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot);
+  return pJson;
+}
+
+char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) {
+  cJSON* pJson = syncSnapshotSend2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// for debug ----------------------
+void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) {
+  char* serialized = syncSnapshotSend2Str(pMsg);
+  printf("syncSnapshotSendPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
   taosMemoryFree(serialized);
 }
+
+void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) {
+  char* serialized = syncSnapshotSend2Str(pMsg);
+  printf("syncSnapshotSendPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) {
+  char* serialized = syncSnapshotSend2Str(pMsg);
+  sTrace("syncSnapshotSendLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncSnapshotSend2Str(pMsg);
+    sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
+// ---------------------------------------------
+SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) {
+  uint32_t         bytes = sizeof(SyncSnapshotRsp);
+  SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_SNAPSHOT_RSP;
+  return pMsg;
+}
+
+void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen) {
+  assert(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg) {
+  memcpy(pMsg, buf, len);
+  assert(len == pMsg->bytes);
+}
+
+char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  assert(buf != NULL);
+  syncSnapshotRspSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len) {
+  uint32_t         bytes = *((uint32_t*)buf);
+  SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes);
+  assert(pMsg != NULL);
+  syncSnapshotRspDeserialize(buf, len, pMsg);
+  assert(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncSnapshotRsp2RpcMsg(const SyncSnapshotRsp* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncSnapshotRspSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg) {
+  syncSnapshotRspDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncSnapshotRsp* pMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  assert(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128];
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex);
+    cJSON_AddStringToObject(pRoot, "lastIndex", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm);
+    cJSON_AddStringToObject(pRoot, "lastTerm", u64buf);
+
+    cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack);
+    cJSON_AddNumberToObject(pRoot, "code", pMsg->code);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncSnapshotRsp", pRoot);
+  return pJson;
+}
+
+char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg) {
+  cJSON* pJson = syncSnapshotRsp2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// for debug ----------------------
+void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg) {
+  char* serialized = syncSnapshotRsp2Str(pMsg);
+  printf("syncSnapshotRspPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg) {
+  char* serialized = syncSnapshotRsp2Str(pMsg);
+  printf("syncSnapshotRspPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg) {
+  char* serialized = syncSnapshotRsp2Str(pMsg);
+  sTrace("syncSnapshotRspLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncSnapshotRsp2Str(pMsg);
+    sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}

source/libs/sync/src/syncRaftCfg.c

@@ -148,6 +148,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
   cJSON *pRoot = cJSON_CreateObject();
   cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg)));
   cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy);
+  cJSON_AddNumberToObject(pRoot, "snapshotEnable", pRaftCfg->snapshotEnable);
 
   cJSON *pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "RaftCfg", pRoot);
@@ -161,7 +162,7 @@ char *raftCfg2Str(SRaftCfg *pRaftCfg) {
   return serialized;
 }
 
-int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) {
+int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
   assert(pCfg != NULL);
 
   TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
@@ -169,7 +170,8 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, int8_t isStandBy, const char *path) {
 
   SRaftCfg raftCfg;
   raftCfg.cfg = *pCfg;
-  raftCfg.isStandBy = isStandBy;
+  raftCfg.isStandBy = meta.isStandBy;
+  raftCfg.snapshotEnable = meta.snapshotEnable;
   char *s = raftCfg2Str(&raftCfg);
 
   char buf[CONFIG_FILE_LEN] = {0};
@@ -194,6 +196,9 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
   cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy");
   pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy);
 
+  cJSON *pJsonSnapshotEnable = cJSON_GetObjectItem(pJson, "snapshotEnable");
+  pRaftCfg->snapshotEnable = cJSON_GetNumberValue(pJsonSnapshotEnable);
+
   cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
   int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
   ASSERT(code == 0);

source/libs/sync/src/syncRaftLog.c

@@ -16,6 +16,23 @@
 #include "syncRaftLog.h"
 #include "wal.h"
 
+// refactor, log[0 .. n] ==> log[m .. n]
+static int32_t   raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
+static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore);
+static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore);
+static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore);
+static bool      raftLogIsEmpty(struct SSyncLogStore* pLogStore);
+static int32_t   raftLogEntryCount(struct SSyncLogStore* pLogStore);
+static bool      raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index);
+static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore);
+static SyncTerm  raftLogLastTerm(struct SSyncLogStore* pLogStore);
+static int32_t   raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
+static int32_t   raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry);
+static int32_t   raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
+
+static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry);
+
+//-------------------------------
 static SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore);
 static SyncIndex       logStoreLastIndex(SSyncLogStore* pLogStore);
 static SyncTerm        logStoreLastTerm(SSyncLogStore* pLogStore);
@@ -25,6 +42,202 @@ static int32_t         logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex from
 static int32_t         logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index);
 static SyncIndex       logStoreGetCommitIndex(SSyncLogStore* pLogStore);
 
+// refactor, log[0 .. n] ==> log[m .. n]
+static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) {
+  sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex);
+
+  // if beginIndex == 0, donot need call this funciton
+  ASSERT(beginIndex > 0);
+
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  pData->beginIndex = beginIndex;
+  walRestoreFromSnapshot(pWal, beginIndex - 1);
+  return 0;
+}
+
+int32_t raftLogResetBeginIndex(struct SSyncLogStore* pLogStore) { return 0; }
+
+static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  return pData->beginIndex;
+}
+
+static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); }
+
+static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) {
+  SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
+  SyncIndex endIndex = raftLogEndIndex(pLogStore);
+  return (endIndex < beginIndex);
+}
+
+static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) {
+  SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
+  SyncIndex endIndex = raftLogEndIndex(pLogStore);
+  int32_t   count = endIndex - beginIndex + 1;
+  return count > 0 ? count : 0;
+}
+
+static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) {
+  SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
+  SyncIndex endIndex = raftLogEndIndex(pLogStore);
+  if (index >= beginIndex && index <= endIndex) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) {
+  SyncIndex          lastIndex;
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  SyncIndex          lastVer = walGetLastVer(pWal);
+  SyncIndex          firstVer = walGetFirstVer(pWal);
+
+  if (lastVer < firstVer) {
+    // no record
+    lastIndex = -1;
+
+  } else {
+    if (firstVer >= 0) {
+      lastIndex = lastVer;
+    } else if (firstVer == -1) {
+      lastIndex = -1;
+    } else {
+      ASSERT(0);
+    }
+  }
+
+  return lastIndex;
+}
+
+static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  SyncIndex          lastVer = walGetLastVer(pWal);
+  return lastVer + 1;
+}
+
+static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
+  SyncTerm lastTerm = 0;
+  if (raftLogEntryCount(pLogStore) == 0) {
+    lastTerm = 0;
+  } else {
+    SSyncRaftEntry* pLastEntry;
+    int32_t         code = raftLogGetLastEntry(pLogStore, &pLastEntry);
+    ASSERT(code == 0);
+    if (pLastEntry != NULL) {
+      lastTerm = pLastEntry->term;
+      taosMemoryFree(pLastEntry);
+    }
+  }
+  return lastTerm;
+}
+
+static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+
+  SyncIndex writeIndex = raftLogWriteIndex(pLogStore);
+  ASSERT(pEntry->index == writeIndex);
+
+  int          code = 0;
+  SSyncLogMeta syncMeta;
+  syncMeta.isWeek = pEntry->isWeak;
+  syncMeta.seqNum = pEntry->seqNum;
+  syncMeta.term = pEntry->term;
+  code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
+  if (code != 0) {
+    int32_t     err = terrno;
+    const char* errStr = tstrerror(err);
+    int32_t     linuxErr = errno;
+    const char* linuxErrMsg = strerror(errno);
+    sError("raftLogAppendEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
+    ASSERT(0);
+  }
+
+  walFsync(pWal, true);
+
+  sTrace("sync event write index:%" PRId64, pEntry->index);
+
+  return code;
+}
+
+static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  int32_t            code;
+
+  *ppEntry = NULL;
+  if (raftLogInRange(pLogStore, index)) {
+    SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
+    ASSERT(pWalHandle != NULL);
+
+    code = walReadWithHandle(pWalHandle, index);
+    if (code != 0) {
+      int32_t     err = terrno;
+      const char* errStr = tstrerror(err);
+      int32_t     linuxErr = errno;
+      const char* linuxErrMsg = strerror(errno);
+      sError("raftLogGetEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+             linuxErrMsg);
+      ASSERT(0);
+      walCloseReadHandle(pWalHandle);
+      return code;
+    }
+
+    *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
+    ASSERT(*ppEntry != NULL);
+    (*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST;
+    (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType;
+    (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum;
+    (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek;
+    (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term;
+    (*ppEntry)->index = index;
+    ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen);
+    memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
+
+    // need to hold, do not new every time!!
+    walCloseReadHandle(pWalHandle);
+
+  } else {
+    // index not in range
+    code = 0;
+  }
+
+  return code;
+}
+
+static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  int32_t            code = walRollback(pWal, fromIndex);
+  if (code != 0) {
+    int32_t     err = terrno;
+    const char* errStr = tstrerror(err);
+    int32_t     linuxErr = errno;
+    const char* linuxErrMsg = strerror(errno);
+    sError("raftLogTruncate error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr,
+           linuxErrMsg);
+    ASSERT(0);
+  }
+  return code;
+}
+
+static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
+  *ppLastEntry = NULL;
+  if (raftLogEntryCount(pLogStore) == 0) {
+    return 0;
+  }
+  SyncIndex lastIndex = raftLogLastIndex(pLogStore);
+  int32_t   code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
+  return code;
+}
+
+//-------------------------------
 SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
   SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore));
   assert(pLogStore != NULL);
@@ -36,6 +249,16 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
   pData->pSyncNode = pSyncNode;
   pData->pWal = pSyncNode->pWal;
 
+  SyncIndex firstVer = walGetFirstVer(pData->pWal);
+  SyncIndex lastVer = walGetLastVer(pData->pWal);
+  if (firstVer >= 0) {
+    pData->beginIndex = firstVer;
+  } else if (firstVer == -1) {
+    pData->beginIndex = lastVer + 1;
+  } else {
+    ASSERT(0);
+  }
+
   pLogStore->appendEntry = logStoreAppendEntry;
   pLogStore->getEntry = logStoreGetEntry;
   pLogStore->truncate = logStoreTruncate;
@@ -43,6 +266,20 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
   pLogStore->getLastTerm = logStoreLastTerm;
   pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
   pLogStore->getCommitIndex = logStoreGetCommitIndex;
+
+  pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex;
+  pLogStore->syncLogBeginIndex = raftLogBeginIndex;
+  pLogStore->syncLogEndIndex = raftLogEndIndex;
+  pLogStore->syncLogIsEmpty = raftLogIsEmpty;
+  pLogStore->syncLogEntryCount = raftLogEntryCount;
+  pLogStore->syncLogInRange = raftLogInRange;
+  pLogStore->syncLogLastIndex = raftLogLastIndex;
+  pLogStore->syncLogLastTerm = raftLogLastTerm;
+  pLogStore->syncLogAppendEntry = raftLogAppendEntry;
+  pLogStore->syncLogGetEntry = raftLogGetEntry;
+  pLogStore->syncLogTruncate = raftLogTruncate;
+  pLogStore->syncLogWriteIndex = raftLogWriteIndex;
+
   return pLogStore;
 }
 
@@ -53,6 +290,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) {
   }
 }
 
+//-------------------------------
 int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
@@ -78,6 +316,8 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   // assert(code == 0);
 
   walFsync(pWal, true);
+
+  sTrace("sync event old write wal: %ld", pEntry->index);
   return code;
 }
 
@@ -136,7 +376,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
            linuxErrMsg);
     ASSERT(0);
   }
-  return 0;  // to avoid compiler error
+  return 0;
 }
 
 SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) {
@@ -169,7 +409,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
     sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg);
     ASSERT(0);
   }
-  return 0;  // to avoid compiler error
+  return 0;
 }
 
 SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) {
@@ -199,15 +439,32 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) {
     cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
     cJSON_AddStringToObject(pRoot, "pWal", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore));
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex);
+    cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
+
+    SyncIndex endIndex = raftLogEndIndex(pLogStore);
+    snprintf(u64buf, sizeof(u64buf), "%ld", endIndex);
+    cJSON_AddStringToObject(pRoot, "endIndex", u64buf);
+
+    int32_t count = raftLogEntryCount(pLogStore);
+    cJSON_AddNumberToObject(pRoot, "entryCount", count);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore));
+    cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore));
+    cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore));
     cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore));
+    snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore));
     cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
 
     cJSON* pEntries = cJSON_CreateArray();
     cJSON_AddItemToObject(pRoot, "pEntries", pEntries);
-    SyncIndex lastIndex = logStoreLastIndex(pLogStore);
+
-    for (SyncIndex i = 0; i <= lastIndex; ++i) {
+    for (SyncIndex i = pData->beginIndex; i <= endIndex; ++i) {
       SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i);
       cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
       syncEntryDestory(pEntry);
@@ -236,9 +493,26 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) {
     cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
     cJSON_AddStringToObject(pRoot, "pWal", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore));
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex);
+    cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
+
+    SyncIndex endIndex = raftLogEndIndex(pLogStore);
+    snprintf(u64buf, sizeof(u64buf), "%ld", endIndex);
+    cJSON_AddStringToObject(pRoot, "endIndex", u64buf);
+
+    int32_t count = raftLogEntryCount(pLogStore);
+    cJSON_AddNumberToObject(pRoot, "entryCount", count);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore));
+    cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore));
+    cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore));
     cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
-    snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore));
+    snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore));
     cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
   }
 
@@ -254,6 +528,12 @@ char* logStoreSimple2Str(SSyncLogStore* pLogStore) {
   return serialized;
 }
 
+SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) {
+  SSyncLogStoreData* pData = pLogStore->data;
+  SWal*              pWal = pData->pWal;
+  return walGetFirstVer(pWal);
+}
+
 // for debug -----------------
 void logStorePrint(SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
@@ -303,7 +583,9 @@ void logStoreSimpleLog(SSyncLogStore* pLogStore) {
 }
 
 void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) {
-  char* serialized = logStoreSimple2Str(pLogStore);
+  if (gRaftDetailLog) {
-  sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    char* serialized = logStoreSimple2Str(pLogStore);
-  taosMemoryFree(serialized);
+    sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
 }

source/libs/sync/src/syncReplication.c

@@ -16,9 +16,11 @@
 #include "syncReplication.h"
 #include "syncIndexMgr.h"
 #include "syncMessage.h"
+#include "syncRaftCfg.h"
 #include "syncRaftEntry.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
+#include "syncSnapshot.h"
 #include "syncUtil.h"
 
 // TLA+ Spec
@@ -59,6 +61,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
 
     // set prevLogIndex
     SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId);
+
     SyncIndex preLogIndex = nextIndex - 1;
 
     // set preLogTerm
@@ -113,9 +116,87 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
   return ret;
 }
 
+int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) {
+  ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
+
+  syncIndexMgrLog2("begin append entries peers pNextIndex:", pSyncNode->pNextIndex);
+  syncIndexMgrLog2("begin append entries peers pMatchIndex:", pSyncNode->pMatchIndex);
+  logStoreSimpleLog2("begin append entries peers LogStore:", pSyncNode->pLogStore);
+  {
+    SSnapshot snapshot;
+    pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+    sTrace("begin append entries peers, snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
+           snapshot.lastApplyIndex, snapshot.lastApplyTerm);
+  }
+
+  int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    SRaftId* pDestId = &(pSyncNode->peersId[i]);
+
+    // next index
+    SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId);
+
+    // pre index, pre term
+    SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex);
+    SyncTerm  preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex);
+
+    // batch optimized
+    // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex);
+
+    // prepare entry
+    SyncAppendEntries* pMsg = NULL;
+
+    SSyncRaftEntry* pEntry;
+    int32_t         code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, nextIndex, &pEntry);
+    ASSERT(code == 0);
+
+    if (pEntry != NULL) {
+      pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId);
+      ASSERT(pMsg != NULL);
+
+      // add pEntry into msg
+      uint32_t len;
+      char*    serialized = syncEntrySerialize(pEntry, &len);
+      assert(len == pEntry->bytes);
+      memcpy(pMsg->data, serialized, len);
+
+      taosMemoryFree(serialized);
+      syncEntryDestory(pEntry);
+
+    } else {
+      // no entry in log
+      pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId);
+      ASSERT(pMsg != NULL);
+    }
+
+    // prepare msg
+    ASSERT(pMsg != NULL);
+    pMsg->srcId = pSyncNode->myRaftId;
+    pMsg->destId = *pDestId;
+    pMsg->term = pSyncNode->pRaftStore->currentTerm;
+    pMsg->prevLogIndex = preLogIndex;
+    pMsg->prevLogTerm = preLogTerm;
+    pMsg->commitIndex = pSyncNode->commitIndex;
+    pMsg->privateTerm = 0;
+    // pMsg->privateTerm = syncIndexMgrGetTerm(pSyncNode->pNextIndex, pDestId);
+
+    // send msg
+    syncNodeAppendEntries(pSyncNode, pDestId, pMsg);
+    syncAppendEntriesDestroy(pMsg);
+  }
+
+  return ret;
+}
+
 int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
   // start replicate
-  int32_t ret = syncNodeAppendEntriesPeers(pSyncNode);
+  int32_t ret = 0;
+
+  if (pSyncNode->pRaftCfg->snapshotEnable) {
+    ret = syncNodeAppendEntriesPeersSnapshot(pSyncNode);
+  } else {
+    ret = syncNodeAppendEntriesPeers(pSyncNode);
+  }
   return ret;
 }
 

source/libs/sync/src/syncRequestVote.c

@@ -15,6 +15,7 @@
 
 #include "syncRequestVote.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
@@ -62,6 +63,9 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
     // maybe has already voted for pMsg->srcId
     // vote again, no harm
     raftStoreVote(ths->pRaftStore, &(pMsg->srcId));
+
+    // forbid elect for this round
+    syncNodeResetElectTimer(ths);
   }
 
   SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId);
@@ -77,3 +81,64 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
 
   return ret;
 }
+
+static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) {
+  SyncTerm  myLastTerm = syncNodeGetLastTerm(pSyncNode);
+  SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
+
+  if (pMsg->lastLogTerm > myLastTerm) {
+    return true;
+  }
+  if (pMsg->lastLogTerm == myLastTerm && pMsg->lastLogIndex >= myLastIndex) {
+    return true;
+  }
+
+  return false;
+}
+
+int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) {
+  int32_t ret = 0;
+
+  // print log
+  char logBuf[128] = {0};
+  snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVote, currentTerm:%lu", ths->pRaftStore->currentTerm);
+  syncRequestVoteLog2(logBuf, pMsg);
+
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
+    sInfo("recv SyncRequestVote maybe replica already dropped");
+    return ret;
+  }
+
+  // maybe update term
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    syncNodeUpdateTerm(ths, pMsg->term);
+  }
+  ASSERT(pMsg->term <= ths->pRaftStore->currentTerm);
+
+  bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg);
+  bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK &&
+               ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId))));
+  if (grant) {
+    // maybe has already voted for pMsg->srcId
+    // vote again, no harm
+    raftStoreVote(ths->pRaftStore, &(pMsg->srcId));
+
+    // forbid elect for this round
+    syncNodeResetElectTimer(ths);
+  }
+
+  // send msg
+  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId);
+  pReply->srcId = ths->myRaftId;
+  pReply->destId = pMsg->srcId;
+  pReply->term = ths->pRaftStore->currentTerm;
+  pReply->voteGranted = grant;
+
+  SRpcMsg rpcMsg;
+  syncRequestVoteReply2RpcMsg(pReply, &rpcMsg);
+  syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+  syncRequestVoteReplyDestroy(pReply);
+
+  return ret;
+}

source/libs/sync/src/syncRequestVoteReply.c

@@ -15,6 +15,7 @@
 
 #include "syncRequestVoteReply.h"
 #include "syncInt.h"
+#include "syncRaftCfg.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
@@ -92,3 +93,68 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg)
 
   return ret;
 }
+
+int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {
+  int32_t ret = 0;
+
+  // print log
+  char logBuf[128] = {0};
+  snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%lu", ths->pRaftStore->currentTerm);
+  syncRequestVoteReplyLog2(logBuf, pMsg);
+
+  // if already drop replica, do not process
+  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
+    sInfo("recv SyncRequestVoteReply, maybe replica already dropped");
+    return ret;
+  }
+
+  // drop stale response
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term,
+           ths->pRaftStore->currentTerm);
+    return ret;
+  }
+
+  // assert(!(pMsg->term > ths->pRaftStore->currentTerm));
+  //  no need this code, because if I receive reply.term, then I must have sent for that term.
+  //   if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //     syncNodeUpdateTerm(ths, pMsg->term);
+  //   }
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128] = {0};
+    snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, error term, receive_term:%lu current_term:%lu",
+             pMsg->term, ths->pRaftStore->currentTerm);
+    syncNodePrint2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
+
+  ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
+
+  // This tallies votes even when the current state is not Candidate,
+  // but they won't be looked at, so it doesn't matter.
+  if (ths->state == TAOS_SYNC_STATE_CANDIDATE) {
+    votesRespondAdd(ths->pVotesRespond, pMsg);
+    if (pMsg->voteGranted) {
+      // add vote
+      voteGrantedVote(ths->pVotesGranted, pMsg);
+
+      // maybe to leader
+      if (voteGrantedMajority(ths->pVotesGranted)) {
+        if (!ths->pVotesGranted->toLeader) {
+          syncNodeCandidate2Leader(ths);
+
+          // prevent to leader again!
+          ths->pVotesGranted->toLeader = true;
+        }
+      }
+    } else {
+      ;
+      // do nothing
+      // UNCHANGED <<votesGranted, voterLog>>
+    }
+  }
+
+  return ret;
+}

source/libs/sync/src/syncSnapshot.c

@@ -14,23 +14,598 @@
  */
 
 #include "syncSnapshot.h"
+#include "syncIndexMgr.h"
+#include "syncRaftLog.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "wal.h"
 
-SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode) { return NULL; }
+static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm);
 
-void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {}
+//----------------------------------
+SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) {
+  bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) &&
+                   (pSyncNode->pFsm->FpSnapshotDoRead != NULL);
 
-int32_t snapshotSend(SSyncSnapshotSender *pSender) { return 0; }
+  SSyncSnapshotSender *pSender = NULL;
+  if (condition) {
+    pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender));
+    ASSERT(pSender != NULL);
+    memset(pSender, 0, sizeof(*pSender));
 
-cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { return NULL; }
+    pSender->start = false;
+    pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID;
+    pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
+    pSender->pReader = NULL;
+    pSender->pCurrentBlock = NULL;
+    pSender->blockLen = 0;
+    pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS;
+    pSender->pSyncNode = pSyncNode;
+    pSender->replicaIndex = replicaIndex;
+    pSender->term = pSyncNode->pRaftStore->currentTerm;
+    pSender->privateTerm = taosGetTimestampMs() + 100;
+    pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot));
+    pSender->finish = false;
+  } else {
+    sError("snapshotSenderCreate cannot create sender");
+  }
+  return pSender;
+}
 
-char *snapshotSender2Str(SSyncSnapshotSender *pSender) { return NULL; }
+void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
+  if (pSender != NULL) {
+    if (pSender->pCurrentBlock != NULL) {
+      taosMemoryFree(pSender->pCurrentBlock);
+    }
+    taosMemoryFree(pSender);
+  }
+}
 
-SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode) { return NULL; }
+bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
 
-void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {}
+// begin send snapshot (current term, seq begin)
+void snapshotSenderStart(SSyncSnapshotSender *pSender) {
+  ASSERT(!snapshotSenderIsStart(pSender));
 
-int32_t snapshotReceive(SSyncSnapshotReceiver *pReceiver) { return 0; }
+  pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
+  pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
 
-cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { return NULL; }
+  // open snapshot reader
+  ASSERT(pSender->pReader == NULL);
+  int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader));
+  ASSERT(ret == 0);
 
-char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { return NULL; }
+  if (pSender->pCurrentBlock != NULL) {
+    taosMemoryFree(pSender->pCurrentBlock);
+  }
+
+  pSender->blockLen = 0;
+
+  // get current snapshot info
+  pSender->pSyncNode->pFsm->FpGetSnapshot(pSender->pSyncNode->pFsm, &(pSender->snapshot));
+
+  pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS;
+  pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
+  ++(pSender->privateTerm);
+  pSender->finish = false;
+  pSender->start = true;
+
+  // build begin msg
+  SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
+  pMsg->srcId = pSender->pSyncNode->myRaftId;
+  pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
+  pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
+  pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
+  pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
+  pMsg->seq = pSender->seq;  // SYNC_SNAPSHOT_SEQ_BEGIN
+  pMsg->privateTerm = pSender->privateTerm;
+
+  // send msg
+  SRpcMsg rpcMsg;
+  syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
+  syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
+
+  char    *msgStr = syncSnapshotSend2Str(pMsg);
+  char     host[128];
+  uint16_t port;
+  syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port);
+  sTrace("sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s", host,
+         port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, msgStr);
+  taosMemoryFree(msgStr);
+
+  syncSnapshotSendDestroy(pMsg);
+}
+
+#if 0
+// when entry in snapshot, start sender
+void snapshotSenderStart(SSyncSnapshotSender *pSender) {
+  if (!(pSender->start)) {
+    // start
+    snapshotSenderDoStart(pSender);
+    pSender->start = true;
+  } else {
+    // already start
+    ASSERT(pSender->pSyncNode->pRaftStore->currentTerm >= pSender->term);
+
+    // if current term is higher, need start again
+    if (pSender->pSyncNode->pRaftStore->currentTerm > pSender->term) {
+      // force peer rollback
+      SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
+      pMsg->srcId = pSender->pSyncNode->myRaftId;
+      pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
+      pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
+      pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
+      pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
+      pMsg->seq = SYNC_SNAPSHOT_SEQ_FORCE_CLOSE;
+
+      SRpcMsg rpcMsg;
+      syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
+      syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
+
+      char *msgStr = syncSnapshotSend2Str(pMsg);
+      sTrace("snapshot send force close seq:%d ack:%d send msg:%s", pSender->seq, pSender->ack, msgStr);
+      taosMemoryFree(msgStr);
+
+      syncSnapshotSendDestroy(pMsg);
+
+      // close reader
+      int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
+      ASSERT(ret == 0);
+      pSender->pReader = NULL;
+
+      // start again
+      snapshotSenderDoStart(pSender);
+      pSender->start = true;
+    } else {
+      // current term, do nothing
+      ASSERT(pSender->pSyncNode->pRaftStore->currentTerm == pSender->term);
+    }
+  }
+
+  char *s = snapshotSender2Str(pSender);
+  sInfo("snapshotSenderStart %s", s);
+  taosMemoryFree(s);
+}
+#endif
+
+void snapshotSenderStop(SSyncSnapshotSender *pSender) {
+  if (pSender->pReader != NULL) {
+    int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
+    ASSERT(ret == 0);
+    pSender->pReader = NULL;
+  }
+
+  if (pSender->pCurrentBlock != NULL) {
+    taosMemoryFree(pSender->pCurrentBlock);
+    pSender->pCurrentBlock = NULL;
+    pSender->blockLen = 0;
+  }
+
+  pSender->start = false;
+
+  char *s = snapshotSender2Str(pSender);
+  sInfo("snapshotSenderStop %s", s);
+  taosMemoryFree(s);
+}
+
+// when sender receiver ack, call this function to send msg from seq
+// seq = ack + 1, already updated
+int32_t snapshotSend(SSyncSnapshotSender *pSender) {
+  // free memory last time (seq - 1)
+  if (pSender->pCurrentBlock != NULL) {
+    taosMemoryFree(pSender->pCurrentBlock);
+    pSender->pCurrentBlock = NULL;
+    pSender->blockLen = 0;
+  }
+
+  // read data
+  int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotDoRead(pSender->pSyncNode->pFsm, pSender->pReader,
+                                                           &(pSender->pCurrentBlock), &(pSender->blockLen));
+  ASSERT(ret == 0);
+  if (pSender->blockLen > 0) {
+    // has read data
+  } else {
+    // read finish
+    pSender->seq = SYNC_SNAPSHOT_SEQ_END;
+  }
+
+  // build msg
+  SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId);
+  pMsg->srcId = pSender->pSyncNode->myRaftId;
+  pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
+  pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
+  pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
+  pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
+  pMsg->seq = pSender->seq;
+  pMsg->privateTerm = pSender->privateTerm;
+  memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
+
+  // send msg
+  SRpcMsg rpcMsg;
+  syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
+  syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
+
+  char    *msgStr = syncSnapshotSend2Str(pMsg);
+  char     host[128];
+  uint16_t port;
+  syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port);
+  if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
+    sTrace("sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s",
+           host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
+           msgStr);
+  } else {
+    sTrace("sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu send msg:%s",
+           host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
+           msgStr);
+  }
+  taosMemoryFree(msgStr);
+
+  syncSnapshotSendDestroy(pMsg);
+  return 0;
+}
+
+// send snapshot data from cache
+int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
+  if (pSender->pCurrentBlock != NULL) {
+    SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId);
+    pMsg->srcId = pSender->pSyncNode->myRaftId;
+    pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
+    pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
+    pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
+    pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
+    pMsg->seq = pSender->seq;
+    memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
+
+    SRpcMsg rpcMsg;
+    syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
+    syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
+
+    char    *msgStr = syncSnapshotSend2Str(pMsg);
+    char     host[128];
+    uint16_t port;
+    syncUtilU642Addr(pSender->pSyncNode->replicasId[pSender->replicaIndex].addr, host, sizeof(host), &port);
+    sTrace("sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", host, port, pSender->seq, pSender->ack,
+           msgStr);
+    taosMemoryFree(msgStr);
+
+    syncSnapshotSendDestroy(pMsg);
+  }
+  return 0;
+}
+
+cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  if (pSender != NULL) {
+    cJSON_AddNumberToObject(pRoot, "start", pSender->start);
+    cJSON_AddNumberToObject(pRoot, "seq", pSender->seq);
+    cJSON_AddNumberToObject(pRoot, "ack", pSender->ack);
+
+    snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader);
+    cJSON_AddStringToObject(pRoot, "pReader", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock);
+    cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf);
+    cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen);
+
+    if (pSender->pCurrentBlock != NULL) {
+      char *s;
+      s = syncUtilprintBin((char *)(pSender->pCurrentBlock), pSender->blockLen);
+      cJSON_AddStringToObject(pRoot, "pCurrentBlock", s);
+      taosMemoryFree(s);
+      s = syncUtilprintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen);
+      cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s);
+      taosMemoryFree(s);
+    }
+
+    cJSON *pSnapshot = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyIndex);
+    cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf);
+    snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm);
+    cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf);
+    cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS);
+    cJSON_AddStringToObject(pRoot, "sendingMS", u64buf);
+    snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode);
+    cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+    cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex);
+    snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+    snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+    cJSON_AddNumberToObject(pRoot, "finish", pSender->finish);
+  }
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot);
+  return pJson;
+}
+
+char *snapshotSender2Str(SSyncSnapshotSender *pSender) {
+  cJSON *pJson = snapshotSender2Json(pSender);
+  char  *serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// -------------------------------------
+SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, int32_t replicaIndex) {
+  bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) &&
+                   (pSyncNode->pFsm->FpSnapshotDoWrite != NULL);
+
+  SSyncSnapshotReceiver *pReceiver = NULL;
+  if (condition) {
+    pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver));
+    ASSERT(pReceiver != NULL);
+    memset(pReceiver, 0, sizeof(*pReceiver));
+
+    pReceiver->start = false;
+    pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
+    pReceiver->pWriter = NULL;
+    pReceiver->pSyncNode = pSyncNode;
+    pReceiver->replicaIndex = replicaIndex;
+    pReceiver->term = pSyncNode->pRaftStore->currentTerm;
+    pReceiver->privateTerm = 0;
+
+  } else {
+    sInfo("snapshotReceiverCreate cannot create receiver");
+  }
+
+  return pReceiver;
+}
+
+void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
+  if (pReceiver != NULL) {
+    taosMemoryFree(pReceiver);
+  }
+}
+
+bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; }
+
+// begin receive snapshot msg (current term, seq begin)
+static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) {
+  pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
+  pReceiver->privateTerm = privateTerm;
+  pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
+
+  ASSERT(pReceiver->pWriter == NULL);
+  int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter));
+  ASSERT(ret == 0);
+}
+
+// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
+// if already start, force close, start again
+void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm) {
+  if (!snapshotReceiverIsStart(pReceiver)) {
+    // start
+    snapshotReceiverDoStart(pReceiver, privateTerm);
+    pReceiver->start = true;
+
+  } else {
+    // already start
+
+    // force close, abandon incomplete data
+    int32_t ret =
+        pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
+    ASSERT(ret == 0);
+    pReceiver->pWriter = NULL;
+
+    // start again
+    snapshotReceiverDoStart(pReceiver, privateTerm);
+    pReceiver->start = true;
+
+    ASSERT(0);
+  }
+
+  char *s = snapshotReceiver2Str(pReceiver);
+  sInfo("snapshotReceiverStart %s", s);
+  taosMemoryFree(s);
+}
+
+void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) {
+  if (pReceiver->pWriter != NULL) {
+    int32_t ret =
+        pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
+    ASSERT(ret == 0);
+    pReceiver->pWriter = NULL;
+  }
+
+  pReceiver->start = false;
+
+  if (apply) {
+    ++(pReceiver->privateTerm);
+  }
+
+  char *s = snapshotReceiver2Str(pReceiver);
+  sInfo("snapshotReceiverStop %s", s);
+  taosMemoryFree(s);
+}
+
+cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  if (pReceiver != NULL) {
+    cJSON_AddNumberToObject(pRoot, "start", pReceiver->start);
+    cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack);
+
+    snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter);
+    cJSON_AddStringToObject(pRoot, "pWriter", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode);
+    cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+    cJSON_AddNumberToObject(pRoot, "replicaIndex", pReceiver->replicaIndex);
+    snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+  }
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot);
+  return pJson;
+}
+
+char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) {
+  cJSON *pJson = snapshotReceiver2Json(pReceiver);
+  char  *serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// receiver do something
+int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
+  // get receiver
+  SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
+  bool                   needRsp = false;
+  int32_t                writeCode = 0;
+
+  // state, term, seq/ack
+  if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
+    if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
+      if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
+        // begin
+        snapshotReceiverStart(pReceiver, pMsg->privateTerm);
+        pReceiver->ack = pMsg->seq;
+        needRsp = true;
+
+        char    *msgStr = syncSnapshotSend2Str(pMsg);
+        char     host[128];
+        uint16_t port;
+        syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+        sTrace("sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port,
+               pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr);
+        taosMemoryFree(msgStr);
+
+      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
+        // end, finish FSM
+        writeCode = pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen);
+        ASSERT(writeCode == 0);
+
+        pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true);
+
+        pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1);
+        char     *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore);
+        SSnapshot snapshot;
+        pSyncNode->pFsm->FpGetSnapshot(pSyncNode->pFsm, &snapshot);
+        char     host[128];
+        uint16_t port;
+        syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+        sInfo(
+            "sync event snapshot recv from %s:%d finish, update log begin index:%ld, snapshot.lastApplyIndex:%ld, "
+            "snapshot.lastApplyTerm:%lu, raft log:%s",
+            host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, logSimpleStr);
+        taosMemoryFree(logSimpleStr);
+
+        pReceiver->pWriter = NULL;
+        snapshotReceiverStop(pReceiver, true);
+        pReceiver->ack = pMsg->seq;
+        needRsp = true;
+
+        char *msgStr = syncSnapshotSend2Str(pMsg);
+        sTrace("sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host, port,
+               pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr);
+        taosMemoryFree(msgStr);
+
+      } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
+        pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false);
+        snapshotReceiverStop(pReceiver, false);
+        needRsp = false;
+
+        char     host[128];
+        uint16_t port;
+        syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+
+        char *msgStr = syncSnapshotSend2Str(pMsg);
+        sTrace("sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host,
+               port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr);
+
+        taosMemoryFree(msgStr);
+
+      } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
+        // transfering
+        if (pMsg->seq == pReceiver->ack + 1) {
+          writeCode =
+              pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen);
+          ASSERT(writeCode == 0);
+          pReceiver->ack = pMsg->seq;
+        }
+        needRsp = true;
+
+        char    *msgStr = syncSnapshotSend2Str(pMsg);
+        char     host[128];
+        uint16_t port;
+        syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
+        sTrace("sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, recv msg:%s", host,
+               port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, msgStr);
+        taosMemoryFree(msgStr);
+
+      } else {
+        ASSERT(0);
+      }
+
+      if (needRsp) {
+        SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId);
+        pRspMsg->srcId = pSyncNode->myRaftId;
+        pRspMsg->destId = pMsg->srcId;
+        pRspMsg->term = pSyncNode->pRaftStore->currentTerm;
+        pRspMsg->lastIndex = pMsg->lastIndex;
+        pRspMsg->lastTerm = pMsg->lastTerm;
+        pRspMsg->ack = pReceiver->ack;
+        pRspMsg->code = writeCode;
+        pRspMsg->privateTerm = pReceiver->privateTerm;
+
+        SRpcMsg rpcMsg;
+        syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg);
+        syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
+
+        syncSnapshotRspDestroy(pRspMsg);
+      }
+    }
+  } else {
+    syncNodeLog2("syncNodeOnSnapshotSendCb not follower", pSyncNode);
+  }
+
+  return 0;
+}
+
+// sender receives ack, set seq = ack + 1, send msg from seq
+// if ack == SYNC_SNAPSHOT_SEQ_END, stop sender
+int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
+  // get sender
+  SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId));
+  ASSERT(pSender != NULL);
+
+  // state, term, seq/ack
+  if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+    if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
+      // receiver ack is finish, close sender
+      if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
+        pSender->finish = true;
+        snapshotSenderStop(pSender);
+        return 0;
+      }
+
+      // send next msg
+      if (pMsg->ack == pSender->seq) {
+        // update sender ack
+        pSender->ack = pMsg->ack;
+        (pSender->seq)++;
+        snapshotSend(pSender);
+
+      } else if (pMsg->ack == pSender->seq - 1) {
+        snapshotReSend(pSender);
+
+      } else {
+        ASSERT(0);
+      }
+    }
+  } else {
+    syncNodeLog2("syncNodeOnSnapshotRspCb not leader", pSyncNode);
+  }
+
+  return 0;
+}

source/libs/sync/src/syncUtil.c

@@ -240,4 +240,26 @@ bool syncUtilUserRollback(tmsg_t msgType) {
     return true;
   }
   return false;
+}
+
+void syncUtilJson2Line(char* jsonStr) {
+  int p, q, len;
+  p = 0;
+  q = 1;
+  len = strlen(jsonStr);
+  while (1) {
+    if (jsonStr[q] == '\0') {
+      jsonStr[p + 1] = '\0';
+      break;
+    }
+
+    if (jsonStr[q] == '\n' || jsonStr[q] == ' ' || jsonStr[q] == '\t') {
+      q++;
+      continue;
+    } else {
+      jsonStr[p + 1] = jsonStr[q];
+      p++;
+      q++;
+    }
+  }
 }

source/libs/sync/test/CMakeLists.txt

@@ -38,6 +38,15 @@ add_executable(syncRespMgrTest "")
 add_executable(syncSnapshotTest "")
 add_executable(syncApplyMsgTest "")
 add_executable(syncConfigChangeTest "")
+add_executable(syncConfigChangeSnapshotTest "")
+add_executable(syncSnapshotSendTest "")
+add_executable(syncSnapshotRspTest "")
+add_executable(syncSnapshotSenderTest "")
+add_executable(syncSnapshotReceiverTest "")
+add_executable(syncTestTool "")
+add_executable(syncRaftLogTest "")
+add_executable(syncRaftLogTest2 "")
+add_executable(syncRaftLogTest3 "")
 
 
 target_sources(syncTest
@@ -200,6 +209,42 @@ target_sources(syncConfigChangeTest
     PRIVATE
     "syncConfigChangeTest.cpp"
 )
+target_sources(syncConfigChangeSnapshotTest
+    PRIVATE
+    "syncConfigChangeSnapshotTest.cpp"
+)
+target_sources(syncSnapshotSendTest
+    PRIVATE
+    "syncSnapshotSendTest.cpp"
+)
+target_sources(syncSnapshotRspTest
+    PRIVATE
+    "syncSnapshotRspTest.cpp"
+)
+target_sources(syncSnapshotSenderTest
+    PRIVATE
+    "syncSnapshotSenderTest.cpp"
+)
+target_sources(syncSnapshotReceiverTest
+    PRIVATE
+    "syncSnapshotReceiverTest.cpp"
+)
+target_sources(syncTestTool
+    PRIVATE
+    "syncTestTool.cpp"
+)
+target_sources(syncRaftLogTest
+    PRIVATE
+    "syncRaftLogTest.cpp"
+)
+target_sources(syncRaftLogTest2
+    PRIVATE
+    "syncRaftLogTest2.cpp"
+)
+target_sources(syncRaftLogTest3
+    PRIVATE
+    "syncRaftLogTest3.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -402,6 +447,51 @@ target_include_directories(syncConfigChangeTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncConfigChangeSnapshotTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncSnapshotSendTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncSnapshotRspTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncSnapshotSenderTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncSnapshotReceiverTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncTestTool
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncRaftLogTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncRaftLogTest2
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncRaftLogTest3
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -564,6 +654,42 @@ target_link_libraries(syncConfigChangeTest
     sync
     gtest_main
 )
+target_link_libraries(syncConfigChangeSnapshotTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncSnapshotSendTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncSnapshotRspTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncSnapshotSenderTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncSnapshotReceiverTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncTestTool
+    sync
+    gtest_main
+)
+target_link_libraries(syncRaftLogTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncRaftLogTest2
+    sync
+    gtest_main
+)
+target_link_libraries(syncRaftLogTest3
+    sync
+    gtest_main
+)
 
 
 enable_testing()

source/libs/sync/test/syncAppendEntriesReplyTest.cpp

@@ -22,6 +22,8 @@ SyncAppendEntriesReply *createMsg() {
   pMsg->destId.vgId = 100;
   pMsg->success = true;
   pMsg->matchIndex = 77;
+  pMsg->term = 33;
+  pMsg->privateTerm = 44;
   return pMsg;
 }
 

source/libs/sync/test/syncAppendEntriesTest.cpp

@@ -23,6 +23,7 @@ SyncAppendEntries *createMsg() {
   pMsg->prevLogIndex = 11;
   pMsg->prevLogTerm = 22;
   pMsg->commitIndex = 33;
+  pMsg->privateTerm = 44;
   strcpy(pMsg->data, "hello world");
   return pMsg;
 }

source/libs/sync/test/syncConfigChangeSnapshotTest.cpp

@@ -0,0 +1,366 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "os.h"
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncUtil.h"
+#include "wal.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t    gPorts[] = {7010, 7110, 7210, 7310, 7410};
+const char* gDir = "./syncReplicateTest";
+int32_t     gVgId = 1234;
+SyncIndex   gSnapshotLastApplyIndex;
+SyncIndex   gSnapshotLastApplyTerm;
+
+void init() {
+  int code = walInit();
+  assert(code == 0);
+
+  code = syncInit();
+  assert(code == 0);
+
+  sprintf(tsTempDir, "%s", ".");
+}
+
+void cleanup() { walCleanUp(); }
+
+void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  if (pFsm->FpGetSnapshot != NULL) {
+    SSnapshot snapshot;
+    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    char logBuf[256] = {0};
+    snprintf(logBuf, sizeof(logBuf),
+             "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu \n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
+             cbMeta.flag, cbMeta.term);
+    syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+  } else {
+    sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index);
+  }
+}
+
+void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm,
+           cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag);
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm,
+           cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag);
+  syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg);
+}
+
+int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex;
+  pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm;
+  return 0;
+}
+
+int32_t SnapshotStartRead(struct SSyncFSM* pFsm, void** ppReader) {
+  *ppReader = (void*)0xABCD;
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartRead== pFsm:%p, *ppReader:%p", pFsm, *ppReader);
+  sTrace("%s", logBuf);
+  return 0;
+}
+
+int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopRead== pFsm:%p, pReader:%p", pFsm, pReader);
+  sTrace("%s", logBuf);
+  return 0;
+}
+
+int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) {
+  static int readIter = 0;
+
+  if (readIter == 5) {
+    *len = 0;
+    *ppBuf = NULL;
+  } else if (readIter < 5) {
+    *len = 20;
+    *ppBuf = taosMemoryMalloc(*len);
+    snprintf((char*)*ppBuf, *len, "data iter:%d", readIter);
+  }
+
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==SnapshotDoRead== pFsm:%p, pReader:%p, *len:%d, *ppBuf:%s, readIter:%d", pFsm, pReader, *len,
+           (char*)(*ppBuf), readIter);
+  sTrace("%s", logBuf);
+
+  readIter++;
+  return 0;
+}
+
+int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void** ppWriter) {
+  *ppWriter = (void*)0xCDEF;
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStartWrite== pFsm:%p, *ppWriter:%p", pFsm, *ppWriter);
+  sTrace("%s", logBuf);
+  return 0;
+}
+
+int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter,
+           isApply);
+  sTrace("%s", logBuf);
+
+  if (isApply) {
+    gSnapshotLastApplyIndex = 10;
+    gSnapshotLastApplyTerm = 1;
+  }
+
+  return 0;
+}
+
+int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) {
+  char logBuf[256] = {0};
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotDoWrite== pFsm:%p, pWriter:%p, len:%d pBuf:%s", pFsm,
+           pWriter, len, (char*)pBuf);
+  sTrace("%s", logBuf);
+  return 0;
+}
+
+void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); }
+
+void ReConfigCb(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
+  sTrace("==callback== ==ReConfigCb== flag:0x%lX, isDrop:%d, index:%ld, code:%d, currentTerm:%lu, term:%lu",
+         cbMeta.flag, cbMeta.isDrop, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term);
+}
+
+SSyncFSM* createFsm() {
+  SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM));
+  memset(pFsm, 0, sizeof(*pFsm));
+
+  pFsm->FpCommitCb = CommitCb;
+  pFsm->FpPreCommitCb = PreCommitCb;
+  pFsm->FpRollBackCb = RollBackCb;
+
+  pFsm->FpGetSnapshot = GetSnapshotCb;
+  pFsm->FpRestoreFinishCb = RestoreFinishCb;
+  pFsm->FpSnapshotStartRead = SnapshotStartRead;
+  pFsm->FpSnapshotStopRead = SnapshotStopRead;
+  pFsm->FpSnapshotDoRead = SnapshotDoRead;
+  pFsm->FpSnapshotStartWrite = SnapshotStartWrite;
+  pFsm->FpSnapshotStopWrite = SnapshotStopWrite;
+  pFsm->FpSnapshotDoWrite = SnapshotDoWrite;
+
+  pFsm->FpReConfigCb = ReConfigCb;
+
+  return pFsm;
+}
+
+SWal* createWal(char* path, int32_t vgId) {
+  SWalCfg walCfg;
+  memset(&walCfg, 0, sizeof(SWalCfg));
+  walCfg.vgId = vgId;
+  walCfg.fsyncPeriod = 1000;
+  walCfg.retentionPeriod = 1000;
+  walCfg.rollPeriod = 1000;
+  walCfg.retentionSize = 1000;
+  walCfg.segSize = 1000;
+  walCfg.level = TAOS_WAL_FSYNC;
+  SWal* pWal = walOpen(path, &walCfg);
+  assert(pWal != NULL);
+  return pWal;
+}
+
+int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = vgId;
+  syncInfo.msgcb = &gSyncIO->msgcb;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = createFsm();
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex);
+  syncInfo.pWal = pWal;
+  syncInfo.isStandBy = isStandBy;
+  syncInfo.snapshotEnable = true;
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+
+  if (isStandBy) {
+    pCfg->myIndex = 0;
+    pCfg->replicaNum = 1;
+    pCfg->nodeInfo[0].nodePort = gPorts[myIndex];
+    taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  } else {
+    pCfg->myIndex = myIndex;
+    pCfg->replicaNum = replicaNum;
+
+    for (int i = 0; i < replicaNum; ++i) {
+      pCfg->nodeInfo[i].nodePort = gPorts[i];
+      taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn);
+      // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    }
+  }
+
+  int64_t rid = syncOpen(&syncInfo);
+  assert(rid > 0);
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
+  gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest;
+
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+
+  gSyncIO->FpOnSyncSnapshotSend = pSyncNode->FpOnSnapshotSend;
+  gSyncIO->FpOnSyncSnapshotRsp = pSyncNode->FpOnSnapshotRsp;
+
+  gSyncIO->pSyncNode = pSyncNode;
+  syncNodeRelease(pSyncNode);
+
+  return rid;
+}
+
+void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) {
+  SSyncCfg syncCfg;
+
+  syncCfg.myIndex = myIndex;
+  syncCfg.replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    syncCfg.nodeInfo[i].nodePort = gPorts[i];
+    taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn);
+  }
+
+  syncReconfig(rid, &syncCfg);
+}
+
+void usage(char* exe) {
+  printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange lastApplyTerm \n", exe);
+}
+
+SRpcMsg* createRpcMsg(int i, int count, int myIndex) {
+  SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg));
+  memset(pMsg, 0, sizeof(SRpcMsg));
+  pMsg->msgType = 9999;
+  pMsg->contLen = 256;
+  pMsg->pCont = rpcMallocCont(pMsg->contLen);
+  snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs());
+  return pMsg;
+}
+
+int main(int argc, char** argv) {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_INFO;
+  if (argc != 8) {
+    usage(argv[0]);
+    exit(-1);
+  }
+
+  int32_t replicaNum = atoi(argv[1]);
+  int32_t myIndex = atoi(argv[2]);
+  int32_t lastApplyIndex = atoi(argv[3]);
+  int32_t writeRecordNum = atoi(argv[4]);
+  bool    isStandBy = atoi(argv[5]);
+  bool    isConfigChange = atoi(argv[6]);
+  int32_t lastApplyTerm = atoi(argv[7]);
+
+  sTrace(
+      "args: replicaNum:%d, myIndex:%d, lastApplyIndex:%d, writeRecordNum:%d, isStandBy:%d, isConfigChange:%d, "
+      "lastApplyTerm:%d",
+      replicaNum, myIndex, lastApplyIndex, writeRecordNum, isStandBy, isConfigChange, lastApplyTerm);
+
+  gSnapshotLastApplyIndex = lastApplyIndex;
+  gSnapshotLastApplyTerm = lastApplyTerm;
+
+  if (!isStandBy) {
+    assert(replicaNum >= 1 && replicaNum <= 5);
+    assert(myIndex >= 0 && myIndex < replicaNum);
+    assert(lastApplyIndex >= -1);
+    assert(writeRecordNum >= 0);
+  }
+
+  init();
+  int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]);
+  assert(ret == 0);
+
+  char walPath[128];
+  snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex);
+  SWal* pWal = createWal(walPath, gVgId);
+
+  int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy);
+  assert(rid > 0);
+
+  syncStart(rid);
+
+  /*
+    if (isStandBy) {
+      syncStartStandBy(rid);
+    } else {
+      syncStart(rid);
+    }
+  */
+
+  SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid);
+  assert(pSyncNode != NULL);
+
+  if (isConfigChange) {
+    configChange(rid, 2, myIndex);
+  }
+
+  //---------------------------
+  int32_t alreadySend = 0;
+  while (1) {
+    char* s = syncNode2SimpleStr(pSyncNode);
+
+    if (alreadySend < writeRecordNum) {
+      SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
+      int32_t  ret = syncPropose(rid, pRpcMsg, false);
+      if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+        sTrace("%s value%d write not leader", s, alreadySend);
+      } else {
+        assert(ret == 0);
+        sTrace("%s value%d write ok", s, alreadySend);
+      }
+      alreadySend++;
+
+      rpcFreeCont(pRpcMsg->pCont);
+      taosMemoryFree(pRpcMsg);
+    } else {
+      sTrace("%s", s);
+    }
+
+    taosMsleep(1000);
+    taosMemoryFree(s);
+    taosMsleep(1000);
+  }
+
+  syncNodeRelease(pSyncNode);
+  syncStop(rid);
+  walClose(pWal);
+  syncIOStop();
+  cleanup();
+  return 0;
+}

source/libs/sync/test/syncConfigChangeTest.cpp

@@ -93,7 +93,6 @@ SSyncFSM* createFsm() {
   pFsm->FpGetSnapshot = GetSnapshotCb;
   pFsm->FpRestoreFinishCb = RestoreFinishCb;
 
-
   pFsm->FpReConfigCb = ReConfigCb;
 
   return pFsm;

source/libs/sync/test/syncIndexMgrTest.cpp

@@ -22,55 +22,23 @@ int32_t  replicaNum = 3;
 int32_t  myIndex = 0;
 
 SRaftId    ids[TSDB_MAX_REPLICA];
-SSyncInfo  syncInfo;
-SSyncFSM*  pFsm;
 SSyncNode* pSyncNode;
 
 SSyncNode* syncNodeInit() {
-  syncInfo.vgId = 1234;
+  pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
-  syncInfo.msgcb = &gSyncIO->msgcb;
+  memset(pSyncNode, 0, sizeof(SSyncNode));
-  syncInfo.FpSendMsg = syncIOSendMsg;
+  pSyncNode->replicaNum = replicaNum;
-  syncInfo.FpEqMsg = syncIOEqMsg;
-  syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
-
-  SSyncCfg* pCfg = &syncInfo.syncCfg;
-  pCfg->myIndex = myIndex;
-  pCfg->replicaNum = replicaNum;
-
   for (int i = 0; i < replicaNum; ++i) {
-    pCfg->nodeInfo[i].nodePort = ports[i];
+    pSyncNode->replicasId[i].addr = syncUtilAddr2U64("127.0.0.1", ports[i]);
-    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    pSyncNode->replicasId[i].vgId = 1234;
-    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+    ids[i].addr = pSyncNode->replicasId[i].addr;
+    ids[i].vgId = pSyncNode->replicasId[i].vgId;
   }
 
-  pSyncNode = syncNodeOpen(&syncInfo);
-  assert(pSyncNode != NULL);
-
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
-  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
-  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
-  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
-  gSyncIO->pSyncNode = pSyncNode;
-
   return pSyncNode;
 }
 
-SSyncNode* syncInitTest() { return syncNodeInit(); }
-
-void initRaftId(SSyncNode* pSyncNode) {
-  for (int i = 0; i < replicaNum; ++i) {
-    ids[i] = pSyncNode->replicasId[i];
-    char* s = syncUtilRaftId2Str(&ids[i]);
-    printf("raftId[%d] : %s\n", i, s);
-    taosMemoryFree(s);
-  }
-}
-
 int main(int argc, char** argv) {
   tsAsyncLog = 0;
   sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
@@ -80,58 +48,52 @@ int main(int argc, char** argv) {
     myIndex = atoi(argv[1]);
   }
 
-  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  SSyncNode* pSyncNode = syncNodeInit();
-  assert(ret == 0);
-
-  ret = syncEnvStart();
-  assert(ret == 0);
-
-  SSyncNode* pSyncNode = syncInitTest();
   assert(pSyncNode != NULL);
 
-  char* serialized = syncNode2Str(pSyncNode);
+  printf("---------------------------------------\n");
-  printf("%s\n", serialized);
-  taosMemoryFree(serialized);
-
-  initRaftId(pSyncNode);
-
   SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode);
   assert(pSyncIndexMgr != NULL);
-
-  printf("---------------------------------------\n");
   {
     char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     taosMemoryFree(serialized);
   }
+  printf("---------------------------------------\n");
 
+  printf("---------------------------------------\n");
   syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100);
   syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200);
   syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300);
-
+  // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[0], 700);
-  printf("---------------------------------------\n");
+  // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[1], 800);
+  // syncIndexMgrSetTerm(pSyncIndexMgr, &ids[2], 900);
   {
     char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     taosMemoryFree(serialized);
   }
+  printf("---------------------------------------\n");
 
   printf("---------------------------------------\n");
   for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
     SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]);
-    printf("index %d : %lu \n", i, idx);
+    // SyncTerm  term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]);
+    // printf("%d: index:%ld term:%lu \n", i, idx, term);
   }
-
-  syncIndexMgrClear(pSyncIndexMgr);
   printf("---------------------------------------\n");
+
+  printf("---------------------------------------\n");
+  syncIndexMgrClear(pSyncIndexMgr);
   {
     char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     taosMemoryFree(serialized);
   }
+  printf("---------------------------------------\n");
 
   syncIndexMgrDestroy(pSyncIndexMgr);
   return 0;

source/libs/sync/test/syncRaftCfgTest.cpp

@@ -71,7 +71,10 @@ void test3() {
   if (taosCheckExistFile(s)) {
     printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s);
   } else {
-    raftCfgCreateFile(pCfg, 7, s);
+    SRaftCfgMeta meta;
+    meta.isStandBy = 7;
+    meta.snapshotEnable = 9;
+    raftCfgCreateFile(pCfg, meta, s);
     printf("%s create json file: %s \n", (char*)__FUNCTION__, s);
   }
 
@@ -94,6 +97,7 @@ void test5() {
 
   pCfg->cfg.myIndex = taosGetTimestampSec();
   pCfg->isStandBy += 2;
+  pCfg->snapshotEnable += 3;
   raftCfgPersist(pCfg);
 
   printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex);