Merge remote-tracking branch 'origin/3.0' into feature/3.0_liaohj

include/common/taosdef.h

@@ -33,7 +33,7 @@ typedef enum {
   TSDB_SUPER_TABLE = 1,   // super table
   TSDB_CHILD_TABLE = 2,   // table created from super table
   TSDB_NORMAL_TABLE = 3,  // ordinary table
-  TSDB_STREAM_TABLE = 4,  // table created from stream computing
+  TSDB_STREAM_TABLE = 4,  // table created by stream processing
   TSDB_TEMP_TABLE = 5,    // temp table created by nest query
   TSDB_TABLE_MAX = 6
 } ETableType;
@@ -50,7 +50,12 @@ typedef enum {
   TSDB_CHECK_ITEM_MAX
 } ECheckItemType;
 
-typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig;
+typedef enum {
+  TD_ROW_DISCARD_UPDATE = 0,
+  TD_ROW_OVERWRITE_UPDATE = 1,
+  TD_ROW_PARTIAL_UPDATE = 2,
+} TDUpdateConfig;
+
 typedef enum {
   TSDB_STATIS_OK = 0,    // statis part exist and load successfully
   TSDB_STATIS_NONE = 1,  // statis part not exist
@@ -61,6 +66,12 @@ typedef enum {
   TSDB_SMA_STAT_EXPIRED = 1,  // not ready or expired
 } ETsdbSmaStat;
 
+typedef enum {
+  TSDB_SMA_TYPE_BLOCK = 0,       // Block-wise SMA
+  TSDB_SMA_TYPE_TIME_RANGE = 1,  // Time-range-wise SMA
+  TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
+} ETsdbSmaType;
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tcommon.h

@@ -140,6 +140,23 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
   return (void*)buf;
 }
 
+static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
+  if (pBlock == NULL) {
+    return;
+  }
+
+  // int32_t numOfOutput = pBlock->info.numOfCols;
+  int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
+  for (int32_t i = 0; i < sz; ++i) {
+    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
+    tfree(pColInfoData->pData);
+  }
+
+  taosArrayDestroy(pBlock->pDataBlock);
+  tfree(pBlock->pBlockAgg);
+  // tfree(pBlock);
+}
+
 static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
   int32_t tlen = 0;
   int32_t sz = 0;
@@ -182,23 +199,6 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
   return buf;
 }
 
-static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
-  if (pBlock == NULL) {
-    return;
-  }
-
-  // int32_t numOfOutput = pBlock->info.numOfCols;
-  int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
-  for (int32_t i = 0; i < sz; ++i) {
-    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
-    tfree(pColInfoData->pData);
-  }
-
-  taosArrayDestroy(pBlock->pDataBlock);
-  tfree(pBlock->pBlockAgg);
-  // tfree(pBlock);
-}
-
 static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
   if (pRsp->schemas) {
     if (pRsp->schemas->nCols) {
@@ -208,10 +208,6 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
   }
   taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock);
   pRsp->pBlockData = NULL;
-  // for (int32_t i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) {
-  // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-  // tDeleteSSDataBlock(pDataBlock);
-  //}
 }
 
 //======================================================================================================================

include/common/tmsg.h

@@ -190,7 +190,10 @@ typedef struct SEp {
 
 typedef struct {
   int32_t contLen;
-  int32_t vgId;
+  union {
+    int32_t vgId;
+    int32_t streamTaskId;
+  };
 } SMsgHead;
 
 // Submit message for one table
@@ -1139,6 +1142,17 @@ int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateS
 int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq);
 void    tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq);
 
+typedef struct {
+  char    name[TSDB_TOPIC_FNAME_LEN];
+  int64_t streamId;
+  char*   sql;
+  char*   executorMsg;
+} SMVCreateStreamReq, SMSCreateStreamReq;
+
+typedef struct {
+  int64_t streamId;
+} SMVCreateStreamRsp, SMSCreateStreamRsp;
+
 typedef struct {
   char   name[TSDB_TOPIC_FNAME_LEN];
   int8_t igExists;

include/dnode/snode/snode.h

@@ -23,6 +23,9 @@
 extern "C" {
 #endif
 
+#define SND_UNIQUE_THREAD_NUM 2
+#define SND_SHARED_THREAD_NUM 2
+
 /* ------------------------ TYPES EXPOSED ------------------------ */
 typedef struct SDnode SDnode;
 typedef struct SSnode SSnode;
@@ -80,6 +83,10 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
  */
 int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
 
+int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
+
+int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
+
 /**
  * @brief Drop a snode.
  *

include/libs/transport/trpc.h

@@ -71,6 +71,10 @@ typedef struct SRpcInit {
   // call back to keep conn or not
   bool (*pfp)(void *parent, tmsg_t msgType);
 
+  // to support Send messages multiple times on a link  
+  // 
+  void* (*mfp)(void *parent, tmsg_t msgType);
+
   void *parent;
 } SRpcInit;
 
@@ -89,6 +93,9 @@ void    rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp)
 int     rpcReportProgress(void *pConn, char *pCont, int contLen);
 void    rpcCancelRequest(int64_t rid);
 
+// just release client conn to rpc instance, no close sock
+void rpcReleaseHandle(void *handle);   
+
 void rpcRefHandle(void *handle, int8_t type);
 void rpcUnrefHandle(void *handle, int8_t type);
 

include/os/osLocale.h

@@ -17,12 +17,16 @@
 #define _TD_OS_LOCALE_H_
 
 #include "os.h"
-#include "osString.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define setlocale SETLOCALE_FUNC_TAOS_FORBID
+#endif
+
 char *taosCharsetReplace(char *charsetstr);
 void  taosGetSystemLocale(char *outLocale, char *outCharset);
 void  taosSetSystemLocale(const char *inLocale, const char *inCharSet);

include/util/taoserror.h

@@ -353,6 +353,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_MESSED_MSG                TAOS_DEF_ERROR_CODE(0, 0x0614)
 #define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0615)
 #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0616)
+#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x0617)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)

include/util/tuuid.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+#include "taoserror.h"
+#include "thash.h"
+
+/**
+ * Generate an non-negative signed 32bit id
+ *+------------+-----+-----------+---------------+
+ *| uid|localIp| PId | timestamp | serial number |
+ *+------------+-----+-----------+---------------+
+ *|  6bit      |6bit |   12bit   |      8bit     |
+ *+------------+-----+-----------+---------------+
+ * @return
+ */
+int32_t tGenIdPI32(void);
+
+/**
+ * Generate an non-negative signed 64bit id
+ *+------------+-----+-----------+---------------+
+ *| uid|localIp| PId | timestamp | serial number |
+ *+------------+-----+-----------+---------------+
+ *| 12bit      |12bit|24bit      |16bit          |
+ *+------------+-----+-----------+---------------+
+ * @return
+ */
+int64_t tGenIdPI64(void);

source/common/src/tmsg.c

@@ -2666,7 +2666,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
-
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -90,9 +90,11 @@ typedef struct {
   int32_t      refCount;
   int8_t       deployed;
   int8_t       dropped;
+  int8_t       uniqueWorkerInUse;
   SSnode      *pSnode;
   SRWLatch     latch;
-  SDnodeWorker writeWorker;
+  SArray      *uniqueWorkers;  // SArray<SDnodeWorker*>
+  SDnodeWorker sharedWorker;
 } SSnodeMgmt;
 
 typedef struct {
@@ -153,4 +155,4 @@ int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
 }
 #endif
 
-#endif /*_TD_DND_ENV_H_*/
+#endif /*_TD_DND_ENV_H_*/

source/dnode/mgmt/impl/inc/dndInt.h

@@ -70,4 +70,4 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
 }
 #endif
 
-#endif /*_TD_DND_INT_H_*/
+#endif /*_TD_DND_INT_H_*/

source/dnode/mgmt/impl/src/dndSnode.c

@@ -19,7 +19,20 @@
 #include "dndTransport.h"
 #include "dndWorker.h"
 
-static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
+typedef struct {
+  int32_t     vgId;
+  int32_t     refCount;
+  int32_t     snVersion;
+  int8_t      dropped;
+  char       *path;
+  SSnode     *pImpl;
+  STaosQueue *pSharedQ;
+  STaosQueue *pUniqueQ;
+} SSnodeObj;
+
+static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg);
+
+static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs);
 
 static SSnode *dndAcquireSnode(SDnode *pDnode) {
   SSnodeMgmt *pMgmt = &pDnode->smgmt;
@@ -152,8 +165,21 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
 
 static int32_t dndStartSnodeWorker(SDnode *pDnode) {
   SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, dndProcessSnodeQueue) != 0) {
+  pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *));
-    dError("failed to start snode write worker since %s", terrstr());
+  for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
+    SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker));
+    if (pUniqueWorker == NULL) {
+      return -1;
+    }
+    if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) {
+      dError("failed to start snode unique worker since %s", terrstr());
+      return -1;
+    }
+    taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker);
+  }
+  if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM,
+                    SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) {
+    dError("failed to start snode shared worker since %s", terrstr());
     return -1;
   }
 
@@ -169,9 +195,13 @@ static void dndStopSnodeWorker(SDnode *pDnode) {
 
   while (pMgmt->refCount > 0) {
     taosMsleep(10);
-  } 
+  }
 
-  dndCleanupWorker(&pMgmt->writeWorker);
+  for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
+    SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i);
+    dndCleanupWorker(worker);
+  }
+  taosArrayDestroy(pMgmt->uniqueWorkers);
 }
 
 static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
@@ -292,17 +322,36 @@ int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
   }
 }
 
-static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
   SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  SRpcMsg    *pRsp = NULL;
   int32_t     code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
 
   SSnode *pSnode = dndAcquireSnode(pDnode);
   if (pSnode != NULL) {
-    code = sndProcessMsg(pSnode, pMsg, &pRsp);
+    for (int32_t i = 0; i < numOfMsgs; i++) {
+      SRpcMsg *pMsg = NULL;
+      taosGetQitem(qall, (void **)&pMsg);
+
+      sndProcessUMsg(pSnode, pMsg);
+
+      rpcFreeCont(pMsg->pCont);
+      taosFreeQitem(pMsg);
+    }
+  }
+  dndReleaseSnode(pDnode, pSnode);
+}
+
+static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+  SSnodeMgmt *pMgmt = &pDnode->smgmt;
+  int32_t     code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
+
+  SSnode *pSnode = dndAcquireSnode(pDnode);
+  if (pSnode != NULL) {
+    code = sndProcessSMsg(pSnode, pMsg);
   }
   dndReleaseSnode(pDnode, pSnode);
 
+#if 0
   if (pMsg->msgType & 1u) {
     if (pRsp != NULL) {
       pRsp->ahandle = pMsg->ahandle;
@@ -314,11 +363,58 @@ static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
       rpcSendResponse(&rpcRsp);
     }
   }
+#endif
 
   rpcFreeCont(pMsg->pCont);
   taosFreeQitem(pMsg);
 }
 
+static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) {
+  SMsgHead *pHead = pMsg->pCont;
+  pHead->streamTaskId = htonl(pHead->streamTaskId);
+  return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM;
+}
+
+static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) {
+  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
+
+  SSnode *pSnode = dndAcquireSnode(pDnode);
+  if (pSnode != NULL) {
+    int32_t       index = dndGetSWIdFromMsg(pMsg);
+    SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index);
+    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
+  }
+
+  dndReleaseSnode(pDnode, pSnode);
+
+  if (code != 0) {
+    if (pMsg->msgType & 1u) {
+      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
+      rpcSendResponse(&rsp);
+    }
+    rpcFreeCont(pMsg->pCont);
+  }
+}
+
+static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) {
+  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
+
+  SSnode *pSnode = dndAcquireSnode(pDnode);
+  if (pSnode != NULL) {
+    SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0);
+    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
+  }
+  dndReleaseSnode(pDnode, pSnode);
+
+  if (code != 0) {
+    if (pMsg->msgType & 1u) {
+      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
+      rpcSendResponse(&rsp);
+    }
+    rpcFreeCont(pMsg->pCont);
+  }
+}
+
 static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
   int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
 
@@ -337,8 +433,16 @@ static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpc
   }
 }
 
-void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.writeWorker, pMsg);
+  dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg);
+}
+
+void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
+}
+
+void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
 }
 
 int32_t dndInitSnode(SDnode *pDnode) {

source/dnode/mgmt/impl/src/dndWorker.c

@@ -109,4 +109,4 @@ int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen)
   }
 
   return 0;
-}
+}

source/dnode/mnode/impl/src/mnode.c

@@ -30,6 +30,7 @@
 #include "mndShow.h"
 #include "mndSnode.h"
 #include "mndStb.h"
+#include "mndStream.h"
 #include "mndSubscribe.h"
 #include "mndSync.h"
 #include "mndTelem.h"
@@ -220,6 +221,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;

source/dnode/snode/inc/sndInt.h

@@ -30,41 +30,42 @@ extern "C" {
 #endif
 
 enum {
-  STREAM_STATUS__READY = 1,
+  STREAM_STATUS__RUNNING = 1,
   STREAM_STATUS__STOPPED,
   STREAM_STATUS__CREATING,
   STREAM_STATUS__STOPING,
-  STREAM_STATUS__RESUMING,
+  STREAM_STATUS__RESTORING,
   STREAM_STATUS__DELETING,
 };
 
 enum {
-  STREAM_RUNNER__RUNNING = 1,
+  STREAM_TASK_STATUS__RUNNING = 1,
-  STREAM_RUNNER__STOP,
+  STREAM_TASK_STATUS__STOP,
 };
 
+typedef struct {
+  SHashObj* pHash;  // taskId -> streamTask
+} SStreamMeta;
+
 typedef struct SSnode {
-  SSnodeOpt cfg;
+  SStreamMeta* pMeta;
+  SSnodeOpt    cfg;
 } SSnode;
 
 typedef struct {
   int64_t streamId;
+  int32_t taskId;
   int32_t IdxInLevel;
   int32_t level;
-} SStreamInfo;
+} SStreamTaskInfo;
 
 typedef struct {
-  SStreamInfo meta;
+  SStreamTaskInfo meta;
-  int8_t      status;
+  int8_t          status;
-  void*       executor;
+  void*           executor;
-  STaosQueue* queue;
+  void*           stateStore;
-  void*       stateStore;
   // storage handle
-} SStreamRunner;
+} SStreamTask;
-
-typedef struct {
-  SHashObj* pHash;
-} SStreamMeta;
 
 int32_t sndCreateStream();
 int32_t sndDropStream();

source/dnode/snode/src/snode.c

@@ -14,6 +14,7 @@
  */
 
 #include "sndInt.h"
+#include "tuuid.h"
 
 SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
   SSnode *pSnode = calloc(1, sizeof(SSnode));
@@ -31,3 +32,25 @@ int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 }
 
 void sndDestroy(const char *path) {}
+
+static int32_t sndDeployTask(SSnode *pSnode, SRpcMsg *pMsg) {
+  SStreamTask *task = malloc(sizeof(SStreamTask));
+  if (task == NULL) {
+    return -1;
+  }
+  task->meta.taskId = tGenIdPI32();
+  taosHashPut(pSnode->pMeta->pHash, &task->meta.taskId, sizeof(int32_t), &task, sizeof(void *));
+  return 0;
+}
+
+int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
+  // stream deployment
+  // stream stop/resume
+  // operator exec
+  return 0;
+}
+
+int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
+  // operator exec
+  return 0;
+}

source/dnode/vnode/inc/tsdb.h

@@ -95,6 +95,7 @@ int    tsdbCommit(STsdb *pTsdb);
  * @return int32_t
  */
 int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg);
+int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg);
 
 /**
  * @brief Insert RSma(Time-range-wise Rollup SMA) data.
@@ -105,6 +106,12 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg);
  */
 int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
 
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
+int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
+                            int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey,
+                            int32_t nMaxResult);
+
+
 // STsdbCfg
 int  tsdbOptionsInit(STsdbCfg *);
 void tsdbOptionsClear(STsdbCfg *);

source/dnode/vnode/src/inc/tsdbDBDef.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_DB_DEF_H_
+#define _TD_TSDB_DB_DEF_H_
+
+#include "db.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SDBFile SDBFile;
+typedef DB_ENV*        TDBEnv;
+
+struct SDBFile {
+  DB*   pDB;
+  char* path;
+};
+
+int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
+void    tsdbCloseDBF(SDBFile* pDBF);
+int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path);
+void    tsdbCloseBDBEnv(DB_ENV* pEnv);
+int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize);
+void*   tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TSDB_DB_DEF_H_*/

source/dnode/vnode/src/inc/tsdbDef.h

@@ -27,6 +27,7 @@
 #include "ttime.h"
 
 #include "tsdb.h"
+#include "tsdbDBDef.h"
 #include "tsdbCommit.h"
 #include "tsdbFS.h"
 #include "tsdbFile.h"
@@ -37,12 +38,15 @@
 #include "tsdbReadImpl.h"
 #include "tsdbSma.h"
 
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 struct STsdb {
   int32_t               vgId;
+  bool                  repoLocked;
+  pthread_mutex_t       mutex;
   char *                path;
   STsdbCfg              config;
   STsdbMemTable *       mem;
@@ -52,12 +56,17 @@ struct STsdb {
   STsdbFS *             fs;
   SMeta *               pMeta;
   STfs *                pTfs;
-  SSmaStat *            pSmaStat;
+  SSmaEnv *             pTSmaEnv;
+  SSmaEnv *             pRSmaEnv;
 };
 
-#define REPO_ID(r) ((r)->vgId)
+#define REPO_ID(r)        ((r)->vgId)
-#define REPO_CFG(r) (&(r)->config)
+#define REPO_CFG(r)       (&(r)->config)
-#define REPO_FS(r) (r)->fs
+#define REPO_FS(r)        (r)->fs
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+
+int tsdbLockRepo(STsdb *pTsdb);
+int tsdbUnlockRepo(STsdb *pTsdb);
 
 static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) {
   return pTable->pSchema;

source/dnode/vnode/src/inc/tsdbFile.h

@@ -329,21 +329,23 @@ static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
 // =============== SDFileSet
 typedef struct {
   int      fid;
-  int8_t   state;    // -128~127
+  int8_t   state;  // -128~127
-  uint8_t  ver;      // 0~255, DFileSet version
+  uint8_t  ver;    // 0~255, DFileSet version
   uint16_t reserve;
   SDFile   files[TSDB_FILE_MAX];
 } SDFileSet;
 
 typedef struct {
-  int     fid;
+  int      fid;
-  int8_t  state;
+  int8_t   state;
-  uint8_t ver;
+  uint8_t  ver;
+  uint16_t reserve;
 #if 0
   SDFInfo   info;
 #endif
   STfsFile  f;
   TdFilePtr pFile;
+
 } SSFile;  // files split by days with fid
 
 #define TSDB_LATEST_FSET_VER 0

source/dnode/vnode/src/inc/tsdbSma.h

@@ -17,27 +17,29 @@
 #define _TD_TSDB_SMA_H_
 
 typedef struct SSmaStat SSmaStat;
+typedef struct SSmaEnv  SSmaEnv;
 
-// insert/update interface
+struct SSmaEnv {
-int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
+  pthread_rwlock_t lock;
-int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
+  TDBEnv           dbEnv;
+  char *           path;
+  SSmaStat *       pStat;
+};
 
+#define SMA_ENV_LOCK(env)       ((env)->lock)
+#define SMA_ENV_ENV(env)        ((env)->dbEnv)
+#define SMA_ENV_PATH(env)       ((env)->path)
+#define SMA_ENV_STAT(env)       ((env)->pStat)
+#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
 
-// query interface
+void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-// TODO: This is the basic params, and should wrap the params to a queryHandle.
+void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult);
-
-// management interface
-int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg);
-int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
 #if 0
 int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
 int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
 #endif
 
 // internal func
-
-
 static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) {
   int32_t len = 0;
   len += taosEncodeFixedI64(pData, tableUid);
@@ -46,4 +48,31 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId,
   return len;
 }
 
+static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) {
+  int code = pthread_rwlock_rdlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) {
+  int code = pthread_rwlock_wrlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) {
+  int code = pthread_rwlock_unlock(&(pEnv->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
 #endif /* _TD_TSDB_SMA_H_ */

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -231,30 +231,31 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   void *pBuf = NULL, *qBuf = NULL;
   DBT   key1 = {0}, value1 = {0};
 
-  {
+  // save sma info
-    // save sma info
+  int32_t len = tEncodeTSma(NULL, pSmaCfg);
-    int32_t len = tEncodeTSma(NULL, pSmaCfg);
+  pBuf = calloc(len, 1);
-    pBuf = calloc(len, 1);
+  if (pBuf == NULL) {
-    if (pBuf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
-      return -1;
-    }
-
-    key1.data = (void *)&pSmaCfg->indexUid;
-    key1.size = sizeof(pSmaCfg->indexUid);
-
-    qBuf = pBuf;
-    tEncodeTSma(&qBuf, pSmaCfg);
-
-    value1.data = pBuf;
-    value1.size = POINTER_DISTANCE(qBuf, pBuf);
-    value1.app_data = pSmaCfg;
   }
 
+  key1.data = (void *)&pSmaCfg->indexUid;
+  key1.size = sizeof(pSmaCfg->indexUid);
+
+  qBuf = pBuf;
+  tEncodeTSma(&qBuf, pSmaCfg);
+
+  value1.data = pBuf;
+  value1.size = POINTER_DISTANCE(qBuf, pBuf);
+  value1.app_data = pSmaCfg;
+
   metaDBWLock(pMeta->pDB);
   pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0);
   metaDBULock(pMeta->pDB);
 
+  // release
+  tfree(pBuf);
+
   return 0;
 }
 

source/dnode/vnode/src/tq/tqRead.c

@@ -83,8 +83,8 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
 }
 
 int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) {
-  /*int32_t         sversion = pHandle->pBlock->sversion;*/
+  // currently only rows are used
-  /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/
+
   pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList);
   pBlockInfo->rows = pHandle->pBlock->numOfRows;
   pBlockInfo->uid = pHandle->pBlock->uid;

source/dnode/vnode/src/tsdb/tsdbBDBImpl.c

@@ -12,3 +12,162 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#define ALLOW_FORBID_FUNC
+#include "db.h"
+
+#include "taoserror.h"
+#include "tcoding.h"
+#include "thash.h"
+#include "tsdbDBDef.h"
+#include "tsdbLog.h"
+
+#define IMPL_WITH_LOCK 1
+
+static int  tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup);
+static void tsdbCloseBDBDb(DB *pDB);
+
+#define BDB_PERR(info, code) fprintf(stderr, "%s:%d " info " reason: %s\n", __FILE__, __LINE__, db_strerror(code))
+
+int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile *pDBF) {
+  // TDBEnv is shared by a group of SDBFile
+  if (!pEnv) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
+
+  // Open DBF
+  if (tsdbOpenBDBDb(&(pDBF->pDB), pEnv, pDBF->path, false) < 0) {
+    terrno = TSDB_CODE_TDB_INIT_FAILED;
+    tsdbCloseBDBDb(pDBF->pDB);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseDBF(SDBFile *pDBF) {
+  if (pDBF->pDB) {
+    tsdbCloseBDBDb(pDBF->pDB);
+    pDBF->pDB = NULL;
+  }
+  tfree(pDBF->path);
+}
+
+int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) {
+  int     ret = 0;
+  DB_ENV *pEnv = NULL;
+
+  if (path == NULL) return 0;
+
+  ret = db_env_create(&pEnv, 0);
+  if (ret != 0) {
+    BDB_PERR("Failed to create tsdb env", ret);
+    return -1;
+  }
+
+  ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0);
+  if (ret != 0) {
+    // BDB_PERR("Failed to open tsdb env", ret);
+    tsdbWarn("Failed to open tsdb env for path %s since %d", path ? path : "NULL", ret);
+    return -1;
+  }
+
+  *ppEnv = pEnv;
+
+  return 0;
+}
+
+void tsdbCloseBDBEnv(DB_ENV *pEnv) {
+  if (pEnv) {
+    pEnv->close(pEnv, 0);
+  }
+}
+
+static int tsdbOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName, bool isDup) {
+  int ret;
+  DB *pDB;
+
+  ret = db_create(&(pDB), pEnv, 0);
+  if (ret != 0) {
+    BDB_PERR("Failed to create DBP", ret);
+    return -1;
+  }
+
+  if (isDup) {
+    ret = pDB->set_flags(pDB, DB_DUPSORT);
+    if (ret != 0) {
+      BDB_PERR("Failed to set DB flags", ret);
+      return -1;
+    }
+  }
+
+  ret = pDB->open(pDB, NULL, pFName, NULL, DB_BTREE, DB_CREATE, 0);
+  if (ret) {
+    BDB_PERR("Failed to open DBF", ret);
+    return -1;
+  }
+
+  *ppDB = pDB;
+
+  return 0;
+}
+
+static void tsdbCloseBDBDb(DB *pDB) {
+  if (pDB) {
+    pDB->close(pDB, 0);
+  }
+}
+
+int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *key, uint32_t keySize, void *data, uint32_t dataSize) {
+  int ret;
+  DBT key1 = {0}, value1 = {0};
+
+  key1.data = key;
+  key1.size = keySize;
+
+  value1.data = data;
+  value1.size = dataSize;
+
+  // TODO: lock
+  ret = pDBF->pDB->put(pDBF->pDB, NULL, &key1, &value1, 0);
+  if (ret) {
+    BDB_PERR("Failed to put data to DBF", ret);
+    // TODO: unlock
+    return -1;
+  }
+  // TODO: unlock
+
+  return 0;
+}
+
+void *tsdbGetSmaDataByKey(SDBFile *pDBF, void* key, uint32_t keySize, uint32_t *valueSize) {
+  void *result = NULL;
+  DBT   key1 = {0};
+  DBT   value1 = {0};
+  int   ret;
+
+  // Set key/value
+  key1.data = key;
+  key1.size = keySize;
+
+  // Query
+  // TODO: lock
+  ret = pDBF->pDB->get(pDBF->pDB, NULL, &key1, &value1, 0);
+  // TODO: unlock
+  if (ret != 0) {
+    return NULL;
+  }
+
+  result = calloc(1, value1.size);
+
+  if (result == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  *valueSize = value1.size;
+  memcpy(result, value1.data, value1.size);
+
+  return result;
+}

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -80,6 +80,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
   pTsdb->pmaf = pMAF;
   pTsdb->pMeta = pMeta;
   pTsdb->pTfs = pTfs;
+  pTsdb->pTSmaEnv = NULL;
+  pTsdb->pRSmaEnv = NULL;
 
   pTsdb->fs = tsdbNewFS(pTsdbCfg);
 
@@ -88,8 +90,9 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
 
 static void tsdbFree(STsdb *pTsdb) {
   if (pTsdb) {
+    tsdbFreeSmaEnv(pTsdb->pRSmaEnv);
+    tsdbFreeSmaEnv(pTsdb->pTSmaEnv);
     tsdbFreeFS(pTsdb->fs);
-    tsdbDestroySmaState(pTsdb->pSmaStat);
     tfree(pTsdb->path);
     free(pTsdb);
   }
@@ -105,6 +108,30 @@ static void tsdbCloseImpl(STsdb *pTsdb) {
   tsdbCloseFS(pTsdb);
   // TODO
 }
+
+int tsdbLockRepo(STsdb *pTsdb) {
+  int code = pthread_mutex_lock(&pTsdb->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  pTsdb->repoLocked = true;
+  return 0;
+}
+
+int tsdbUnlockRepo(STsdb *pTsdb) {
+  ASSERT(IS_REPO_LOCKED(pTsdb));
+  pTsdb->repoLocked = false;
+  int code = pthread_mutex_unlock(&pTsdb->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
 #if 0
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -15,7 +15,9 @@
 
 #include "tsdbDef.h"
 
+#undef SMA_PRINT_DEBUG_LOG
 #define SMA_STORAGE_TSDB_DAYS   30
+#define SMA_STORAGE_TSDB_TIMES  30
 #define SMA_STORAGE_SPLIT_HOURS 24
 #define SMA_KEY_LEN             18  // tableUid_colId_TSKEY 8+2+8
 
@@ -23,7 +25,7 @@
 #define SMA_STATE_ITEM_HASH_SLOT 32
 
 #define SMA_TEST_INDEX_NAME "smaTestIndexName"  // TODO: just for test
-#define SMA_TEST_INDEX_UID  123456              // TODO: just for test
+#define SMA_TEST_INDEX_UID  2000000001          // TODO: just for test
 typedef enum {
   SMA_STORAGE_LEVEL_TSDB = 0,     // use days of self-defined  e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2t200.dat
   SMA_STORAGE_LEVEL_DFILESET = 1  // use days of TS data       e.g. vnode${N}/tsdb/rsma/sma_index_uid/v2r200.dat
@@ -31,23 +33,22 @@ typedef enum {
 
 typedef struct {
   STsdb * pTsdb;
-  char *  pDFile;     // TODO: use the real DFile type, not char*
+  SDBFile dFile;
-  int32_t interval;   // interval with the precision of DB
+  int32_t interval;  // interval with the precision of DB
-  // TODO
 } STSmaWriteH;
 
 typedef struct {
   int32_t iter;
+  int32_t fid;
 } SmaFsIter;
 typedef struct {
   STsdb *   pTsdb;
-  char *    pDFile;     // TODO: use the real DFile type, not char*
+  SDBFile   dFile;
   int32_t   interval;   // interval with the precision of DB
   int32_t   blockSize;  // size of SMA block item
   int8_t    storageLevel;
   int8_t    days;
   SmaFsIter smaFsIter;
-  // TODO
 } STSmaReadH;
 
 typedef struct {
@@ -68,18 +69,117 @@ struct SSmaStat {
 };
 
 // declaration of static functions
-static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData);
+static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
-static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData);
+static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
-static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit);
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
-static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData);
+static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
-static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen);
+                                   int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey,
+                                   int32_t nMaxResult);
+static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg);
+
+static int32_t  tsdbInitSmaStat(SSmaStat **pSmaStat);
+static int32_t  tsdbDestroySmaState(SSmaStat *pSmaStat);
+static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path);
+static int32_t  tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv);
+static int32_t  tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData);
+static void     tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH);
+static int32_t  tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit);
+static int32_t  tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
+static int32_t  tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData);
+static int32_t  tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen);
 
 static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
+static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel);
 static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid);
+static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey);
+static bool    tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
 
-static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData);
+static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path) {
-static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin);
+  SSmaEnv *pEnv = NULL;
-static bool    tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin);
+
+  pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv));
+  if (pEnv == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  int code = pthread_rwlock_init(&(pEnv->lock), NULL);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    free(pEnv);
+    return NULL;
+  }
+
+  ASSERT(path && (strlen(path) > 0));
+  pEnv->path = strdup(path);
+  if (pEnv->path == NULL) {
+    tsdbFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) {
+    tsdbFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  if (tsdbOpenBDBEnv(&pEnv->dbEnv, pEnv->path) != TSDB_CODE_SUCCESS) {
+    tsdbFreeSmaEnv(pEnv);
+    return NULL;
+  }
+
+  return pEnv;
+}
+
+static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SSmaEnv **pEnv) {
+  if (!pEnv) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return TSDB_CODE_FAILED;
+  }
+
+  if (pEnv && *pEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (tsdbLockRepo(pTsdb) != 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (*pEnv == NULL) {
+    if ((*pEnv = tsdbNewSmaEnv(pTsdb, path)) == NULL) {
+      tsdbUnlockRepo(pTsdb);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  if (tsdbUnlockRepo(pTsdb) != 0) {
+    tsdbFreeSmaEnv(*pEnv);
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaEnv
+ * @return int32_t
+ */
+void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) {
+  if (pSmaEnv) {
+    tsdbDestroySmaState(pSmaEnv->pStat);
+    tfree(pSmaEnv->pStat);
+    tfree(pSmaEnv->path);
+    pthread_rwlock_destroy(&(pSmaEnv->lock));
+    tsdbCloseBDBEnv(pSmaEnv->dbEnv);
+  }
+}
+
+void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) {
+  tsdbDestroySmaEnv(pSmaEnv);
+  tfree(pSmaEnv);
+  return NULL;
+}
 
 static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
   ASSERT(pSmaStat != NULL);
@@ -125,6 +225,12 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
   return pItem;
 }
 
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaStat
+ * @return int32_t
+ */
 int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
   if (pSmaStat) {
     // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
@@ -135,7 +241,6 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
       item = taosHashIterate(pSmaStat->smaStatItems, item);
     }
     taosHashCleanup(pSmaStat->smaStatItems);
-    free(pSmaStat);
   }
 }
 
@@ -143,22 +248,35 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
  * @brief Update expired window according to msg from stream computing module.
  *
  * @param pTsdb
+ * @param smaType ETsdbSmaType
  * @param msg
  * @return int32_t
  */
-int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
+int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
-  if (msg == NULL) {
+  STsdbCfg *pCfg = REPO_CFG(pTsdb);
+  SSmaEnv * pEnv = NULL;
+
+  if (!msg || !pTsdb->pMeta) {
+    terrno = TSDB_CODE_INVALID_PTR;
     return TSDB_CODE_FAILED;
   }
 
-  // lazy mode
+  char smaPath[TSDB_FILENAME_LEN] = "/proj/.sma/";
-  if (tsdbInitSmaStat(&pTsdb->pSmaStat) != TSDB_CODE_SUCCESS) {
+  if (tsdbInitSmaEnv(pTsdb, smaPath, &pEnv) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_FAILED;
   }
 
+  if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
+    pTsdb->pTSmaEnv = pEnv;
+  } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+    pTsdb->pRSmaEnv = pEnv;
+  } else {
+    ASSERT(0);
+  }
+
   // TODO: decode the msg => start
-  int64_t       indexUid = SMA_TEST_INDEX_UID;
+  int64_t indexUid = SMA_TEST_INDEX_UID;
-  const char *  indexName = SMA_TEST_INDEX_NAME;
+  // const char *  indexName = SMA_TEST_INDEX_NAME;
   const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
   TSKEY         expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
   int64_t       now = taosGetTimestampMs();
@@ -167,9 +285,9 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
   }
 
   // TODO: decode the msg <= end
-  SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems;
+  SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
 
-  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName));
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
   if (pItem == NULL) {
     pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED);  // TODO use the real state
     if (pItem == NULL) {
@@ -181,20 +299,28 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
     // cache smaMeta
     STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid);
     if (pSma == NULL) {
+      terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
       taosHashCleanup(pItem->expiredWindows);
       free(pItem);
+      tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid,
+               tstrerror(terrno));
       return TSDB_CODE_FAILED;
     }
     pItem->pSma = pSma;
 
     // TODO: change indexName to indexUid
-    if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) {
+    if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) {
       // If error occurs during put smaStatItem, free the resources of pItem
       taosHashCleanup(pItem->expiredWindows);
       free(pItem);
       return TSDB_CODE_FAILED;
     }
   }
+#if 0
+  SSmaStatItem *pItem1 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
+  int size1 = taosHashGetSize(pItem1->expiredWindows);
+  tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d before hashPut", REPO_ID(pTsdb), indexUid, size1);
+#endif
 
   int8_t state = TSDB_SMA_STAT_EXPIRED;
   for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
@@ -207,21 +333,28 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
       // windows failed to put into hash table.
       taosHashCleanup(pItem->expiredWindows);
       tfree(pItem->pSma);
-      taosHashRemove(pItemsHash, indexName, sizeof(indexName));
+      taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
       return TSDB_CODE_FAILED;
     }
   }
 
+#if 0
+  SSmaStatItem *pItem2 = (SSmaStatItem *)taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
+  int size2 = taosHashGetSize(pItem1->expiredWindows);
+  tsdbWarn("vgId:%d smaIndex %" PRIi64 " size is %d after hashPut", REPO_ID(pTsdb), indexUid, size2);
+#endif
+
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, int64_t indexUid, TSKEY skey) {
+static int32_t tsdbResetExpiredWindow(SSmaStat *pStat, int64_t indexUid, TSKEY skey) {
   SSmaStatItem *pItem = NULL;
 
-  if (pTsdb->pSmaStat && pTsdb->pSmaStat->smaStatItems) {
+  // TODO: If HASH_ENTRY_LOCK used, whether rwlock needed to handle cases of removing hashNode?
-    pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, &indexUid, sizeof(indexUid));
+  if (pStat && pStat->smaStatItems) {
+    pItem = (SSmaStatItem *)taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
   }
-
+#if 0
   if (pItem != NULL) {
     // TODO: reset time window for the sma data blocks
     if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
@@ -231,6 +364,7 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, int64_t indexUid, TSKEY skey
   } else {
     // error handling
   }
+#endif
   return TSDB_CODE_SUCCESS;
 }
 
@@ -241,7 +375,7 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, int64_t indexUid, TSKEY skey
  * @param intervalUnit
  * @return int32_t
  */
-static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) {
+static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
   // TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
   switch (intervalUnit) {
     case TD_TIME_UNIT_HOUR:
@@ -281,18 +415,35 @@ static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) {
 }
 
 /**
- * @brief Insert TSma data blocks to B+Tree
+ * @brief Insert TSma data blocks to DB File build by B+Tree
  *
- * @param bTree
+ * @param pSmaH
  * @param smaKey
+ * @param keyLen
  * @param pData
  * @param dataLen
  * @return int32_t
  */
-static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) {
+static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
+  SDBFile *pDBFile = &pSmaH->dFile;
+
   // TODO: insert sma data blocks into B+Tree
-  tsdbDebug("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
+  tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
-         *(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
+           REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
+           *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
+
+  if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+#ifdef SMA_PRINT_DEBUG_LOG
+  uint32_t valueSize = 0;
+  void *   data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize);
+  ASSERT(data != NULL);
+  for (uint32_t v = 0; v < valueSize; v += 8) {
+    tsdbWarn("vgId:%d sma data - val[%d] is %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v));
+  }
+#endif
   return TSDB_CODE_SUCCESS;
 }
 
@@ -324,41 +475,41 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit
     }
   }
 
-  switch (intervalUnit) {
+  switch (precision) {
-    case TD_TIME_UNIT_MILLISEC:
+    case TSDB_TIME_PRECISION_MILLI:
-      if (TSDB_TIME_PRECISION_MILLI == precision) {
+      if (TD_TIME_UNIT_MICROSEC == intervalUnit) {  // us
-        return interval;
-      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
-        return interval * 1e3;
-      } else {  //  nano second
-        return interval * 1e6;
-      }
-      break;
-    case TD_TIME_UNIT_MICROSEC:
-      if (TSDB_TIME_PRECISION_MILLI == precision) {
         return interval / 1e3;
-      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+      } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) {  //  nano second
-        return interval;
-      } else {  //  nano second
-        return interval * 1e3;
-      }
-      break;
-    case TD_TIME_UNIT_NANOSEC:
-      if (TSDB_TIME_PRECISION_MILLI == precision) {
         return interval / 1e6;
-      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+      } else {
-        return interval / 1e3;
-      } else {  // nano second
         return interval;
       }
       break;
-    default:
+    case TSDB_TIME_PRECISION_MICRO:
-      if (TSDB_TIME_PRECISION_MILLI == precision) {
+      if (TD_TIME_UNIT_MICROSEC == intervalUnit) {  // us
+        return interval;
+      } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) {  //  nano second
+        return interval / 1e3;
+      } else {
         return interval * 1e3;
-      } else if (TSDB_TIME_PRECISION_MICRO == precision) {
+      }
+      break;
+    case TSDB_TIME_PRECISION_NANO:
+      if (TD_TIME_UNIT_MICROSEC == intervalUnit) {
+        return interval * 1e3;
+      } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) {  // nano second
+        return interval;
+      } else {
         return interval * 1e6;
-      } else {  // nano second
+      }
-        return interval * 1e9;
+      break;
+    default:                                        // ms
+      if (TD_TIME_UNIT_MICROSEC == intervalUnit) {  // us
+        return interval / 1e3;
+      } else if (TD_TIME_UNIT_NANOSEC == intervalUnit) {  //  nano second
+        return interval / 1e6;
+      } else {
+        return interval;
       }
       break;
   }
@@ -381,8 +532,6 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p
 
   // TODO: check the data integrity
 
-  void *bTree = pSmaH->pDFile;
-
   int32_t len = 0;
   while (true) {
     if (len >= pData->dataLen) {
@@ -405,7 +554,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p
              pData->indexUid, pData->skey, pTbData->tableUid, pColData->colId);
 #endif
       tsdbEncodeTSmaKey(pTbData->tableUid, pColData->colId, pData->skey, (void **)&pSmaKey);
-      if (tsdbInsertTSmaBlocks(bTree, smaKey, pColData->data, pColData->blockSize) < 0) {
+      if (tsdbInsertTSmaBlocks(pSmaH, smaKey, SMA_KEY_LEN, pColData->data, pColData->blockSize) < 0) {
         tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
       }
       tbLen += (sizeof(STSmaColData) + pColData->blockSize);
@@ -419,15 +568,43 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p
 static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) {
   pSmaH->pTsdb = pTsdb;
   pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision);
+  return TSDB_CODE_SUCCESS;
+}
+
+static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
+  if (pSmaH) {
+    tsdbCloseDBF(&pSmaH->dFile);
+  }
 }
 
 static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t storageLevel, int32_t fid) {
-  // TODO
+  STsdb *pTsdb = pSmaH->pTsdb;
-  pSmaH->pDFile = "tSma_interval_file_name";
+  ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL);
-
+  char tSmaFile[TSDB_FILENAME_LEN] = {0};
+  snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pTsdb), fid);
+  pSmaH->dFile.path = strdup(tSmaFile);
   return TSDB_CODE_SUCCESS;
-} 
+}
 
+/**
+ * @brief
+ *
+ * @param pTsdb
+ * @param interval Interval calculated by DB's precision
+ * @param storageLevel
+ * @return int32_t
+ */
+static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel) {
+  STsdbCfg *pCfg = REPO_CFG(pTsdb);
+  int32_t   daysPerFile = pCfg->daysPerFile;
+
+  if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
+    int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]);
+    daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
+  }
+
+  return daysPerFile;
+}
 
 /**
  * @brief Insert/Update Time-range-wise SMA data.
@@ -441,51 +618,72 @@ static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData,
  * @param msg
  * @return int32_t
  */
-int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
+static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
-  STsdbCfg *  pCfg = REPO_CFG(pTsdb);
+  STsdbCfg *        pCfg = REPO_CFG(pTsdb);
   STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
-  STSmaWriteH tSmaH = {0};
 
-  tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData);
+  if (!pTsdb->pTSmaEnv) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
+    return terrno;
+  }
 
   if (pData->dataLen <= 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
+    return TSDB_CODE_FAILED;
   }
 
-  // Step 1: Judge the storage level
+  STSmaWriteH tSmaH = {0};
-  int32_t storageLevel = tsdbJudgeStorageLevel(pData->interval, pData->intervalUnit);
+
-  int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
+  if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  // Step 1: Judge the storage level and days
+  int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
+  int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel);
+  int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision));
 
   // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
   //         - Set and open the DFile or the B+Tree file
-
-  int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision));
-
-  // Save all the TSma data to one file
   // TODO: tsdbStartTSmaCommit();
   tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid);
-  tsdbInsertTSmaDataSection(&tSmaH, pData);
+  if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) {
+    tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
+             tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
+    tsdbDestroyTSmaWriteH(&tSmaH);
+    return TSDB_CODE_FAILED;
+  }
+
+  if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) {
+    tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    tsdbDestroyTSmaWriteH(&tSmaH);
+    return TSDB_CODE_FAILED;
+  }
   // TODO:tsdbEndTSmaCommit();
 
-  // reset the SSmaStat
+  // Step 3: reset the SSmaStat
-  tsdbResetExpiredWindow(pTsdb, pData->indexUid, pData->skey);
+  tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey);
 
+  tsdbDestroyTSmaWriteH(&tSmaH);
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) {
-  // TODO
+  STsdb *pTsdb = pSmaH->pTsdb;
-  pSmaH->pDFile = "rSma_interval_file_name";
+
+  char tSmaFile[TSDB_FILENAME_LEN] = {0};
+  snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.rsma", REPO_ID(pTsdb), fid);
+  pSmaH->dFile.path = strdup(tSmaFile);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
+static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
-  STsdbCfg *  pCfg = REPO_CFG(pTsdb);
+  STsdbCfg *        pCfg = REPO_CFG(pTsdb);
   STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
-  STSmaWriteH tSmaH = {0};
+  STSmaWriteH       tSmaH = {0};
 
   tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData);
 
@@ -496,7 +694,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
   }
 
   // Step 1: Judge the storage level
-  int32_t storageLevel = tsdbJudgeStorageLevel(pData->interval, pData->intervalUnit);
+  int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
   int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
 
   // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
@@ -507,45 +705,46 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
   // Save all the TSma data to one file
   // TODO: tsdbStartTSmaCommit();
   tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid);
+
   tsdbInsertTSmaDataSection(&tSmaH, pData);
   // TODO:tsdbEndTSmaCommit();
 
   // reset the SSmaStat
-  tsdbResetExpiredWindow(pTsdb, pData->indexUid, pData->skey);
+  tsdbResetExpiredWindow(SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey);
 
   return TSDB_CODE_SUCCESS;
 }
 
 /**
- * @brief Init of tSma ReadH
+ * @brief
  *
  * @param pSmaH
  * @param pTsdb
- * @param param
+ * @param interval
- * @param pData
+ * @param intervalUnit
  * @return int32_t
  */
-static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) {
+static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit) {
   pSmaH->pTsdb = pTsdb;
-  pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision);
+  pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision);
-  // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
+  pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit);
+  pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel);
 }
 
 /**
  * @brief Init of tSma FS
  *
  * @param pReadH
- * @param param
+ * @param skey
- * @param queryWin
  * @return int32_t
  */
-static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) {
+static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, TSKEY skey) {
-  int32_t storageLevel = 0;   //tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
+  int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, REPO_CFG(pSmaH->pTsdb)->precision));
-  int32_t daysPerFile =
+  char    tSmaFile[TSDB_FILENAME_LEN] = {0};
-      storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile;
+  snprintf(tSmaFile, TSDB_FILENAME_LEN, "v%df%d.tsma", REPO_ID(pSmaH->pTsdb), fid);
-  pReadH->storageLevel = storageLevel;
+  pSmaH->dFile.path = strdup(tSmaFile);
-  pReadH->days = daysPerFile;
+  pSmaH->smaFsIter.iter = 0;
-  pReadH->smaFsIter.iter = 0;
+  pSmaH->smaFsIter.fid = fid;
 }
 
 /**
@@ -557,17 +756,18 @@ static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) {
  * @return true
  * @return false
  */
-static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) {
+static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
   SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf;
   int32_t nSmaFs = taosArrayGetSize(smaFs);
 
-  pReadH->pDFile = NULL;
+  tsdbCloseDBF(&pReadH->dFile);
 
+#if 0
   while (pReadH->smaFsIter.iter < nSmaFs) {
     void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
     if (pSmaFile) {  // match(indexName, queryWindow)
       // TODO: select the file by index_name ...
-      pReadH->pDFile = pSmaFile;
+      pReadH->dFile = pSmaFile;
       ++pReadH->smaFsIter.iter;
       break;
     }
@@ -578,41 +778,83 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STimeWindow *queryWin) {
     tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
     return true;
   }
+#endif
 
   return false;
 }
 
 /**
- * @brief Return the data between queryWin and fill the pData.
+ * @brief
  *
- * @param pTsdb
+ * @param pTsdb Return the data between queryWin and fill the pData.
- * @param param
  * @param pData
- * @param queryWin
+ * @param indexUid
+ * @param interval
+ * @param intervalUnit
+ * @param tableUid
+ * @param colId
+ * @param pQuerySKey
  * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
  * @return int32_t
  */
-int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *queryWin, int32_t nMaxResult) {
+static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval,
-  SSmaStatItem *pItem =
+                                   int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySkey,
-      (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, &pData->indexUid, sizeof(pData->indexUid));
+                                   int32_t nMaxResult) {
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(SMA_ENV_STAT_ITEMS(pTsdb->pTSmaEnv), &indexUid, sizeof(indexUid));
   if (pItem == NULL) {
     // mark all window as expired and notify query module to query raw TS data.
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t nQueryWin = 0;
+#if 0
+  int32_t nQueryWin = taosArrayGetSize(pQuerySKey);
   for (int32_t n = 0; n < nQueryWin; ++n) {
-    TSKEY thisWindow = n;
+    TSKEY skey = taosArrayGet(pQuerySKey, n);
-    if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) {
+    if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY)) != NULL) {
       // TODO: mark this window as expired.
     }
   }
-
+#endif
+#if 0
+  if (taosHashGet(pItem->expiredWindows, &querySkey, sizeof(TSKEY)) != NULL) {
+    // TODO: mark this window as expired.
+  }
+#endif
   STSmaReadH tReadH = {0};
-  tsdbInitTSmaReadH(&tReadH, pTsdb, pData);
+  tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit);
+  tsdbCloseDBF(&tReadH.dFile);
 
-  tsdbInitTSmaFile(&tReadH, queryWin);
+  tsdbInitTSmaFile(&tReadH, querySkey);
+  if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) {
+    tsdbWarn("vgId:%d open DBF %s failed since %s", REPO_ID(pTsdb), tReadH.dFile.path, tstrerror(terrno));
+    return TSDB_CODE_FAILED;
+  }
 
+  char  smaKey[SMA_KEY_LEN] = {0};
+  void *pSmaKey = &smaKey;
+  tsdbEncodeTSmaKey(tableUid, colId, querySkey, (void **)&pSmaKey);
+
+  tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb),
+           tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
+           *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN);
+
+  void *   result = NULL;
+  uint32_t valueSize = 0;
+  if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) {
+    tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64
+             " since %s",
+             REPO_ID(pTsdb), indexUid, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
+             *(int64_t *)POINTER_SHIFT(smaKey, 10), tstrerror(terrno));
+    tsdbCloseDBF(&tReadH.dFile);
+    return TSDB_CODE_FAILED;
+  }
+  tfree(result);
+#ifdef SMA_PRINT_DEBUG_LOG
+  for (uint32_t v = 0; v < valueSize; v += 8) {
+    tsdbWarn("vgId:%d v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v));
+  }
+#endif
+#if 0
   int32_t nResult = 0;
   int64_t lastKey = 0;
 
@@ -634,8 +876,9 @@ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, STimeWindow *
       }
     }
   }
-
+#endif
   // read data from file and fill the result
+  tsdbCloseDBF(&tReadH.dFile);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -673,4 +916,55 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) {
   // }
   return TSDB_CODE_SUCCESS;
 }
-#endif
+#endif
+
+/**
+ * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine
+ *
+ * @param pTsdb
+ * @param param
+ * @param msg
+ * @return int32_t
+ * TODO: Who is responsible for resource allocate and release?
+ */
+int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) {
+    tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
+}
+
+int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) {
+    tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
+}
+
+/**
+ * @brief Insert Time-range-wise Rollup Sma(RSma) data
+ *
+ * @param pTsdb
+ * @param param
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) {
+    tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
+}
+
+int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit,
+                        tb_uid_t tableUid, col_id_t colId, TSKEY querySkey, int32_t nMaxResult) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySkey,
+                                  nMaxResult)) < 0) {
+    tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -34,6 +34,7 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
   return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
 }
 
+#if 0
 /**
  * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine
  * 
@@ -51,6 +52,14 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) {
   return code;
 }
 
+int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) {
+    tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
+}
+
 /**
  * @brief Insert Time-range-wise Rollup Sma(RSma) data
  *
@@ -65,4 +74,6 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) {
     tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
   }
   return code;
-}
+}
+
+#endif

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -33,7 +33,7 @@ int main(int argc, char **argv) {
   return RUN_ALL_TESTS();
 }
 
-TEST(testCase, tSmaEncodeDecodeTest) {
+TEST(testCase, tSma_Meta_Encode_Decode_Test) {
   // encode
   STSma tSma = {0};
   tSma.version = 0;
@@ -87,8 +87,9 @@ TEST(testCase, tSmaEncodeDecodeTest) {
   tdDestroyTSma(&tSma);
   tdDestroyTSmaWrapper(&dstTSmaWrapper);
 }
+
 #if 1
-TEST(testCase, tSma_DB_Put_Get_Del_Test) {
+TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   const char *   smaIndexName1 = "sma_index_test_1";
   const char *   smaIndexName2 = "sma_index_test_2";
   const char *   timezone = "Asia/Shanghai";
@@ -220,13 +221,84 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
 #endif
 
 #if 1
-TEST(testCase, tSmaInsertTest) {
+TEST(testCase, tSma_Data_Insert_Query_Test) {
-  const int64_t     indexUid = 2000000002;
+  // step 1: prepare meta
+  const char *   smaIndexName1 = "sma_index_test_1";
+  const char *   timezone = "Asia/Shanghai";
+  const char *   expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;";
+  const char *   tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'";
+  const char *   smaTestDir = "./smaTest";
+  const tb_uid_t tbUid = 1234567890;
+  const int64_t  indexUid1 = 2000000001;
+  const int64_t  interval1 = 1;
+  const int8_t   intervalUnit1 = TD_TIME_UNIT_DAY;
+  const uint32_t nCntTSma = 2;
+  TSKEY          skey1 = 1646987196;
+  const int64_t  testSmaData1 = 100;
+  const int64_t  testSmaData2 = 200;
+  // encode
+  STSma tSma = {0};
+  tSma.version = 0;
+  tSma.intervalUnit = TD_TIME_UNIT_DAY;
+  tSma.interval = 1;
+  tSma.slidingUnit = TD_TIME_UNIT_HOUR;
+  tSma.sliding = 0;
+  tSma.indexUid = indexUid1;
+  tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN);
+  tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN);
+  tSma.tableUid = tbUid;
+
+  tSma.exprLen = strlen(expr);
+  tSma.expr = (char *)calloc(tSma.exprLen + 1, 1);
+  tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
+
+  tSma.tagsFilterLen = strlen(tagsFilter);
+  tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1);
+  tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
+
+  SMeta *         pMeta = NULL;
+  STSma *         pSmaCfg = &tSma;
+  const SMetaCfg *pMetaCfg = &defaultMetaOptions;
+
+  taosRemoveDir(smaTestDir);
+
+  pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
+  assert(pMeta != NULL);
+  // save index 1
+  EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
+
+  // step 2: insert data
   STSmaDataWrapper *pSmaData = NULL;
   STsdb             tsdb = {0};
   STsdbCfg *        pCfg = &tsdb.config;
 
-  pCfg->daysPerFile = 1;
+  tsdb.pMeta = pMeta;
+  tsdb.vgId = 2;
+  tsdb.config.daysPerFile = 10;  // default days is 10
+  tsdb.config.keep1 = 30;
+  tsdb.config.keep2 = 90;
+  tsdb.config.keep = 365;
+  tsdb.config.precision = TSDB_TIME_PRECISION_MILLI;
+  tsdb.config.update = TD_ROW_OVERWRITE_UPDATE;
+  tsdb.config.compression = TWO_STAGE_COMP;
+
+  switch (tsdb.config.precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+      skey1 *= 1e3;
+      break;
+    case TSDB_TIME_PRECISION_MICRO:
+      skey1 *= 1e6;
+      break;
+    case TSDB_TIME_PRECISION_NANO:
+      skey1 *= 1e9;
+      break;
+    default:  // ms
+      skey1 *= 1e3;
+      break;
+  }
+
+  char *msg = (char *)calloc(100, 1);
+  EXPECT_EQ(tsdbUpdateSmaWindow(&tsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0);
 
   // init
   int32_t allocCnt = 0;
@@ -235,21 +307,21 @@ TEST(testCase, tSmaInsertTest) {
   void *  buf = NULL;
   EXPECT_EQ(tsdbMakeRoom(&buf, allocStep), 0);
   int32_t  bufSize = taosTSizeof(buf);
-  int32_t  numOfTables = 25;
+  int32_t  numOfTables = 10;
   col_id_t numOfCols = 4096;
   EXPECT_GT(numOfCols, 0);
 
   pSmaData = (STSmaDataWrapper *)buf;
   printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData);
-  pSmaData->skey = 1646987196;
+  pSmaData->skey = skey1;
-  pSmaData->interval = 10;
+  pSmaData->interval = interval1;
-  pSmaData->intervalUnit = TD_TIME_UNIT_MINUTE;
+  pSmaData->intervalUnit = intervalUnit1;
-  pSmaData->indexUid = indexUid;
+  pSmaData->indexUid = indexUid1;
 
   int32_t len = sizeof(STSmaDataWrapper);
   for (int32_t t = 0; t < numOfTables; ++t) {
     STSmaTbData *pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len);
-    pTbData->tableUid = t;
+    pTbData->tableUid = tbUid + t;
 
     int32_t tableDataLen = sizeof(STSmaTbData);
     for (col_id_t c = 0; c < numOfCols; ++c) {
@@ -262,8 +334,17 @@ TEST(testCase, tSmaInsertTest) {
       }
       STSmaColData *pColData = (STSmaColData *)POINTER_SHIFT(pSmaData, len + tableDataLen);
       pColData->colId = c + PRIMARYKEY_TIMESTAMP_COL_ID;
-      pColData->blockSize = ((c & 1) == 0) ? 8 : 16;
+
       // TODO: fill col data
+      if ((c & 1) == 0) {
+        pColData->blockSize = 8;
+        memcpy(pColData->data, &testSmaData1, 8);
+      } else {
+        pColData->blockSize = 16;
+        memcpy(pColData->data, &testSmaData1, 8);
+        memcpy(POINTER_SHIFT(pColData->data, 8), &testSmaData2, 8);
+      }
+
       tableDataLen += (sizeof(STSmaColData) + pColData->blockSize);
     }
     pTbData->dataLen = (tableDataLen - sizeof(STSmaTbData));
@@ -277,8 +358,24 @@ TEST(testCase, tSmaInsertTest) {
   // execute
   EXPECT_EQ(tsdbInsertTSmaData(&tsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
 
-  // release
+  // step 3: query
+  uint32_t checkDataCnt = 0;
+  for (int32_t t = 0; t < numOfTables; ++t) {
+    for (col_id_t c = 0; c < numOfCols; ++c) {
+      EXPECT_EQ(tsdbGetTSmaData(&tsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t,
+                                c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1),
+                TSDB_CODE_SUCCESS);
+      ++checkDataCnt;
+    }
+  }
+  
+  printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
+
+  // release data
   taosTZfree(buf);
+  // release meta
+  tdDestroyTSma(&tSma);
+  metaClose(pMeta);
 }
 #endif
 

source/libs/sync/inc/syncElection.h

@@ -39,7 +39,6 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode);
-
 int32_t syncNodeElect(SSyncNode* pSyncNode);
 int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg);
 

source/libs/sync/inc/syncEnv.h

@@ -29,6 +29,7 @@ extern "C" {
 #include "ttimer.h"
 
 #define TIMER_MAX_MS 0x7FFFFFFF
+#define ENV_TICK_TIMER_MS 1000
 #define PING_TIMER_MS 1000
 #define ELECT_TIMER_MS_MIN 150
 #define ELECT_TIMER_MS_MAX 300
@@ -38,17 +39,28 @@ extern "C" {
 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})
 
 typedef struct SSyncEnv {
-  tmr_h pEnvTickTimer;
+  // tick timer
+  tmr_h             pEnvTickTimer;
+  int32_t           envTickTimerMS;
+  uint64_t          envTickTimerLogicClock;  // if use queue, should pass logic clock into queue item
+  uint64_t          envTickTimerLogicClockUser;
+  TAOS_TMR_CALLBACK FpEnvTickTimer;  // Timer Fp
+  uint64_t          envTickTimerCounter;
+
+  // timer manager
   tmr_h pTimerManager;
-  char  name[128];
+
+  // other resources shared by SyncNodes
+  // ...
+
 } SSyncEnv;
 
 extern SSyncEnv* gSyncEnv;
 
 int32_t syncEnvStart();
 int32_t syncEnvStop();
-tmr_h   syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void* param);
+int32_t syncEnvStartTimer();
-void    syncEnvStopTimer(tmr_h* pTimer);
+int32_t syncEnvStopTimer();
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncIO.h

@@ -29,22 +29,28 @@ extern "C" {
 #include "tqueue.h"
 #include "trpc.h"
 
+#define TICK_Q_TIMER_MS 1000
+#define TICK_Ping_TIMER_MS 1000
+
 typedef struct SSyncIO {
   STaosQueue *pMsgQ;
-  STaosQset * pQset;
+  STaosQset  *pQset;
   pthread_t   consumerTid;
 
-  void * serverRpc;
+  void  *serverRpc;
-  void * clientRpc;
+  void  *clientRpc;
   SEpSet myAddr;
 
-  void *ioTimerTickQ;
+  tmr_h   qTimer;
-  void *ioTimerTickPing;
+  int32_t qTimerMS;
-  void *ioTimerManager;
+  tmr_h   pingTimer;
+  int32_t pingTimerMS;
+  tmr_h   timerMgr;
 
   void *pSyncNode;
   int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg);
   int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg);
+  int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg);
   int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg);
   int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg);
   int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg);
@@ -59,11 +65,14 @@ extern SSyncIO *gSyncIO;
 
 int32_t syncIOStart(char *host, uint16_t port);
 int32_t syncIOStop();
-int32_t syncIOTickQ();
-int32_t syncIOTickPing();
 int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg);
 int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg);
 
+int32_t syncIOQTimerStart();
+int32_t syncIOQTimerStop();
+int32_t syncIOPingTimerStart();
+int32_t syncIOPingTimerStop();
+
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/inc/syncInt.h

@@ -67,12 +67,12 @@ extern "C" {
     }                                                                   \
   }
 
-struct SRaft;
-typedef struct SRaft SRaft;
-
 struct SyncTimeout;
 typedef struct SyncTimeout SyncTimeout;
 
+struct SyncClientRequest;
+typedef struct SyncClientRequest SyncClientRequest;
+
 struct SyncPing;
 typedef struct SyncPing SyncPing;
 
@@ -117,8 +117,10 @@ typedef struct SSyncNode {
   SSyncCfg    syncCfg;
   char        path[TSDB_FILENAME_LEN];
   char        raftStorePath[TSDB_FILENAME_LEN * 2];
-  SWal*       pWal;
+
-  void*       rpcClient;
+  // sync io
+  SWal* pWal;
+  void* rpcClient;
   int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
   void* queue;
   int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
@@ -164,7 +166,7 @@ typedef struct SSyncNode {
   int32_t           pingTimerMS;
   uint64_t          pingTimerLogicClock;
   uint64_t          pingTimerLogicClockUser;
-  TAOS_TMR_CALLBACK FpPingTimer;  // Timer Fp
+  TAOS_TMR_CALLBACK FpPingTimerCB;  // Timer Fp
   uint64_t          pingTimerCounter;
 
   // elect timer
@@ -172,7 +174,7 @@ typedef struct SSyncNode {
   int32_t           electTimerMS;
   uint64_t          electTimerLogicClock;
   uint64_t          electTimerLogicClockUser;
-  TAOS_TMR_CALLBACK FpElectTimer;  // Timer Fp
+  TAOS_TMR_CALLBACK FpElectTimerCB;  // Timer Fp
   uint64_t          electTimerCounter;
 
   // heartbeat timer
@@ -180,12 +182,13 @@ typedef struct SSyncNode {
   int32_t           heartbeatTimerMS;
   uint64_t          heartbeatTimerLogicClock;
   uint64_t          heartbeatTimerLogicClockUser;
-  TAOS_TMR_CALLBACK FpHeartbeatTimer;  // Timer Fp
+  TAOS_TMR_CALLBACK FpHeartbeatTimerCB;  // Timer Fp
   uint64_t          heartbeatTimerCounter;
 
   // callback
   int32_t (*FpOnPing)(SSyncNode* ths, SyncPing* pMsg);
   int32_t (*FpOnPingReply)(SSyncNode* ths, SyncPingReply* pMsg);
+  int32_t (*FpOnClientRequest)(SSyncNode* ths, SyncClientRequest* pMsg);
   int32_t (*FpOnRequestVote)(SSyncNode* ths, SyncRequestVote* pMsg);
   int32_t (*FpOnRequestVoteReply)(SSyncNode* ths, SyncRequestVoteReply* pMsg);
   int32_t (*FpOnAppendEntries)(SSyncNode* ths, SyncAppendEntries* pMsg);
@@ -194,26 +197,47 @@ typedef struct SSyncNode {
 
 } SSyncNode;
 
+// open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
-int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+// ping --------------
-int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
 int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg);
-int32_t syncNodePingAll(SSyncNode* pSyncNode);
-int32_t syncNodePingPeers(SSyncNode* pSyncNode);
 int32_t syncNodePingSelf(SSyncNode* pSyncNode);
+int32_t syncNodePingPeers(SSyncNode* pSyncNode);
+int32_t syncNodePingAll(SSyncNode* pSyncNode);
 
+// timer control --------------
 int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms);
 int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode);
 int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms);
+int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
+
+// utils --------------
+int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
 cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
 char*   syncNode2Str(const SSyncNode* pSyncNode);
 
+// raft state change --------------
+void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
+void syncNodeBecomeFollower(SSyncNode* pSyncNode);
+void syncNodeBecomeLeader(SSyncNode* pSyncNode);
+
+void syncNodeCandidate2Leader(SSyncNode* pSyncNode);
+void syncNodeFollower2Candidate(SSyncNode* pSyncNode);
+void syncNodeLeader2Follower(SSyncNode* pSyncNode);
+void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
+
+// raft vote --------------
+void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId);
+void syncNodeVoteForSelf(SSyncNode* pSyncNode);
+void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode);
+
 // for debug --------------
 void syncNodePrint(SSyncNode* pObj);
 void syncNodePrint2(char* s, SSyncNode* pObj);

source/libs/sync/inc/syncMessage.h

@@ -39,6 +39,7 @@ typedef enum ESyncMessageType {
   SYNC_REQUEST_VOTE_REPLY = 111,
   SYNC_APPEND_ENTRIES = 113,
   SYNC_APPEND_ENTRIES_REPLY = 115,
+  SYNC_RESPONSE = 119,
 
 } ESyncMessageType;
 
@@ -195,7 +196,7 @@ typedef struct SyncRequestVote {
   SRaftId  srcId;
   SRaftId  destId;
   // private data
-  SyncTerm  currentTerm;
+  SyncTerm  term;
   SyncIndex lastLogIndex;
   SyncTerm  lastLogTerm;
 } SyncRequestVote;
@@ -254,6 +255,7 @@ typedef struct SyncAppendEntries {
   SRaftId  srcId;
   SRaftId  destId;
   // private data
+  SyncTerm  term;
   SyncIndex prevLogIndex;
   SyncTerm  prevLogTerm;
   SyncIndex commitIndex;
@@ -286,6 +288,7 @@ typedef struct SyncAppendEntriesReply {
   SRaftId  srcId;
   SRaftId  destId;
   // private data
+  SyncTerm  term;
   bool      success;
   SyncIndex matchIndex;
 } SyncAppendEntriesReply;

source/libs/sync/inc/syncRaftLog.h

@@ -27,6 +27,9 @@ extern "C" {
 #include "syncRaftEntry.h"
 #include "taosdef.h"
 
+#define SYNC_INDEX_BEGIN 0
+#define SYNC_INDEX_INVALID -1
+
 typedef struct SSyncLogStoreData {
   SSyncNode* pSyncNode;
   SWal*      pWal;

source/libs/sync/inc/syncRaftStore.h

@@ -43,6 +43,12 @@ int32_t     raftStorePersist(SRaftStore *pRaftStore);
 int32_t     raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len);
 int32_t     raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len);
 
+bool raftStoreHasVoted(SRaftStore *pRaftStore);
+void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId);
+void raftStoreClearVote(SRaftStore *pRaftStore);
+void raftStoreNextTerm(SRaftStore *pRaftStore);
+void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term);
+
 // for debug -------------------
 void raftStorePrint(SRaftStore *pObj);
 void raftStorePrint2(char *s, SRaftStore *pObj);

source/libs/sync/inc/syncReplication.h

@@ -52,7 +52,6 @@ extern "C" {
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode);
-
 int32_t syncNodeReplicate(SSyncNode* pSyncNode);
 int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg);
 

source/libs/sync/inc/syncUtil.h

@@ -34,6 +34,7 @@ void     syncUtilnodeInfo2EpSet(const SNodeInfo* pNodeInfo, SEpSet* pEpSet);
 void     syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet);
 void     syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId);
 bool     syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
+bool     syncUtilEmptyId(const SRaftId* pId);
 
 // ---- SSyncBuffer ----
 void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len);
@@ -52,6 +53,8 @@ const char* syncUtilState2String(ESyncState state);
 bool        syncUtilCanPrint(char c);
 char*       syncUtilprintBin(char* ptr, uint32_t len);
 char*       syncUtilprintBin2(char* ptr, uint32_t len);
+SyncIndex   syncUtilMinIndex(SyncIndex a, SyncIndex b);
+SyncIndex   syncUtilMaxIndex(SyncIndex a, SyncIndex b);
 
 #ifdef __cplusplus
 }

source/libs/sync/src/syncAppendEntries.c

@@ -14,6 +14,11 @@
  */
 
 #include "syncAppendEntries.h"
+#include "syncInt.h"
+#include "syncRaftLog.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -80,4 +85,121 @@
 //                       /\ UNCHANGED <<serverVars, commitIndex, messages>>
 //       /\ UNCHANGED <<candidateVars, leaderVars>>
 //
-int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {}
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
+  int32_t ret = 0;
+  syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg);
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    syncNodeUpdateTerm(ths, pMsg->term);
+  }
+  assert(pMsg->term <= ths->pRaftStore->currentTerm);
+
+  if (pMsg->term == ths->pRaftStore->currentTerm) {
+    ths->leaderCache = pMsg->srcId;
+    syncNodeResetElectTimer(ths);
+  }
+  assert(pMsg->dataLen >= 0);
+
+  SyncTerm localPreLogTerm = 0;
+  if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
+    SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm);
+    assert(pEntry != NULL);
+    localPreLogTerm = pEntry->term;
+    syncEntryDestory(pEntry);
+  }
+
+  bool logOK =
+      (pMsg->prevLogIndex == SYNC_INDEX_INVALID) ||
+      ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) &&
+       (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogIndex == localPreLogTerm));
+
+  // reject
+  if ((pMsg->term < ths->pRaftStore->currentTerm) ||
+      ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) {
+    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+    pReply->srcId = ths->myRaftId;
+    pReply->destId = pMsg->srcId;
+    pReply->term = ths->pRaftStore->currentTerm;
+    pReply->success = false;
+    pReply->matchIndex = SYNC_INDEX_INVALID;
+
+    SRpcMsg rpcMsg;
+    syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+    syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+    syncAppendEntriesReplyDestroy(pReply);
+
+    return ret;
+  }
+
+  // return to follower state
+  if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) {
+    syncNodeBecomeFollower(ths);
+  }
+
+  // accept request
+  if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) {
+    bool matchSuccess = false;
+    if (pMsg->prevLogIndex == SYNC_INDEX_INVALID &&
+        ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) {
+      matchSuccess = true;
+    }
+    if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
+      SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogTerm);
+      assert(pEntry != NULL);
+      if (pMsg->prevLogTerm == pEntry->term) {
+        matchSuccess = true;
+      }
+      syncEntryDestory(pEntry);
+    }
+
+    if (matchSuccess) {
+      // delete conflict entries
+      if (ths->pLogStore->getLastIndex(ths->pLogStore) > pMsg->prevLogIndex) {
+        SyncIndex fromIndex = pMsg->prevLogIndex + 1;
+        ths->pLogStore->truncate(ths->pLogStore, fromIndex);
+      }
+
+      // append one entry
+      if (pMsg->dataLen > 0) {
+        SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
+        ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+        syncEntryDestory(pEntry);
+      }
+
+      SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+      pReply->srcId = ths->myRaftId;
+      pReply->destId = pMsg->srcId;
+      pReply->term = ths->pRaftStore->currentTerm;
+      pReply->success = true;
+      pReply->matchIndex = pMsg->prevLogIndex + 1;
+
+      SRpcMsg rpcMsg;
+      syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+      syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+
+      syncAppendEntriesReplyDestroy(pReply);
+    } else {
+      SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+      pReply->srcId = ths->myRaftId;
+      pReply->destId = pMsg->srcId;
+      pReply->term = ths->pRaftStore->currentTerm;
+      pReply->success = false;
+      pReply->matchIndex = SYNC_INDEX_INVALID;
+
+      SRpcMsg rpcMsg;
+      syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg);
+      syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+      syncAppendEntriesReplyDestroy(pReply);
+    }
+
+    if (pMsg->commitIndex > ths->commitIndex) {
+      if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) {
+        // commit
+        ths->commitIndex = pMsg->commitIndex;
+        ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex);
+      }
+    }
+  }
+
+  return ret;
+}

source/libs/sync/src/syncAppendEntriesReply.c

@@ -14,6 +14,12 @@
  */
 
 #include "syncAppendEntriesReply.h"
+#include "syncIndexMgr.h"
+#include "syncInt.h"
+#include "syncRaftLog.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
 
 // TLA+ Spec
 // HandleAppendEntriesResponse(i, j, m) ==
@@ -28,4 +34,41 @@
 //    /\ Discard(m)
 //    /\ UNCHANGED <<serverVars, candidateVars, logVars, elections>>
 //
-int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {}
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
+  int32_t ret = 0;
+  syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg);
+
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm);
+    return ret;
+  }
+
+  // no need this code, because if I receive reply.term, then I must have sent for that term.
+  //  if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //    syncNodeUpdateTerm(ths, pMsg->term);
+  //  }
+
+  assert(pMsg->term == ths->pRaftStore->currentTerm);
+
+  if (pMsg->success) {
+    // nextIndex = reply.matchIndex + 1
+    syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1);
+
+    // matchIndex = reply.matchIndex
+    syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
+
+    // maybe commit
+    syncNodeMaybeAdvanceCommitIndex(ths);
+
+  } else {
+    SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId));
+    if (nextIndex > SYNC_INDEX_BEGIN) {
+      --nextIndex;
+    } else {
+      nextIndex = SYNC_INDEX_BEGIN;
+    }
+    syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
+  }
+
+  return ret;
+}

source/libs/sync/src/syncElection.c

@@ -16,6 +16,7 @@
 #include "syncElection.h"
 #include "syncMessage.h"
 #include "syncRaftStore.h"
+#include "syncVoteMgr.h"
 
 // TLA+ Spec
 // RequestVote(i, j) ==
@@ -37,7 +38,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
     SyncRequestVote* pMsg = syncRequestVoteBuild();
     pMsg->srcId = pSyncNode->myRaftId;
     pMsg->destId = pSyncNode->peersId[i];
-    pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm;
+    pMsg->term = pSyncNode->pRaftStore->currentTerm;
     pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore);
     pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore);
 
@@ -49,10 +50,22 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
 }
 
 int32_t syncNodeElect(SSyncNode* pSyncNode) {
+  if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
+    syncNodeFollower2Candidate(pSyncNode);
+  }
   assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
 
   // start election
+  raftStoreNextTerm(pSyncNode->pRaftStore);
+  raftStoreClearVote(pSyncNode->pRaftStore);
+  voteGrantedReset(pSyncNode->pVotesGranted, pSyncNode->pRaftStore->currentTerm);
+  votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm);
+
+  syncNodeVoteForSelf(pSyncNode);
   int32_t ret = syncNodeRequestVotePeers(pSyncNode);
+  assert(ret == 0);
+  syncNodeResetElectTimer(pSyncNode);
+
   return ret;
 }
 

source/libs/sync/src/syncEnv.c

@@ -19,19 +19,18 @@
 SSyncEnv *gSyncEnv = NULL;
 
 // local function -----------------
-static void    syncEnvTick(void *param, void *tmrId);
+static SSyncEnv *doSyncEnvStart();
-static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv);
+static int32_t   doSyncEnvStop(SSyncEnv *pSyncEnv);
-static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv);
+static int32_t   doSyncEnvStartTimer(SSyncEnv *pSyncEnv);
-static tmr_h   doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param);
+static int32_t   doSyncEnvStopTimer(SSyncEnv *pSyncEnv);
-static void    doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer);
+static void      syncEnvTick(void *param, void *tmrId);
 // --------------------------------
 
 int32_t syncEnvStart() {
-  int32_t ret;
+  int32_t ret = 0;
   taosSeedRand(taosGetTimestampSec());
-  gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv));
+  gSyncEnv = doSyncEnvStart(gSyncEnv);
   assert(gSyncEnv != NULL);
-  ret = doSyncEnvStart(gSyncEnv);
   return ret;
 }
 
@@ -40,31 +39,52 @@ int32_t syncEnvStop() {
   return ret;
 }
 
-tmr_h syncEnvStartTimer(TAOS_TMR_CALLBACK fp, int mseconds, void *param) {
+int32_t syncEnvStartTimer() {
-  return doSyncEnvStartTimer(gSyncEnv, fp, mseconds, param);
+  int32_t ret = doSyncEnvStartTimer(gSyncEnv);
+  return ret;
 }
 
-void syncEnvStopTimer(tmr_h *pTimer) { doSyncEnvStopTimer(gSyncEnv, pTimer); }
+int32_t syncEnvStopTimer() {
+  int32_t ret = doSyncEnvStopTimer(gSyncEnv);
+  return ret;
+}
 
 // local function -----------------
 static void syncEnvTick(void *param, void *tmrId) {
   SSyncEnv *pSyncEnv = (SSyncEnv *)param;
-  sTrace("syncEnvTick ... name:%s ", pSyncEnv->name);
+  if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) {
+    ++(pSyncEnv->envTickTimerCounter);
+    sTrace(
+        "syncEnvTick do ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, "
+        "envTickTimerMS:%d, tmrId:%p",
+        pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter,
+        pSyncEnv->envTickTimerMS, tmrId);
 
-  pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager);
+    // do something, tick ...
+    taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer);
+  } else {
+    sTrace(
+        "syncEnvTick pass ... envTickTimerLogicClockUser:%lu, envTickTimerLogicClock:%lu, envTickTimerCounter:%lu, "
+        "envTickTimerMS:%d, tmrId:%p",
+        pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter,
+        pSyncEnv->envTickTimerMS, tmrId);
+  }
 }
 
-static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) {
+static SSyncEnv *doSyncEnvStart() {
-  snprintf(pSyncEnv->name, sizeof(pSyncEnv->name), "SyncEnv_%p", pSyncEnv);
+  SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv));
+  assert(pSyncEnv != NULL);
+  memset(pSyncEnv, 0, sizeof(pSyncEnv));
+
+  pSyncEnv->envTickTimerCounter = 0;
+  pSyncEnv->envTickTimerMS = ENV_TICK_TIMER_MS;
+  pSyncEnv->FpEnvTickTimer = syncEnvTick;
+  atomic_store_64(&pSyncEnv->envTickTimerLogicClock, 0);
+  atomic_store_64(&pSyncEnv->envTickTimerLogicClockUser, 0);
 
   // start tmr thread
   pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV");
-
+  return pSyncEnv;
-  // pSyncEnv->pEnvTickTimer = taosTmrStart(syncEnvTick, 1000, pSyncEnv, pSyncEnv->pTimerManager);
-
-  sTrace("SyncEnv start ok, name:%s", pSyncEnv->name);
-
-  return 0;
 }
 
 static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) {
@@ -72,8 +92,18 @@ static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) {
   return 0;
 }
 
-static tmr_h doSyncEnvStartTimer(SSyncEnv *pSyncEnv, TAOS_TMR_CALLBACK fp, int mseconds, void *param) {
+static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) {
-  return taosTmrStart(fp, mseconds, pSyncEnv, pSyncEnv->pTimerManager);
+  int32_t ret = 0;
+  taosTmrReset(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager,
+               &pSyncEnv->pEnvTickTimer);
+  atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser);
+  return ret;
 }
 
-static void doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer) {}
+static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) {
+  int32_t ret = 0;
+  atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1);
+  taosTmrStop(pSyncEnv->pEnvTickTimer);
+  pSyncEnv->pEnvTickTimer = NULL;
+  return ret;
+}

source/libs/sync/src/syncIO.c

@@ -16,6 +16,7 @@
 #include "syncIO.h"
 #include <tdatablock.h>
 #include "syncMessage.h"
+#include "syncUtil.h"
 #include "tglobal.h"
 #include "ttimer.h"
 #include "tutil.h"
@@ -23,33 +24,36 @@
 SSyncIO *gSyncIO = NULL;
 
 // local function ------------
-static int32_t  syncIOStartInternal(SSyncIO *io);
-static int32_t  syncIOStopInternal(SSyncIO *io);
 static SSyncIO *syncIOCreate(char *host, uint16_t port);
 static int32_t  syncIODestroy(SSyncIO *io);
+static int32_t  syncIOStartInternal(SSyncIO *io);
+static int32_t  syncIOStopInternal(SSyncIO *io);
 
-static void *syncIOConsumerFunc(void *param);
+static void   *syncIOConsumerFunc(void *param);
-static int   syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
+static void    syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
-static void  syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
+static void    syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
-static void  syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
+static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
 
-static int32_t syncIOTickQInternal(SSyncIO *io);
+static int32_t syncIOStartQ(SSyncIO *io);
-static void    syncIOTickQFunc(void *param, void *tmrId);
+static int32_t syncIOStopQ(SSyncIO *io);
-static int32_t syncIOTickPingInternal(SSyncIO *io);
+static int32_t syncIOStartPing(SSyncIO *io);
-static void    syncIOTickPingFunc(void *param, void *tmrId);
+static int32_t syncIOStopPing(SSyncIO *io);
+static void    syncIOTickQ(void *param, void *tmrId);
+static void    syncIOTickPing(void *param, void *tmrId);
 // ----------------------------
 
 // public function ------------
 int32_t syncIOStart(char *host, uint16_t port) {
+  int32_t ret = 0;
   gSyncIO = syncIOCreate(host, port);
   assert(gSyncIO != NULL);
 
   taosSeedRand(taosGetTimestampSec());
-  int32_t ret = syncIOStartInternal(gSyncIO);
+  ret = syncIOStartInternal(gSyncIO);
   assert(ret == 0);
 
-  sTrace("syncIOStart ok, gSyncIO:%p gSyncIO->clientRpc:%p", gSyncIO, gSyncIO->clientRpc);
+  sTrace("syncIOStart ok, gSyncIO:%p", gSyncIO);
-  return 0;
+  return ret;
 }
 
 int32_t syncIOStop() {
@@ -61,37 +65,25 @@ int32_t syncIOStop() {
   return ret;
 }
 
-int32_t syncIOTickQ() {
-  int32_t ret = syncIOTickQInternal(gSyncIO);
-  assert(ret == 0);
-  return ret;
-}
-
-int32_t syncIOTickPing() {
-  int32_t ret = syncIOTickPingInternal(gSyncIO);
-  assert(ret == 0);
-  return ret;
-}
-
 int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
-  sTrace(
+  assert(pEpSet->inUse == 0);
-      "<--- syncIOSendMsg ---> clientRpc:%p, numOfEps:%d, inUse:%d, destAddr:%s-%u, pMsg->ahandle:%p, pMsg->handle:%p, "
+  assert(pEpSet->numOfEps == 1);
-      "pMsg->msgType:%d, pMsg->contLen:%d",
+
-      clientRpc, pEpSet->numOfEps, pEpSet->inUse, pEpSet->eps[0].fqdn, pEpSet->eps[0].port, pMsg->ahandle, pMsg->handle,
+  int32_t ret = 0;
-      pMsg->msgType, pMsg->contLen);
+  char    logBuf[256];
-  {
+  snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
-    cJSON *pJson = syncRpcMsg2Json(pMsg);
+  syncRpcMsgPrint2(logBuf, pMsg);
-    char * serialized = cJSON_Print(pJson);
+
-    sTrace("process syncMessage send: pMsg:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
   pMsg->handle = NULL;
   rpcSendRequest(clientRpc, pEpSet, pMsg, NULL);
-  return 0;
+  return ret;
 }
 
 int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) {
+  int32_t ret = 0;
+  char    logBuf[128];
+  syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg);
+
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
   memcpy(pTemp, pMsg, sizeof(SRpcMsg));
@@ -99,11 +91,75 @@ int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) {
   STaosQueue *pMsgQ = queue;
   taosWriteQitem(pMsgQ, pTemp);
 
-  return 0;
+  return ret;
+}
+
+int32_t syncIOQTimerStart() {
+  int32_t ret = syncIOStartQ(gSyncIO);
+  assert(ret == 0);
+  return ret;
+}
+
+int32_t syncIOQTimerStop() {
+  int32_t ret = syncIOStopQ(gSyncIO);
+  assert(ret == 0);
+  return ret;
+}
+
+int32_t syncIOPingTimerStart() {
+  int32_t ret = syncIOStartPing(gSyncIO);
+  assert(ret == 0);
+  return ret;
+}
+
+int32_t syncIOPingTimerStop() {
+  int32_t ret = syncIOStopPing(gSyncIO);
+  assert(ret == 0);
+  return ret;
 }
 
 // local function ------------
+static SSyncIO *syncIOCreate(char *host, uint16_t port) {
+  SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO));
+  memset(io, 0, sizeof(*io));
+
+  io->pMsgQ = taosOpenQueue();
+  io->pQset = taosOpenQset();
+  taosAddIntoQset(io->pQset, io->pMsgQ, NULL);
+
+  io->myAddr.inUse = 0;
+  io->myAddr.numOfEps = 0;
+  addEpIntoEpSet(&io->myAddr, host, port);
+
+  io->qTimerMS = TICK_Q_TIMER_MS;
+  io->pingTimerMS = TICK_Ping_TIMER_MS;
+
+  return io;
+}
+
+static int32_t syncIODestroy(SSyncIO *io) {
+  int32_t ret = 0;
+  int8_t  start = atomic_load_8(&io->isStart);
+  assert(start == 0);
+
+  if (io->serverRpc != NULL) {
+    rpcClose(io->serverRpc);
+    io->serverRpc = NULL;
+  }
+
+  if (io->clientRpc != NULL) {
+    rpcClose(io->clientRpc);
+    io->clientRpc = NULL;
+  }
+
+  taosCloseQueue(io->pMsgQ);
+  taosCloseQset(io->pQset);
+
+  return ret;
+}
+
 static int32_t syncIOStartInternal(SSyncIO *io) {
+  int32_t ret = 0;
   taosBlockSIGPIPE();
 
   rpcInit();
@@ -163,58 +219,24 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
   }
 
   // start tmr thread
-  io->ioTimerManager = taosTmrInit(1000, 50, 10000, "SYNC");
+  io->timerMgr = taosTmrInit(1000, 50, 10000, "SYNC-IO");
 
-  return 0;
+  atomic_store_8(&io->isStart, 1);
+  return ret;
 }
 
 static int32_t syncIOStopInternal(SSyncIO *io) {
+  int32_t ret = 0;
   atomic_store_8(&io->isStart, 0);
   pthread_join(io->consumerTid, NULL);
-  return 0;
+  taosTmrCleanUp(io->timerMgr);
-}
+  return ret;
-
-static SSyncIO *syncIOCreate(char *host, uint16_t port) {
-  SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO));
-  memset(io, 0, sizeof(*io));
-
-  io->pMsgQ = taosOpenQueue();
-  io->pQset = taosOpenQset();
-  taosAddIntoQset(io->pQset, io->pMsgQ, NULL);
-
-  io->myAddr.inUse = 0;
-  addEpIntoEpSet(&io->myAddr, host, port);
-
-  return io;
-}
-
-static int32_t syncIODestroy(SSyncIO *io) {
-  int8_t start = atomic_load_8(&io->isStart);
-  assert(start == 0);
-
-  if (io->serverRpc != NULL) {
-    free(io->serverRpc);
-    io->serverRpc = NULL;
-  }
-
-  if (io->clientRpc != NULL) {
-    free(io->clientRpc);
-    io->clientRpc = NULL;
-  }
-
-  taosCloseQueue(io->pMsgQ);
-  taosCloseQset(io->pQset);
-
-  return 0;
 }
 
 static void *syncIOConsumerFunc(void *param) {
-  SSyncIO *io = param;
+  SSyncIO   *io = param;
-
   STaosQall *qall;
-  SRpcMsg *  pRpcMsg, rpcMsg;
+  SRpcMsg   *pRpcMsg, rpcMsg;
-  int        type;
-
   qall = taosAllocateQall();
 
   while (1) {
@@ -226,77 +248,74 @@ static void *syncIOConsumerFunc(void *param) {
 
     for (int i = 0; i < numOfMsgs; ++i) {
       taosGetQitem(qall, (void **)&pRpcMsg);
+      syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg);
 
-      char *s = syncRpcMsg2Str(pRpcMsg);
+      // use switch case instead of if else
-      sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen,
-             s);
-      free(s);
-
       if (pRpcMsg->msgType == SYNC_PING) {
         if (io->FpOnSyncPing != NULL) {
-          SyncPing *pSyncMsg;
+          SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
+          assert(pSyncMsg != NULL);
+          io->FpOnSyncPing(io->pSyncNode, pSyncMsg);
+          syncPingDestroy(pSyncMsg);
+          /*
           pSyncMsg = syncPingBuild(pRpcMsg->contLen);
           syncPingFromRpcMsg(pRpcMsg, pSyncMsg);
           // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen);
           io->FpOnSyncPing(io->pSyncNode, pSyncMsg);
           syncPingDestroy(pSyncMsg);
+          */
         }
 
       } else if (pRpcMsg->msgType == SYNC_PING_REPLY) {
         if (io->FpOnSyncPingReply != NULL) {
-          SyncPingReply *pSyncMsg;
+          SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncPingReplyBuild(pRpcMsg->contLen);
-          syncPingReplyFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg);
           syncPingReplyDestroy(pSyncMsg);
         }
 
+      } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) {
+        if (io->FpOnSyncClientRequest != NULL) {
+          SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
+          io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg);
+          syncClientRequestDestroy(pSyncMsg);
+        }
+
       } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) {
         if (io->FpOnSyncRequestVote != NULL) {
-          SyncRequestVote *pSyncMsg;
+          SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncRequestVoteBuild(pRpcMsg->contLen);
-          syncRequestVoteFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncRequestVote(io->pSyncNode, pSyncMsg);
           syncRequestVoteDestroy(pSyncMsg);
         }
 
       } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) {
         if (io->FpOnSyncRequestVoteReply != NULL) {
-          SyncRequestVoteReply *pSyncMsg;
+          SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncRequestVoteReplyBuild();
-          syncRequestVoteReplyFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncRequestVoteReply(io->pSyncNode, pSyncMsg);
           syncRequestVoteReplyDestroy(pSyncMsg);
         }
 
       } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) {
         if (io->FpOnSyncAppendEntries != NULL) {
-          SyncAppendEntries *pSyncMsg;
+          SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncAppendEntriesBuild(pRpcMsg->contLen);
-          syncAppendEntriesFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncAppendEntries(io->pSyncNode, pSyncMsg);
           syncAppendEntriesDestroy(pSyncMsg);
         }
 
       } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) {
         if (io->FpOnSyncAppendEntriesReply != NULL) {
-          SyncAppendEntriesReply *pSyncMsg;
+          SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncAppendEntriesReplyBuild();
-          syncAppendEntriesReplyFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncAppendEntriesReply(io->pSyncNode, pSyncMsg);
           syncAppendEntriesReplyDestroy(pSyncMsg);
         }
 
       } else if (pRpcMsg->msgType == SYNC_TIMEOUT) {
         if (io->FpOnSyncTimeout != NULL) {
-          SyncTimeout *pSyncMsg;
+          SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
-          pSyncMsg = syncTimeoutBuild();
-          syncTimeoutFromRpcMsg(pRpcMsg, pSyncMsg);
           io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg);
           syncTimeoutDestroy(pSyncMsg);
         }
       } else {
-        ;
+        sTrace("unknown msgType:%d, no operator", pRpcMsg->msgType);
       }
     }
 
@@ -306,15 +325,16 @@ static void *syncIOConsumerFunc(void *param) {
       rpcFreeCont(pRpcMsg->pCont);
 
       if (pRpcMsg->handle != NULL) {
-        int msgSize = 128;
+        int msgSize = 32;
         memset(&rpcMsg, 0, sizeof(rpcMsg));
+        rpcMsg.msgType = SYNC_RESPONSE;
         rpcMsg.pCont = rpcMallocCont(msgSize);
         rpcMsg.contLen = msgSize;
         snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply");
         rpcMsg.handle = pRpcMsg->handle;
         rpcMsg.code = 0;
 
-        sTrace("syncIOConsumerFunc rpcSendResponse ... msgType:%d contLen:%d", pRpcMsg->msgType, rpcMsg.contLen);
+        syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
         rpcSendResponse(&rpcMsg);
       }
 
@@ -326,71 +346,95 @@ static void *syncIOConsumerFunc(void *param) {
   return NULL;
 }
 
-static int syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
-  // app shall retrieve the auth info based on meterID from DB or a data file
-  // demo code here only for simple demo
-  int ret = 0;
-  return ret;
-}
-
 static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  sTrace("<-- syncIOProcessRequest --> type:%d, contLen:%d, cont:%s", pMsg->msgType, pMsg->contLen,
+  syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg);
-         (char *)pMsg->pCont);
-
   SSyncIO *io = pParent;
   SRpcMsg *pTemp;
-
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
   memcpy(pTemp, pMsg, sizeof(SRpcMsg));
-
   taosWriteQitem(io->pMsgQ, pTemp);
 }
 
 static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  sTrace("syncIOProcessReply: type:%d, contLen:%d msg:%s", pMsg->msgType, pMsg->contLen, (char *)pMsg->pCont);
+  if (pMsg->msgType == SYNC_RESPONSE) {
+    sTrace("==syncIOProcessReply==");
+  } else {
+    syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg);
+  }
   rpcFreeCont(pMsg->pCont);
 }
 
-static int32_t syncIOTickQInternal(SSyncIO *io) {
+static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
-  io->ioTimerTickQ = taosTmrStart(syncIOTickQFunc, 1000, io, io->ioTimerManager);
+  // app shall retrieve the auth info based on meterID from DB or a data file
-  return 0;
+  // demo code here only for simple demo
+  int32_t ret = 0;
+  return ret;
 }
 
-static void syncIOTickQFunc(void *param, void *tmrId) {
+static int32_t syncIOStartQ(SSyncIO *io) {
+  int32_t ret = 0;
+  taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer);
+  return ret;
+}
+
+static int32_t syncIOStopQ(SSyncIO *io) {
+  int32_t ret = 0;
+  taosTmrStop(io->qTimer);
+  io->qTimer = NULL;
+  return ret;
+}
+
+static int32_t syncIOStartPing(SSyncIO *io) {
+  int32_t ret = 0;
+  taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
+  return ret;
+}
+
+static int32_t syncIOStopPing(SSyncIO *io) {
+  int32_t ret = 0;
+  taosTmrStop(io->pingTimer);
+  io->pingTimer = NULL;
+  return ret;
+}
+
+static void syncIOTickQ(void *param, void *tmrId) {
   SSyncIO *io = (SSyncIO *)param;
-  sTrace("<-- syncIOTickQFunc -->");
+
+  SRaftId srcId, destId;
+  srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
+  srcId.vgId = -1;
+  destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
+  destId.vgId = -1;
+  SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ");
 
   SRpcMsg rpcMsg;
-  rpcMsg.contLen = 64;
+  syncPingReply2RpcMsg(pMsg, &rpcMsg);
-  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
-  snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickQ");
-  rpcMsg.handle = NULL;
-  rpcMsg.msgType = 55;
-
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
   memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg));
-
+  syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg);
   taosWriteQitem(io->pMsgQ, pTemp);
-  taosTmrReset(syncIOTickQFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickQ);
+  syncPingReplyDestroy(pMsg);
+
+  taosTmrReset(syncIOTickQ, io->qTimerMS, io, io->timerMgr, &io->qTimer);
 }
 
-static int32_t syncIOTickPingInternal(SSyncIO *io) {
+static void syncIOTickPing(void *param, void *tmrId) {
-  io->ioTimerTickPing = taosTmrStart(syncIOTickPingFunc, 1000, io, io->ioTimerManager);
-  return 0;
-}
-
-static void syncIOTickPingFunc(void *param, void *tmrId) {
   SSyncIO *io = (SSyncIO *)param;
-  sTrace("<-- syncIOTickPingFunc -->");
+
+  SRaftId srcId, destId;
+  srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
+  srcId.vgId = -1;
+  destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
+  destId.vgId = -1;
+  SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing");
+  // SyncPing *pMsg = syncPingBuild3(&srcId, &destId);
 
   SRpcMsg rpcMsg;
-  rpcMsg.contLen = 64;
+  syncPing2RpcMsg(pMsg, &rpcMsg);
-  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg);
-  snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOTickPing");
-  rpcMsg.handle = NULL;
-  rpcMsg.msgType = 77;
-
   rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL);
-  taosTmrReset(syncIOTickPingFunc, 1000, io, io->ioTimerManager, &io->ioTimerTickPing);
+  syncPingDestroy(pMsg);
+
+  taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
 }

source/libs/sync/src/syncMain.c

@@ -17,11 +17,14 @@
 #include "sync.h"
 #include "syncAppendEntries.h"
 #include "syncAppendEntriesReply.h"
+#include "syncElection.h"
 #include "syncEnv.h"
 #include "syncIndexMgr.h"
 #include "syncInt.h"
+#include "syncMessage.h"
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
+#include "syncReplication.h"
 #include "syncRequestVote.h"
 #include "syncRequestVoteReply.h"
 #include "syncTimeout.h"
@@ -31,33 +34,32 @@
 static int32_t tsNodeRefId = -1;
 
 // ------ local funciton ---------
+// enqueue message ----
 static void syncNodeEqPingTimer(void* param, void* tmrId);
 static void syncNodeEqElectTimer(void* param, void* tmrId);
 static void syncNodeEqHeartbeatTimer(void* param, void* tmrId);
 
+// on message ----
 static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
 static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
-
+static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
-static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
-static void syncNodeBecomeFollower(SSyncNode* pSyncNode);
-static void syncNodeBecomeLeader(SSyncNode* pSyncNode);
-static void syncNodeFollower2Candidate(SSyncNode* pSyncNode);
-static void syncNodeCandidate2Leader(SSyncNode* pSyncNode);
-static void syncNodeLeader2Follower(SSyncNode* pSyncNode);
-static void syncNodeCandidate2Follower(SSyncNode* pSyncNode);
 // ---------------------------------
 
 int32_t syncInit() {
-  sTrace("syncInit ok");
+  int32_t ret = syncEnvStart();
-  return 0;
+  return ret;
 }
 
-void syncCleanUp() { sTrace("syncCleanUp ok"); }
+void syncCleanUp() {
+  int32_t ret = syncEnvStop();
+  assert(ret == 0);
+}
 
 int64_t syncStart(const SSyncInfo* pSyncInfo) {
+  int32_t    ret = 0;
   SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo);
   assert(pSyncNode != NULL);
-  return 0;
+  return ret;
 }
 
 void syncStop(int64_t rid) {
@@ -65,9 +67,13 @@ void syncStop(int64_t rid) {
   syncNodeClose(pSyncNode);
 }
 
-int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; }
+int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
+  int32_t ret = 0;
+  return ret;
+}
 
 int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
+  int32_t    ret = 0;
   SSyncNode* pSyncNode = NULL;  // get pointer from rid
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
     SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak);
@@ -75,11 +81,13 @@ int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
     syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
     syncClientRequestDestroy(pSyncMsg);
+    ret = 0;
+
   } else {
     sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state));
-    return -1;  // need define err code !!
+    ret = -1;  // need define err code !!
   }
-  return 0;
+  return ret;
 }
 
 ESyncState syncGetMyRole(int64_t rid) {
@@ -89,6 +97,7 @@ ESyncState syncGetMyRole(int64_t rid) {
 
 void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {}
 
+// open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode));
   assert(pSyncNode != NULL);
@@ -162,7 +171,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->pingTimerMS = PING_TIMER_MS;
   atomic_store_64(&pSyncNode->pingTimerLogicClock, 0);
   atomic_store_64(&pSyncNode->pingTimerLogicClockUser, 0);
-  pSyncNode->FpPingTimer = syncNodeEqPingTimer;
+  pSyncNode->FpPingTimerCB = syncNodeEqPingTimer;
   pSyncNode->pingTimerCounter = 0;
 
   // init elect timer
@@ -170,7 +179,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->electTimerMS = syncUtilElectRandomMS();
   atomic_store_64(&pSyncNode->electTimerLogicClock, 0);
   atomic_store_64(&pSyncNode->electTimerLogicClockUser, 0);
-  pSyncNode->FpElectTimer = syncNodeEqElectTimer;
+  pSyncNode->FpElectTimerCB = syncNodeEqElectTimer;
   pSyncNode->electTimerCounter = 0;
 
   // init heartbeat timer
@@ -178,12 +187,13 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->heartbeatTimerMS = HEARTBEAT_TIMER_MS;
   atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, 0);
   atomic_store_64(&pSyncNode->heartbeatTimerLogicClockUser, 0);
-  pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer;
+  pSyncNode->FpHeartbeatTimerCB = syncNodeEqHeartbeatTimer;
   pSyncNode->heartbeatTimerCounter = 0;
 
   // init callback
   pSyncNode->FpOnPing = syncNodeOnPingCb;
   pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb;
+  pSyncNode->FpOnClientRequest = syncNodeOnClientRequestCb;
   pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
   pSyncNode->FpOnRequestVoteReply = syncNodeOnRequestVoteReplyCb;
   pSyncNode->FpOnAppendEntries = syncNodeOnAppendEntriesCb;
@@ -194,10 +204,153 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
 }
 
 void syncNodeClose(SSyncNode* pSyncNode) {
+  int32_t ret;
   assert(pSyncNode != NULL);
+
+  ret = raftStoreClose(pSyncNode->pRaftStore);
+  assert(ret == 0);
+
+  voteGrantedDestroy(pSyncNode->pVotesGranted);
+  votesRespondDestory(pSyncNode->pVotesRespond);
+  syncIndexMgrDestroy(pSyncNode->pNextIndex);
+  syncIndexMgrDestroy(pSyncNode->pMatchIndex);
+  logStoreDestory(pSyncNode->pLogStore);
+
+  syncNodeStopPingTimer(pSyncNode);
+  syncNodeStopElectTimer(pSyncNode);
+  syncNodeStopHeartbeatTimer(pSyncNode);
+
   free(pSyncNode);
 }
 
+// ping --------------
+int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) {
+  syncPingLog2((char*)"==syncNodePing==", pMsg);
+  int32_t ret = 0;
+
+  SRpcMsg rpcMsg;
+  syncPing2RpcMsg(pMsg, &rpcMsg);
+  syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg);
+
+  ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);
+  return ret;
+}
+
+int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
+  int32_t   ret = 0;
+  SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
+  ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg);
+  assert(ret == 0);
+
+  syncPingDestroy(pMsg);
+  return ret;
+}
+
+int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    SRaftId destId;
+    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
+    ret = syncNodePing(pSyncNode, &destId, pMsg);
+    assert(ret == 0);
+    syncPingDestroy(pMsg);
+  }
+  return ret;
+}
+
+int32_t syncNodePingAll(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) {
+    SRaftId destId;
+    syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
+    ret = syncNodePing(pSyncNode, &destId, pMsg);
+    assert(ret == 0);
+    syncPingDestroy(pMsg);
+  }
+  return ret;
+}
+
+// timer control --------------
+int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
+               &pSyncNode->pPingTimer);
+  atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser);
+  return ret;
+}
+
+int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1);
+  taosTmrStop(pSyncNode->pPingTimer);
+  pSyncNode->pPingTimer = NULL;
+  return ret;
+}
+
+int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
+  int32_t ret = 0;
+  pSyncNode->electTimerMS = ms;
+  taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager,
+               &pSyncNode->pElectTimer);
+  atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser);
+  return ret;
+}
+
+int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1);
+  taosTmrStop(pSyncNode->pElectTimer);
+  pSyncNode->pElectTimer = NULL;
+  return ret;
+}
+
+int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
+  int32_t ret = 0;
+  syncNodeStopElectTimer(pSyncNode);
+  syncNodeStartElectTimer(pSyncNode, ms);
+  return ret;
+}
+
+int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  int32_t electMS = syncUtilElectRandomMS();
+  ret = syncNodeRestartElectTimer(pSyncNode, electMS);
+  return ret;
+}
+
+int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager,
+               &pSyncNode->pHeartbeatTimer);
+  atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
+  return ret;
+}
+
+int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1);
+  taosTmrStop(pSyncNode->pHeartbeatTimer);
+  pSyncNode->pHeartbeatTimer = NULL;
+  return ret;
+}
+
+// utils --------------
+int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
+  SEpSet epSet;
+  syncUtilraftId2EpSet(destRaftId, &epSet);
+  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  return 0;
+}
+
+int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
+  SEpSet epSet;
+  syncUtilnodeInfo2EpSet(nodeInfo, &epSet);
+  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
+  return 0;
+}
+
 cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   char   u64buf[128];
   cJSON* pRoot = cJSON_CreateObject();
@@ -253,12 +406,22 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   // tla+ server vars
   cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state);
   cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state));
+  char tmpBuf[RAFT_STORE_BLOCK_SIZE];
+  raftStoreSerialize(pSyncNode->pRaftStore, tmpBuf, sizeof(tmpBuf));
+  cJSON_AddStringToObject(pRoot, "pRaftStore", tmpBuf);
 
   // tla+ candidate vars
+  cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted));
+  cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond));
 
   // tla+ leader vars
+  cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex));
+  cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex));
 
   // tla+ log vars
+  cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore));
+  snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->commitIndex);
+  cJSON_AddStringToObject(pRoot, "commitIndex", u64buf);
 
   // ping timer
   snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer);
@@ -268,8 +431,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser);
   cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf);
-  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB);
-  cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf);
+  cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter);
   cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf);
 
@@ -281,8 +444,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser);
   cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf);
-  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB);
-  cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf);
+  cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter);
   cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf);
 
@@ -294,8 +457,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser);
   cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf);
-  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB);
-  cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf);
+  cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter);
   cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf);
 
@@ -327,6 +490,107 @@ char* syncNode2Str(const SSyncNode* pSyncNode) {
   return serialized;
 }
 
+// raft state change --------------
+void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) {
+  if (term > pSyncNode->pRaftStore->currentTerm) {
+    raftStoreSetTerm(pSyncNode->pRaftStore, term);
+    syncNodeBecomeFollower(pSyncNode);
+    raftStoreClearVote(pSyncNode->pRaftStore);
+  }
+}
+
+void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
+  if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+    pSyncNode->leaderCache = EMPTY_RAFT_ID;
+  }
+
+  pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
+  syncNodeStopHeartbeatTimer(pSyncNode);
+
+  int32_t electMS = syncUtilElectRandomMS();
+  syncNodeRestartElectTimer(pSyncNode, electMS);
+}
+
+// TLA+ Spec
+// \* Candidate i transitions to leader.
+// BecomeLeader(i) ==
+//     /\ state[i] = Candidate
+//     /\ votesGranted[i] \in Quorum
+//     /\ state'      = [state EXCEPT ![i] = Leader]
+//     /\ nextIndex'  = [nextIndex EXCEPT ![i] =
+//                          [j \in Server |-> Len(log[i]) + 1]]
+//     /\ matchIndex' = [matchIndex EXCEPT ![i] =
+//                          [j \in Server |-> 0]]
+//     /\ elections'  = elections \cup
+//                          {[eterm     |-> currentTerm[i],
+//                            eleader   |-> i,
+//                            elog      |-> log[i],
+//                            evotes    |-> votesGranted[i],
+//                            evoterLog |-> voterLog[i]]}
+//     /\ UNCHANGED <<messages, currentTerm, votedFor, candidateVars, logVars>>
+//
+void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
+  pSyncNode->state = TAOS_SYNC_STATE_LEADER;
+  pSyncNode->leaderCache = pSyncNode->myRaftId;
+
+  for (int i = 0; i < pSyncNode->pNextIndex->replicaNum; ++i) {
+    pSyncNode->pNextIndex->index[i] = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1;
+  }
+
+  for (int i = 0; i < pSyncNode->pMatchIndex->replicaNum; ++i) {
+    pSyncNode->pMatchIndex->index[i] = SYNC_INDEX_INVALID;
+  }
+
+  syncNodeStopElectTimer(pSyncNode);
+  syncNodeStartHeartbeatTimer(pSyncNode);
+  syncNodeReplicate(pSyncNode);
+}
+
+void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+  assert(voteGrantedMajority(pSyncNode->pVotesGranted));
+  syncNodeBecomeLeader(pSyncNode);
+}
+
+void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER);
+  pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE;
+}
+
+void syncNodeLeader2Follower(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
+  syncNodeBecomeFollower(pSyncNode);
+}
+
+void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+  syncNodeBecomeFollower(pSyncNode);
+}
+
+// raft vote --------------
+void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) {
+  assert(term == pSyncNode->pRaftStore->currentTerm);
+  assert(!raftStoreHasVoted(pSyncNode->pRaftStore));
+
+  raftStoreVote(pSyncNode->pRaftStore, pRaftId);
+}
+
+void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
+  syncNodeVoteForTerm(pSyncNode, pSyncNode->pRaftStore->currentTerm, &(pSyncNode->myRaftId));
+
+  SyncRequestVoteReply* pMsg = syncRequestVoteReplyBuild();
+  pMsg->srcId = pSyncNode->myRaftId;
+  pMsg->destId = pSyncNode->myRaftId;
+  pMsg->term = pSyncNode->pRaftStore->currentTerm;
+  pMsg->voteGranted = true;
+
+  voteGrantedVote(pSyncNode->pVotesGranted, pMsg);
+  votesRespondAdd(pSyncNode->pVotesRespond, pMsg);
+  syncRequestVoteReplyDestroy(pMsg);
+}
+
+void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {}
+
 // for debug --------------
 void syncNodePrint(SSyncNode* pObj) {
   char* serialized = syncNode2Str(pObj);
@@ -354,199 +618,24 @@ void syncNodeLog2(char* s, SSyncNode* pObj) {
   free(serialized);
 }
 
-int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
-  SEpSet epSet;
-  syncUtilraftId2EpSet(destRaftId, &epSet);
-  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
-  return 0;
-}
-
-int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
-  SEpSet epSet;
-  syncUtilnodeInfo2EpSet(nodeInfo, &epSet);
-  pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, pMsg);
-  return 0;
-}
-
-int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) {
-  sTrace("syncNodePing pSyncNode:%p ", pSyncNode);
-  int32_t ret = 0;
-
-  SRpcMsg rpcMsg;
-  syncPing2RpcMsg(pMsg, &rpcMsg);
-  syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);
-
-  {
-    cJSON* pJson = syncPing2Json(pMsg);
-    char*  serialized = cJSON_Print(pJson);
-    sTrace("syncNodePing pMsg:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
-
-  {
-    SyncPing* pMsg2 = rpcMsg.pCont;
-    cJSON*    pJson = syncPing2Json(pMsg2);
-    char*     serialized = cJSON_Print(pJson);
-    sTrace("syncNodePing rpcMsg.pCont:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
-
-  return ret;
-}
-
-int32_t syncNodePingAll(SSyncNode* pSyncNode) {
-  sTrace("syncNodePingAll pSyncNode:%p ", pSyncNode);
-  int32_t ret = 0;
-  for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) {
-    SRaftId destId;
-    syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
-    ret = syncNodePing(pSyncNode, &destId, pMsg);
-    assert(ret == 0);
-    syncPingDestroy(pMsg);
-  }
-}
-
-int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
-  int32_t ret = 0;
-  for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    SRaftId destId;
-    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
-    ret = syncNodePing(pSyncNode, &destId, pMsg);
-    assert(ret == 0);
-    syncPingDestroy(pMsg);
-  }
-}
-
-int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
-  int32_t   ret;
-  SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
-  ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg);
-  assert(ret == 0);
-  syncPingDestroy(pMsg);
-}
-
-int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
-  atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser);
-  pSyncNode->pingTimerMS = PING_TIMER_MS;
-  if (pSyncNode->pPingTimer == NULL) {
-    pSyncNode->pPingTimer =
-        taosTmrStart(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager);
-  } else {
-    taosTmrReset(pSyncNode->FpPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
-                 &pSyncNode->pPingTimer);
-  }
-  return 0;
-}
-
-int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) {
-  atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1);
-  pSyncNode->pingTimerMS = TIMER_MAX_MS;
-  return 0;
-}
-
-int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
-  pSyncNode->electTimerMS = ms;
-  atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser);
-  if (pSyncNode->pElectTimer == NULL) {
-    pSyncNode->pElectTimer =
-        taosTmrStart(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager);
-  } else {
-    taosTmrReset(pSyncNode->FpElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager,
-                 &pSyncNode->pElectTimer);
-  }
-  return 0;
-}
-
-int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) {
-  atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1);
-  pSyncNode->electTimerMS = TIMER_MAX_MS;
-  return 0;
-}
-
-int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
-  syncNodeStopElectTimer(pSyncNode);
-  syncNodeStartElectTimer(pSyncNode, ms);
-  return 0;
-}
-
-int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) {
-  atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
-  if (pSyncNode->pHeartbeatTimer == NULL) {
-    pSyncNode->pHeartbeatTimer =
-        taosTmrStart(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager);
-  } else {
-    taosTmrReset(pSyncNode->FpHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager,
-                 &pSyncNode->pHeartbeatTimer);
-  }
-  return 0;
-}
-
-int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) {
-  atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1);
-  pSyncNode->heartbeatTimerMS = TIMER_MAX_MS;
-  return 0;
-}
-
 // ------ local funciton ---------
-static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
+// enqueue message ----
-  int32_t ret = 0;
-  sTrace("<-- syncNodeOnPingCb -->");
-
-  {
-    cJSON* pJson = syncPing2Json(pMsg);
-    char*  serialized = cJSON_Print(pJson);
-    sTrace("process syncMessage recv: syncNodeOnPingCb pMsg:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
-
-  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId);
-  SRpcMsg        rpcMsg;
-  syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
-  syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
-
-  return ret;
-}
-
-static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
-  int32_t ret = 0;
-  sTrace("<-- syncNodeOnPingReplyCb -->");
-
-  {
-    cJSON* pJson = syncPingReply2Json(pMsg);
-    char*  serialized = cJSON_Print(pJson);
-    sTrace("process syncMessage recv: syncNodeOnPingReplyCb pMsg:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
-
-  return ret;
-}
-
 static void syncNodeEqPingTimer(void* param, void* tmrId) {
-  sTrace("<-- syncNodeEqPingTimer -->");
-
   SSyncNode* pSyncNode = (SSyncNode*)param;
   if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) {
     SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock),
                                               pSyncNode->pingTimerMS, pSyncNode);
     SRpcMsg      rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
+    syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
     syncTimeoutDestroy(pSyncMsg);
 
-    // reset timer ms
-    // pSyncNode->pingTimerMS += 100;
-
     taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
                  &pSyncNode->pPingTimer);
   } else {
-    sTrace("syncNodeEqPingTimer: pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", pSyncNode->pingTimerLogicClock,
+    sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu",
-           pSyncNode->pingTimerLogicClockUser);
+           pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser);
   }
 }
 
@@ -555,19 +644,18 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
   if (atomic_load_64(&pSyncNode->electTimerLogicClockUser) <= atomic_load_64(&pSyncNode->electTimerLogicClock)) {
     SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_ELECTION, atomic_load_64(&pSyncNode->electTimerLogicClock),
                                               pSyncNode->electTimerMS, pSyncNode);
-
+    SRpcMsg      rpcMsg;
-    SRpcMsg rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
+    syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
     syncTimeoutDestroy(pSyncMsg);
 
     // reset timer ms
     pSyncNode->electTimerMS = syncUtilElectRandomMS();
-
     taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
                  &pSyncNode->pPingTimer);
   } else {
-    sTrace("syncNodeEqElectTimer: electTimerLogicClock:%lu, electTimerLogicClockUser:%lu",
+    sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%lu, electTimerLogicClockUser:%lu",
            pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser);
   }
 }
@@ -579,80 +667,50 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
     SyncTimeout* pSyncMsg =
         syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock),
                           pSyncNode->heartbeatTimerMS, pSyncNode);
-
     SRpcMsg rpcMsg;
     syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
+    syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
     syncTimeoutDestroy(pSyncMsg);
 
-    // reset timer ms
-    // pSyncNode->heartbeatTimerMS += 100;
-
     taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager,
                  &pSyncNode->pHeartbeatTimer);
   } else {
-    sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu",
+    sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu",
            pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
   }
 }
 
-static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) {
+// on message ----
-  if (term > pSyncNode->pRaftStore->currentTerm) {
+static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
-    pSyncNode->pRaftStore->currentTerm = term;
+  int32_t ret = 0;
-    pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID;
+  syncPingLog2("==syncNodeOnPingCb==", pMsg);
-    raftStorePersist(pSyncNode->pRaftStore);
+  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId);
-    syncNodeBecomeFollower(pSyncNode);
+  SRpcMsg        rpcMsg;
-  }
+  syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
+  syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
+
+  return ret;
 }
 
-static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
+static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
-  if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
+  int32_t ret = 0;
-    pSyncNode->leaderCache = EMPTY_RAFT_ID;
+  syncPingReplyLog2("==syncNodeOnPingReplyCb==", pMsg);
+  return ret;
+}
+
+static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
+  int32_t ret = 0;
+  syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
+
+  if (ths->state == TAOS_SYNC_STATE_LEADER) {
+    SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen);
+    ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    syncNodeReplicate(ths);
+    syncEntryDestory(pEntry);
+  } else {
+    // ths->pFsm->FpCommitCb(-1)
   }
 
-  syncNodeStopHeartbeatTimer(pSyncNode);
+  return ret;
-  int32_t electMS = syncUtilElectRandomMS();
+}
-  syncNodeStartElectTimer(pSyncNode, electMS);
-}
-
-// TLA+ Spec
-// \* Candidate i transitions to leader.
-// BecomeLeader(i) ==
-//     /\ state[i] = Candidate
-//     /\ votesGranted[i] \in Quorum
-//     /\ state'      = [state EXCEPT ![i] = Leader]
-//     /\ nextIndex'  = [nextIndex EXCEPT ![i] =
-//                          [j \in Server |-> Len(log[i]) + 1]]
-//     /\ matchIndex' = [matchIndex EXCEPT ![i] =
-//                          [j \in Server |-> 0]]
-//     /\ elections'  = elections \cup
-//                          {[eterm     |-> currentTerm[i],
-//                            eleader   |-> i,
-//                            elog      |-> log[i],
-//                            evotes    |-> votesGranted[i],
-//                            evoterLog |-> voterLog[i]]}
-//     /\ UNCHANGED <<messages, currentTerm, votedFor, candidateVars, logVars>>
-//
-static void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
-  pSyncNode->state = TAOS_SYNC_STATE_LEADER;
-  pSyncNode->leaderCache = pSyncNode->myRaftId;
-
-  // next Index +=1
-  // match Index = 0;
-
-  syncNodeStopElectTimer(pSyncNode);
-  syncNodeStartHeartbeatTimer(pSyncNode);
-
-  // appendEntries;
-}
-
-static void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
-  assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER);
-  pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE;
-}
-
-static void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {}
-
-static void syncNodeLeader2Follower(SSyncNode* pSyncNode) {}
-
-static void syncNodeCandidate2Follower(SSyncNode* pSyncNode) {}

source/libs/sync/src/syncMessage.c

@@ -65,10 +65,32 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
     pRoot = syncAppendEntriesReply2Json(pSyncMsg);
     syncAppendEntriesReplyDestroy(pSyncMsg);
 
+  } else if (pRpcMsg->msgType == SYNC_RESPONSE) {
+    pRoot = cJSON_CreateObject();
+    char* s;
+    s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen);
+    cJSON_AddStringToObject(pRoot, "pCont", s);
+    free(s);
+    s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen);
+    cJSON_AddStringToObject(pRoot, "pCont2", s);
+    free(s);
+
   } else {
     pRoot = syncRpcUnknownMsg2Json();
+    char* s;
+    s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen);
+    cJSON_AddStringToObject(pRoot, "pCont", s);
+    free(s);
+    s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen);
+    cJSON_AddStringToObject(pRoot, "pCont2", s);
+    free(s);
   }
 
+  cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType);
+  cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen);
+  cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code);
+  // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist);
+
   cJSON* pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "RpcMsg", pRoot);
   return pJson;
@@ -77,7 +99,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
 cJSON* syncRpcUnknownMsg2Json() {
   cJSON* pRoot = cJSON_CreateObject();
   cJSON_AddNumberToObject(pRoot, "msgType", SYNC_UNKNOWN);
-  cJSON_AddStringToObject(pRoot, "data", "known message");
+  cJSON_AddStringToObject(pRoot, "data", "unknown message");
 
   cJSON* pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot);
@@ -798,8 +820,8 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) {
   cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
   cJSON_AddItemToObject(pRoot, "destId", pDestId);
 
-  snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->currentTerm);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
-  cJSON_AddStringToObject(pRoot, "currentTerm", u64buf);
+  cJSON_AddStringToObject(pRoot, "term", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex);
   cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm);
@@ -1086,6 +1108,9 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) {
   cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
   cJSON_AddItemToObject(pRoot, "destId", pDestId);
 
+  snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
+  cJSON_AddStringToObject(pRoot, "term", u64buf);
+
   snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex);
   cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf);
 
@@ -1242,9 +1267,11 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) {
   cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
   cJSON_AddItemToObject(pRoot, "destId", pDestId);
 
+  snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term);
+  cJSON_AddStringToObject(pRoot, "term", u64buf);
   cJSON_AddNumberToObject(pRoot, "success", pMsg->success);
   snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex);
-  cJSON_AddStringToObject(pRoot, "match_index", u64buf);
+  cJSON_AddStringToObject(pRoot, "matchIndex", u64buf);
 
   cJSON* pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "SyncAppendEntriesReply", pRoot);
@@ -1283,4 +1310,4 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
   char* serialized = syncAppendEntriesReply2Str(pMsg);
   sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
   free(serialized);
-}
+}

source/libs/sync/src/syncRaftLog.c

@@ -130,7 +130,7 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) {
   cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
   cJSON_AddStringToObject(pRoot, "pWal", u64buf);
-  snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastIndex(pLogStore));
+  snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore));
   cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
   snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore));
   cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);

source/libs/sync/src/syncRaftStore.c

@@ -15,6 +15,8 @@
 
 #include "syncRaftStore.h"
 #include "cJSON.h"
+#include "syncEnv.h"
+#include "syncUtil.h"
 
 // private function
 static int32_t raftStoreInit(SRaftStore *pRaftStore);
@@ -135,6 +137,33 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) {
   return 0;
 }
 
+bool raftStoreHasVoted(SRaftStore *pRaftStore) {
+  bool b = syncUtilEmptyId(&(pRaftStore->voteFor));
+  return b;
+}
+
+void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) {
+  assert(!raftStoreHasVoted(pRaftStore));
+  assert(!syncUtilEmptyId(pRaftId));
+  pRaftStore->voteFor = *pRaftId;
+  raftStorePersist(pRaftStore);
+}
+
+void raftStoreClearVote(SRaftStore *pRaftStore) {
+  pRaftStore->voteFor = EMPTY_RAFT_ID;
+  raftStorePersist(pRaftStore);
+}
+
+void raftStoreNextTerm(SRaftStore *pRaftStore) {
+  ++(pRaftStore->currentTerm);
+  raftStorePersist(pRaftStore);
+}
+
+void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) {
+  pRaftStore->currentTerm = term;
+  raftStorePersist(pRaftStore);
+}
+
 // for debug -------------------
 void raftStorePrint(SRaftStore *pObj) {
   char serialized[RAFT_STORE_BLOCK_SIZE];

source/libs/sync/src/syncReplication.c

@@ -14,7 +14,11 @@
  */
 
 #include "syncReplication.h"
+#include "syncIndexMgr.h"
 #include "syncMessage.h"
+#include "syncRaftEntry.h"
+#include "syncRaftLog.h"
+#include "syncUtil.h"
 
 // TLA+ Spec
 // AppendEntries(i, j) ==
@@ -42,7 +46,39 @@
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER);
+
   int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    SRaftId*  pDestId = &(pSyncNode->peersId[i]);
+    SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId);
+
+    SyncIndex preLogIndex = nextIndex - 1;
+
+    SyncTerm preLogTerm = 0;
+    if (preLogIndex >= SYNC_INDEX_BEGIN) {
+      SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex);
+      preLogTerm = pPreEntry->term;
+    }
+
+    SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex);
+    assert(nextIndex == lastIndex);
+
+    SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex);
+    assert(pEntry != NULL);
+
+    SyncAppendEntries* pMsg = syncAppendEntriesBuild(pEntry->bytes);
+    pMsg->srcId = pSyncNode->myRaftId;
+    pMsg->destId = *pDestId;
+    pMsg->prevLogIndex = preLogIndex;
+    pMsg->prevLogTerm = preLogTerm;
+    pMsg->commitIndex = pSyncNode->commitIndex;
+    pMsg->dataLen = pEntry->bytes;
+    // add pEntry into msg
+
+    syncNodeAppendEntries(pSyncNode, pDestId, pMsg);
+  }
+
   return ret;
 }
 

source/libs/sync/src/syncRequestVote.c

@@ -14,6 +14,10 @@
  */
 
 #include "syncRequestVote.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
 
 // TLA+ Spec
 // HandleRequestVoteRequest(i, j, m) ==
@@ -37,4 +41,34 @@
 //                 m)
 //       /\ UNCHANGED <<state, currentTerm, candidateVars, leaderVars, logVars>>
 //
-int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {}
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
+  int32_t ret = 0;
+  syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg);
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    syncNodeUpdateTerm(ths, pMsg->term);
+  }
+  assert(pMsg->term <= ths->pRaftStore->currentTerm);
+
+  bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) ||
+               ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) &&
+                (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore)));
+  bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK &&
+               ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId))));
+  if (grant) {
+    raftStoreVote(ths->pRaftStore, &(pMsg->srcId));
+  }
+
+  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild();
+  pReply->srcId = ths->myRaftId;
+  pReply->destId = pMsg->srcId;
+  pReply->term = ths->pRaftStore->currentTerm;
+  pReply->voteGranted = grant;
+
+  SRpcMsg rpcMsg;
+  syncRequestVoteReply2RpcMsg(pReply, &rpcMsg);
+  syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg);
+  syncRequestVoteReplyDestroy(pReply);
+
+  return ret;
+}

source/libs/sync/src/syncRequestVoteReply.c

@@ -14,6 +14,10 @@
  */
 
 #include "syncRequestVoteReply.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
 
 // TLA+ Spec
 // HandleRequestVoteResponse(i, j, m) ==
@@ -32,4 +36,33 @@
 //    /\ Discard(m)
 //    /\ UNCHANGED <<serverVars, votedFor, leaderVars, logVars>>
 //
-int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {}
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) {
+  int32_t ret = 0;
+  syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg);
+
+  if (pMsg->term < ths->pRaftStore->currentTerm) {
+    sTrace("DropStaleResponse, receive term:%lu, current term:%lu", pMsg->term, ths->pRaftStore->currentTerm);
+    return ret;
+  }
+
+  // no need this code, because if I receive reply.term, then I must have sent for that term.
+  //  if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //    syncNodeUpdateTerm(ths, pMsg->term);
+  //  }
+
+  assert(pMsg->term == ths->pRaftStore->currentTerm);
+
+  if (ths->state == TAOS_SYNC_STATE_CANDIDATE) {
+    votesRespondAdd(ths->pVotesRespond, pMsg);
+    if (pMsg->voteGranted) {
+      voteGrantedVote(ths->pVotesGranted, pMsg);
+      if (voteGrantedMajority(ths->pVotesGranted)) {
+        if (ths->pVotesGranted->toLeader) {
+          syncNodeCandidate2Leader(ths);
+          ths->pVotesGranted->toLeader = true;
+        }
+      }
+    }
+  }
+  return ret;
+}

source/libs/sync/src/syncTimeout.c

@@ -19,15 +19,7 @@
 
 int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) {
   int32_t ret = 0;
-  sTrace("<-- syncNodeOnTimeoutCb -->");
+  syncTimeoutLog2("==syncNodeOnTimeoutCb==", pMsg);
-
-  {
-    cJSON* pJson = syncTimeout2Json(pMsg);
-    char*  serialized = cJSON_Print(pJson);
-    sTrace("process syncMessage recv: syncNodeOnTimeoutCb pMsg:%s ", serialized);
-    free(serialized);
-    cJSON_Delete(pJson);
-  }
 
   if (pMsg->timeoutType == SYNC_TIMEOUT_PING) {
     if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) {

source/libs/sync/src/syncUtil.c

@@ -74,6 +74,8 @@ bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) {
   return ret;
 }
 
+bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); }
+
 // ---- SSyncBuffer -----
 void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) {
   syncBuf->len = len;
@@ -184,4 +186,14 @@ char* syncUtilprintBin2(char* ptr, uint32_t len) {
     p += n;
   }
   return s;
+}
+
+SyncIndex syncUtilMinIndex(SyncIndex a, SyncIndex b) {
+  SyncIndex r = a < b ? a : b;
+  return r;
+}
+
+SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) {
+  SyncIndex r = a > b ? a : b;
+  return r;
 }

source/libs/sync/test/CMakeLists.txt

@@ -4,8 +4,8 @@ add_executable(syncPingTimerTest "")
 add_executable(syncIOTickQTest "")
 add_executable(syncIOTickPingTest "")
 add_executable(syncIOSendMsgTest "")
-add_executable(syncIOSendMsgClientTest "")
+add_executable(syncIOClientTest "")
-add_executable(syncIOSendMsgServerTest "")
+add_executable(syncIOServerTest "")
 add_executable(syncRaftStoreTest "")
 add_executable(syncEnqTest "")
 add_executable(syncIndexTest "")
@@ -51,13 +51,13 @@ target_sources(syncIOSendMsgTest
     PRIVATE
     "syncIOSendMsgTest.cpp"
 )
-target_sources(syncIOSendMsgClientTest
+target_sources(syncIOClientTest
     PRIVATE
-    "syncIOSendMsgClientTest.cpp"
+    "syncIOClientTest.cpp"
 )
-target_sources(syncIOSendMsgServerTest
+target_sources(syncIOServerTest
     PRIVATE
-    "syncIOSendMsgServerTest.cpp"
+    "syncIOServerTest.cpp"
 )
 target_sources(syncRaftStoreTest
     PRIVATE
@@ -167,12 +167,12 @@ target_include_directories(syncIOSendMsgTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncIOSendMsgClientTest
+target_include_directories(syncIOClientTest
     PUBLIC
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncIOSendMsgServerTest
+target_include_directories(syncIOServerTest
     PUBLIC
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
@@ -298,11 +298,11 @@ target_link_libraries(syncIOSendMsgTest
     sync
     gtest_main
 )
-target_link_libraries(syncIOSendMsgClientTest
+target_link_libraries(syncIOClientTest
     sync
     gtest_main
 )
-target_link_libraries(syncIOSendMsgServerTest
+target_link_libraries(syncIOServerTest
     sync
     gtest_main
 )

source/libs/sync/test/syncEnqTest.cpp

@@ -1,9 +1,10 @@
+#include <gtest/gtest.h>
 #include <stdio.h>
 #include "syncEnv.h"
 #include "syncIO.h"
 #include "syncInt.h"
-#include "syncMessage.h"
 #include "syncRaftStore.h"
+#include "syncUtil.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -14,64 +15,69 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-uint16_t ports[3] = {7010, 7110, 7210};
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 5;
+int32_t  myIndex = 0;
 
-SSyncNode* doSync(int myIndex) {
+SRaftId   ids[TSDB_MAX_REPLICA];
-  SSyncFSM* pFsm;
+SSyncInfo syncInfo;
+SSyncFSM* pFsm;
 
-  SSyncInfo syncInfo;
+SSyncNode* syncNodeInit() {
-  syncInfo.vgId = 1;
+  syncInfo.vgId = 1234;
   syncInfo.rpcClient = gSyncIO->clientRpc;
   syncInfo.FpSendMsg = syncIOSendMsg;
   syncInfo.queue = gSyncIO->pMsgQ;
   syncInfo.FpEqMsg = syncIOEqMsg;
   syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
 
   SSyncCfg* pCfg = &syncInfo.syncCfg;
   pCfg->myIndex = myIndex;
-  pCfg->replicaNum = 3;
+  pCfg->replicaNum = replicaNum;
 
-  pCfg->nodeInfo[0].nodePort = ports[0];
+  for (int i = 0; i < replicaNum; ++i) {
-  snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
+    pCfg->nodeInfo[i].nodePort = ports[i];
-  // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
-
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
-  pCfg->nodeInfo[1].nodePort = ports[1];
+  }
-  snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
-
-  pCfg->nodeInfo[2].nodePort = ports[2];
-  snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
 
   SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
   assert(pSyncNode != NULL);
 
   gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
   gSyncIO->pSyncNode = pSyncNode;
 
   return pSyncNode;
 }
 
-void timerPingAll(void* param, void* tmrId) {
+SSyncNode* syncInitTest() { return syncNodeInit(); }
-  SSyncNode* pSyncNode = (SSyncNode*)param;
+
-  syncNodePingAll(pSyncNode);
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
 }
 
 int main(int argc, char** argv) {
-  // taosInitLog((char*)"syncPingTest.log", 100000, 10);
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
   sDebugFlag = 143 + 64;
 
-  logTest();
+  myIndex = 0;
-
-  int myIndex = 0;
   if (argc >= 2) {
     myIndex = atoi(argv[1]);
-    if (myIndex > 2 || myIndex < 0) {
-      fprintf(stderr, "myIndex:%d error. should be 0 - 2", myIndex);
-      return 1;
-    }
   }
 
   int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
@@ -80,21 +86,22 @@ int main(int argc, char** argv) {
   ret = syncEnvStart();
   assert(ret == 0);
 
-  SSyncNode* pSyncNode = doSync(myIndex);
+  SSyncNode* pSyncNode = syncInitTest();
-  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  assert(pSyncNode != NULL);
-  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+
+  syncNodePrint2((char*)"syncInitTest", pSyncNode);
+
+  initRaftId(pSyncNode);
+
+  //--------------------------------------------------------------
 
   for (int i = 0; i < 10; ++i) {
-    SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
+    SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncEnqTest");
     SRpcMsg        rpcMsg;
     syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
     taosMsleep(1000);
   }
 
-  while (1) {
-    taosMsleep(1000);
-  }
-
   return 0;
 }

source/libs/sync/test/syncEnvTest.cpp

@@ -14,15 +14,6 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-void *pTimer = NULL;
-void *pTimerMgr = NULL;
-int   g = 300;
-
-static void timerFp(void *param, void *tmrId) {
-  printf("param:%p, tmrId:%p, pTimer:%p, pTimerMgr:%p \n", param, tmrId, pTimer, pTimerMgr);
-  taosTmrReset(timerFp, 1000, param, pTimerMgr, &pTimer);
-}
-
 int main() {
   // taosInitLog((char*)"syncEnvTest.log", 100000, 10);
   tsAsyncLog = 0;
@@ -34,13 +25,20 @@ int main() {
   ret = syncEnvStart();
   assert(ret == 0);
 
-  // timer
+  for (int i = 0; i < 5; ++i) {
-  pTimerMgr = taosTmrInit(1000, 50, 10000, "SYNC-ENV-TEST");
+    ret = syncEnvStartTimer();
-  taosTmrStart(timerFp, 1000, &g, pTimerMgr);
+    assert(ret == 0);
 
-  while (1) {
+    taosMsleep(5000);
-    taosMsleep(1000);
+
+    ret = syncEnvStopTimer();
+    assert(ret == 0);
+
+    taosMsleep(5000);
   }
 
+  ret = syncEnvStop();
+  assert(ret == 0);
+
   return 0;
 }

source/libs/sync/test/syncIOClientTest.cpp

@@ -2,7 +2,8 @@
 #include <stdio.h>
 #include "syncIO.h"
 #include "syncInt.h"
-#include "syncRaftStore.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -22,7 +23,7 @@ int main() {
 
   int32_t ret;
 
-  ret = syncIOStart((char *)"127.0.0.1", 7010);
+  ret = syncIOStart((char*)"127.0.0.1", 7010);
   assert(ret == 0);
 
   for (int i = 0; i < 10; ++i) {
@@ -31,20 +32,19 @@ int main() {
     epSet.numOfEps = 0;
     addEpIntoEpSet(&epSet, "127.0.0.1", 7030);
 
-    SRpcMsg rpcMsg;
+    SRaftId srcId, destId;
-    rpcMsg.contLen = 64;
+    srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
-    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+    srcId.vgId = 100;
-    snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest");
+    destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
-    rpcMsg.handle = NULL;
+    destId.vgId = 100;
-    rpcMsg.msgType = 77;
+
+    SyncPingReply* pSyncMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOClientTest");
+    SRpcMsg        rpcMsg;
+    syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
 
     syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg);
     taosSsleep(1);
   }
 
-  while (1) {
-    taosSsleep(1);
-  }
-
   return 0;
 }

source/libs/sync/test/syncIOSendMsgTest.cpp

@@ -1,8 +1,10 @@
 #include <gtest/gtest.h>
 #include <stdio.h>
+#include "syncEnv.h"
 #include "syncIO.h"
 #include "syncInt.h"
 #include "syncRaftStore.h"
+#include "syncUtil.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -13,37 +15,96 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-int main() {
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 5;
+int32_t  myIndex = 0;
+
+SRaftId   ids[TSDB_MAX_REPLICA];
+SSyncInfo syncInfo;
+SSyncFSM* pFsm;
+
+SSyncNode* syncNodeInit() {
+  syncInfo.vgId = 1234;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = pFsm;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+  pCfg->myIndex = myIndex;
+  pCfg->replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    pCfg->nodeInfo[i].nodePort = ports[i];
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  }
+
+  SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
+  assert(pSyncNode != NULL);
+
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
+  gSyncIO->pSyncNode = pSyncNode;
+
+  return pSyncNode;
+}
+
+SSyncNode* syncInitTest() { return syncNodeInit(); }
+
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
+}
+
+int main(int argc, char** argv) {
   // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
   sDebugFlag = 143 + 64;
 
-  logTest();
+  myIndex = 0;
-
+  if (argc >= 2) {
-  int32_t ret;
+    myIndex = atoi(argv[1]);
-
-  ret = syncIOStart((char *)"127.0.0.1", 7010);
-  assert(ret == 0);
-
-  for (int i = 0; i < 10; ++i) {
-    SEpSet epSet;
-    epSet.inUse = 0;
-    epSet.numOfEps = 0;
-    addEpIntoEpSet(&epSet, "127.0.0.1", 7010);
-
-    SRpcMsg rpcMsg;
-    rpcMsg.contLen = 64;
-    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
-    snprintf((char *)rpcMsg.pCont, rpcMsg.contLen, "%s", "syncIOSendMsgTest");
-    rpcMsg.handle = NULL;
-    rpcMsg.msgType = 77;
-
-    syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg);
-    taosSsleep(1);
   }
 
-  while (1) {
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
-    taosSsleep(1);
+  assert(ret == 0);
+
+  ret = syncEnvStart();
+  assert(ret == 0);
+
+  SSyncNode* pSyncNode = syncInitTest();
+  assert(pSyncNode != NULL);
+
+  syncNodePrint2((char*)"syncInitTest", pSyncNode);
+
+  initRaftId(pSyncNode);
+
+  //--------------------------------------------------------------
+
+  for (int i = 0; i < 10; ++i) {
+    SyncPingReply* pSyncMsg = syncPingReplyBuild2(&pSyncNode->myRaftId, &pSyncNode->myRaftId, "syncIOSendMsgTest");
+    SRpcMsg        rpcMsg;
+    syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
+
+    SEpSet epSet;
+    syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet);
+    pSyncNode->FpSendMsg(pSyncNode->rpcClient, &epSet, &rpcMsg);
+
+    taosMsleep(1000);
   }
 
   return 0;

source/libs/sync/test/syncIOTickPingTest.cpp

@@ -25,8 +25,15 @@ int main() {
   ret = syncIOStart((char*)"127.0.0.1", 7010);
   assert(ret == 0);
 
-  ret = syncIOTickPing();
+  for (int i = 0; i < 3; ++i) {
-  assert(ret == 0);
+    ret = syncIOPingTimerStart();
+    assert(ret == 0);
+    taosMsleep(5000);
+
+    ret = syncIOPingTimerStop();
+    assert(ret == 0);
+    taosMsleep(5000);
+  }
 
   while (1) {
     taosSsleep(1);

source/libs/sync/test/syncIOTickQTest.cpp

@@ -25,11 +25,18 @@ int main() {
   ret = syncIOStart((char*)"127.0.0.1", 7010);
   assert(ret == 0);
 
-  ret = syncIOTickQ();
+  for (int i = 0; i < 3; ++i) {
+    ret = syncIOQTimerStart();
+    assert(ret == 0);
+    taosMsleep(5000);
+
+    ret = syncIOQTimerStop();
+    assert(ret == 0);
+    taosMsleep(5000);
+  }
+
+  ret = syncIOStop();
   assert(ret == 0);
 
-  while (1) {
-    taosSsleep(1);
-  }
   return 0;
 }

source/libs/sync/test/syncLogStoreTest.cpp

@@ -81,7 +81,10 @@ SSyncNode* syncNodeInit() {
 SSyncNode* syncInitTest() { return syncNodeInit(); }
 
 void logStoreTest() {
-  logStorePrint(pSyncNode->pLogStore);
+  logStorePrint2((char*)"logStoreTest2", pSyncNode->pLogStore);
+
+  assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID);
+
   for (int i = 0; i < 5; ++i) {
     int32_t         dataLen = 10;
     SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
@@ -97,6 +100,10 @@ void logStoreTest() {
     // syncEntryPrint2((char*)"write entry:", pEntry);
     pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry);
     syncEntryDestory(pEntry);
+
+    if (i == 0) {
+      assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN);
+    }
   }
   logStorePrint(pSyncNode->pLogStore);
 
@@ -129,6 +136,8 @@ int main(int argc, char** argv) {
   ret = syncEnvStart();
   assert(ret == 0);
 
+  taosRemoveDir("./wal_test");
+
   pSyncNode = syncInitTest();
   assert(pSyncNode != NULL);
 

source/libs/sync/test/syncRequestVoteTest.cpp

@@ -20,7 +20,7 @@ SyncRequestVote *createMsg() {
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
   pMsg->destId.vgId = 100;
-  pMsg->currentTerm = 11;
+  pMsg->term = 11;
   pMsg->lastLogIndex = 22;
   pMsg->lastLogTerm = 33;
   return pMsg;

source/libs/sync/test/syncRpcMsgTest.cpp

@@ -57,7 +57,7 @@ SyncRequestVote *createSyncRequestVote() {
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
   pMsg->destId.vgId = 100;
-  pMsg->currentTerm = 11;
+  pMsg->term = 11;
   pMsg->lastLogIndex = 22;
   pMsg->lastLogTerm = 33;
   return pMsg;
@@ -100,7 +100,7 @@ SyncAppendEntriesReply *createSyncAppendEntriesReply() {
 
 void test1() {
   SyncTimeout *pMsg = createSyncTimeout();
-  SRpcMsg          rpcMsg;
+  SRpcMsg      rpcMsg;
   syncTimeout2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test1", &rpcMsg);
   syncTimeoutDestroy(pMsg);
@@ -108,7 +108,7 @@ void test1() {
 
 void test2() {
   SyncPing *pMsg = createSyncPing();
-  SRpcMsg          rpcMsg;
+  SRpcMsg   rpcMsg;
   syncPing2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test2", &rpcMsg);
   syncPingDestroy(pMsg);
@@ -116,7 +116,7 @@ void test2() {
 
 void test3() {
   SyncPingReply *pMsg = createSyncPingReply();
-  SRpcMsg          rpcMsg;
+  SRpcMsg        rpcMsg;
   syncPingReply2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test3", &rpcMsg);
   syncPingReplyDestroy(pMsg);
@@ -132,7 +132,7 @@ void test4() {
 
 void test5() {
   SyncRequestVoteReply *pMsg = createSyncRequestVoteReply();
-  SRpcMsg          rpcMsg;
+  SRpcMsg               rpcMsg;
   syncRequestVoteReply2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test5", &rpcMsg);
   syncRequestVoteReplyDestroy(pMsg);
@@ -140,7 +140,7 @@ void test5() {
 
 void test6() {
   SyncAppendEntries *pMsg = createSyncAppendEntries();
-  SRpcMsg          rpcMsg;
+  SRpcMsg            rpcMsg;
   syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test6", &rpcMsg);
   syncAppendEntriesDestroy(pMsg);
@@ -148,7 +148,7 @@ void test6() {
 
 void test7() {
   SyncAppendEntriesReply *pMsg = createSyncAppendEntriesReply();
-  SRpcMsg          rpcMsg;
+  SRpcMsg                 rpcMsg;
   syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test7", &rpcMsg);
   syncAppendEntriesReplyDestroy(pMsg);
@@ -156,7 +156,7 @@ void test7() {
 
 void test8() {
   SyncClientRequest *pMsg = createSyncClientRequest();
-  SRpcMsg          rpcMsg;
+  SRpcMsg            rpcMsg;
   syncClientRequest2RpcMsg(pMsg, &rpcMsg);
   syncRpcMsgPrint2((char *)"test8", &rpcMsg);
   syncClientRequestDestroy(pMsg);

source/libs/sync/test/syncUtilTest.cpp

@@ -26,6 +26,7 @@ int main() {
   tsAsyncLog = 0;
   sDebugFlag = 143 + 64;
   logTest();
+
   electRandomMSTest();
 
   return 0;

source/libs/transport/inc/transComm.h

@@ -120,6 +120,10 @@ typedef struct {
   // SEpSet*          pSet;      // for synchronous API
 } SRpcReqContext;
 
+typedef SRpcMsg      STransMsg;
+typedef SRpcInfo     STrans;
+typedef SRpcConnInfo STransHandleInfo;
+
 typedef struct {
   SEpSet epSet;    // ip list provided by app
   void*  ahandle;  // handle provided by app
@@ -134,8 +138,8 @@ typedef struct {
   int8_t  connType;  // connection type
   int64_t rid;       // refId returned by taosAddRef
 
-  SRpcMsg* pRsp;  // for synchronous API
+  STransMsg* pRsp;  // for synchronous API
-  tsem_t*  pSem;  // for synchronous API
+  tsem_t*    pSem;  // for synchronous API
 
   int      hThrdIdx;
   char*    ip;
@@ -249,4 +253,15 @@ void transUnrefSrvHandle(void* handle);
 void transRefCliHandle(void* handle);
 void transUnrefCliHandle(void* handle);
 
+void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg);
+void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg, STransMsg* pRsp);
+void transSendResponse(const STransMsg* pMsg);
+int  transGetConnInfo(void* thandle, STransHandleInfo* pInfo);
+
+void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
+void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
+
+void transCloseClient(void* arg);
+void transCloseServer(void* arg);
+
 #endif

source/libs/transport/inc/transportInt.h

@@ -64,6 +64,7 @@ typedef struct {
   void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
   int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
   bool (*pfp)(void* parent, tmsg_t msgType);
+  void* (*mfp)(void* parent, tmsg_t msgType);
 
   int32_t         refCount;
   void*           parent;

source/libs/transport/src/trans.c

@@ -18,8 +18,9 @@
 #include "transComm.h"
 
 void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) = {
-    taosInitServer, taosInitClient};
+    transInitServer, transInitClient};
-void (*taosCloseHandle[])(void* arg) = {taosCloseServer, taosCloseClient};
+
+void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient};
 
 void* rpcOpen(const SRpcInit* pInit) {
   SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo));
@@ -34,11 +35,12 @@ void* rpcOpen(const SRpcInit* pInit) {
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
   pRpc->pfp = pInit->pfp;
+  pRpc->mfp = pInit->mfp;
 
   if (pInit->connType == TAOS_CONN_SERVER) {
     pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
   } else {
-    pRpc->numOfThreads = pInit->numOfThreads;
+    pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
   }
 
   pRpc->connType = pInit->connType;
@@ -51,7 +53,6 @@ void* rpcOpen(const SRpcInit* pInit) {
   if (pInit->secret) {
     memcpy(pRpc->secret, pInit->secret, strlen(pInit->secret));
   }
-
   return pRpc;
 }
 void rpcClose(void* arg) {
@@ -111,16 +112,19 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) {
 int  rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; }
 void rpcCancelRequest(int64_t rid) { return; }
 
-int32_t rpcInit() {
+void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
-  // impl later
+  char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
-  return 0;
+  uint32_t port = pEpSet->eps[pEpSet->inUse].port;
+  transSendRequest(shandle, ip, port, pMsg);
+}
+void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
+  char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
+  uint32_t port = pEpSet->eps[pEpSet->inUse].port;
+  transSendRecv(shandle, ip, port, pMsg, pRsp);
 }
 
-void rpcCleanup(void) {
+void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); }
-  // impl later
+int  rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); }
-  //
-  return;
-}
 
 void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle};
 void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHandle};
@@ -129,9 +133,19 @@ void rpcRefHandle(void* handle, int8_t type) {
   assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
   (*taosRefHandle[type])(handle);
 }
+
 void rpcUnrefHandle(void* handle, int8_t type) {
   assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
   (*taosUnRefHandle[type])(handle);
 }
 
+int32_t rpcInit() {
+  // impl later
+  return 0;
+}
+void rpcCleanup(void) {
+  // impl later
+  return;
+}
+
 #endif

source/libs/transport/src/transCli.c

@@ -42,7 +42,7 @@ typedef struct SCliConn {
 
 typedef struct SCliMsg {
   STransConnCtx* ctx;
-  SRpcMsg        msg;
+  STransMsg      msg;
   queue          q;
   uint64_t       st;
 } SCliMsg;
@@ -84,7 +84,7 @@ static void      addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co
 // register timer in each thread to clear expire conn
 static void cliTimeoutCb(uv_timer_t* handle);
 // alloc buf for recv
-static void cliAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
+static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
 // callback after  read nbytes from socket
 static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
 // callback after write data to socket
@@ -105,9 +105,9 @@ static void cliHandleExcept(SCliConn* conn);
 static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
 static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd);
 static void cliSendQuit(SCliThrdObj* thrd);
-static void destroyUserdata(SRpcMsg* userdata);
+static void destroyUserdata(STransMsg* userdata);
 
-static int cliRBChoseIdx(SRpcInfo* pTransInst);
+static int cliRBChoseIdx(STrans* pTransInst);
 
 static void destroyCmsg(SCliMsg* cmsg);
 static void transDestroyConnCtx(STransConnCtx* ctx);
@@ -118,11 +118,11 @@ static void         destroyThrdObj(SCliThrdObj* pThrd);
 #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1)
 #define CONN_PERSIST_TIME(para) (para * 1000 * 10)
 
-#define CONN_GET_INST_LABEL(conn) (((SRpcInfo*)(((SCliThrdObj*)conn->hostThrd)->pTransInst))->label)
+#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)conn->hostThrd)->pTransInst))->label)
 #define CONN_HANDLE_THREAD_QUIT(conn, thrd) \
   do {                                      \
     if (thrd->quit) {                       \
-      cliHandleExcept(conn);             \
+      cliHandleExcept(conn);                \
       goto _RETURE;                         \
     }                                       \
   } while (0)
@@ -130,62 +130,76 @@ static void         destroyThrdObj(SCliThrdObj* pThrd);
 #define CONN_HANDLE_BROKEN(conn) \
   do {                           \
     if (conn->broken) {          \
-      cliHandleExcept(conn);  \
+      cliHandleExcept(conn);     \
       goto _RETURE;              \
     }                            \
   } while (0);
 
+#define CONN_SET_PERSIST_BY_APP(conn) \
+  do {                                \
+    if (conn->persist == false) {     \
+      conn->persist = true;           \
+      transRefCliHandle(conn);        \
+    }                                 \
+  } while (0)
+#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->persist == false)
+
 static void* cliWorkThread(void* arg);
 
-static void* cliNotifyApp() {}
+void cliHandleResp(SCliConn* conn) {
-static void  cliHandleResp(SCliConn* conn) {
-  SCliMsg*       pMsg = conn->data;
-  STransConnCtx* pCtx = pMsg->ctx;
-
   SCliThrdObj* pThrd = conn->hostThrd;
-  SRpcInfo*    pTransInst = pThrd->pTransInst;
+  STrans*      pTransInst = pThrd->pTransInst;
 
   STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
   pHead->code = htonl(pHead->code);
   pHead->msgLen = htonl(pHead->msgLen);
 
-  // buf's mem alread translated to rpcMsg.pCont
+  STransMsg transMsg = {0};
+  transMsg.contLen = transContLenFromMsg(pHead->msgLen);
+  transMsg.pCont = transContFromHead((char*)pHead);
+  transMsg.code = pHead->code;
+  transMsg.msgType = pHead->msgType;
+  transMsg.ahandle = NULL;
+
+  SCliMsg*       pMsg = conn->data;
+  STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL;
+  if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) {
+    transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL;
+  } else {
+    transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
+  }
+  // if (rpcMsg.ahandle == NULL) {
+  //  tDebug("%s cli conn %p handle except", CONN_GET_INST_LABEL(conn), conn);
+  //  return;
+  //}
+
+  // buf's mem alread translated to transMsg.pCont
   transClearBuffer(&conn->readBuf);
 
-  SRpcMsg rpcMsg = {0};
+  if (pTransInst->pfp != NULL && (*pTransInst->pfp)(pTransInst->parent, transMsg.msgType)) {
-  rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
+    transMsg.handle = conn;
-  rpcMsg.pCont = transContFromHead((char*)pHead);
+    CONN_SET_PERSIST_BY_APP(conn);
-  rpcMsg.code = pHead->code;
+    tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
-  rpcMsg.msgType = pHead->msgType;
-  rpcMsg.ahandle = pCtx->ahandle;
-
-  if (pTransInst->pfp != NULL && (pTransInst->pfp)(pTransInst->parent, rpcMsg.msgType)) {
-    rpcMsg.handle = conn;
-    transRefCliHandle(conn);
-
-    conn->persist = 1;
-    tDebug("cli conn %p persist by app", conn);
   }
 
   tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn,
          TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
-         inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), rpcMsg.contLen);
+         inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen);
 
   conn->secured = pHead->secured;
 
-  if (pCtx->pSem == NULL) {
+  if (pCtx == NULL || pCtx->pSem == NULL) {
     tTrace("%s cli conn %p handle resp", pTransInst->label, conn);
-    (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL);
+    (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
   } else {
     tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn);
-    memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg));
+    memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg));
     tsem_post(pCtx->pSem);
   }
 
-  uv_read_start((uv_stream_t*)conn->stream, cliAllocBufferCb, cliRecvCb);
+  uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb);
 
-  // user owns conn->persist = 1
+  if (CONN_NO_PERSIST_BY_APP(conn)) {
-  if (conn->persist == 0) {
     addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
   }
   destroyCmsg(conn->data);
@@ -196,29 +210,37 @@ static void  cliHandleResp(SCliConn* conn) {
     // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
   }
 }
-static void cliHandleExcept(SCliConn* pConn) {
+
+void cliHandleExcept(SCliConn* pConn) {
   if (pConn->data == NULL) {
-    // handle conn except in conn pool
+    if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) {
-    transUnrefCliHandle(pConn);
+      transUnrefCliHandle(pConn);
-    return;
+      return;
+    }
   }
   SCliThrdObj* pThrd = pConn->hostThrd;
-  SRpcInfo*    pTransInst = pThrd->pTransInst;
+  STrans*      pTransInst = pThrd->pTransInst;
 
   SCliMsg*       pMsg = pConn->data;
-  STransConnCtx* pCtx = pMsg->ctx;
+  STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL;
 
-  SRpcMsg rpcMsg = {0};
+  STransMsg transMsg = {0};
-  rpcMsg.ahandle = pCtx->ahandle;
+  transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-  rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+  transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0;
-  rpcMsg.msgType = pMsg->msg.msgType + 1;
+  transMsg.ahandle = NULL;
 
-  if (pCtx->pSem == NULL) {
+  if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) {
+    transMsg.ahandle = pTransInst->mfp ? (*pTransInst->mfp)(pTransInst->parent, transMsg.msgType) : NULL;
+  } else {
+    transMsg.ahandle = pCtx ? pCtx->ahandle : NULL;
+  }
+
+  if (pCtx == NULL || pCtx->pSem == NULL) {
     tTrace("%s cli conn %p handle resp", pTransInst->label, pConn);
-    (pTransInst->cfp)(pTransInst->parent, &rpcMsg, NULL);
+    (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
   } else {
     tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn);
-    memcpy((char*)(pCtx->pRsp), (char*)(&rpcMsg), sizeof(rpcMsg));
+    memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg));
     tsem_post(pCtx->pSem);
   }
   destroyCmsg(pConn->data);
@@ -228,11 +250,11 @@ static void cliHandleExcept(SCliConn* pConn) {
   transUnrefCliHandle(pConn);
 }
 
-static void cliTimeoutCb(uv_timer_t* handle) {
+void cliTimeoutCb(uv_timer_t* handle) {
   SCliThrdObj* pThrd = handle->data;
-  SRpcInfo*    pRpc = pThrd->pTransInst;
+  STrans*      pTransInst = pThrd->pTransInst;
   int64_t      currentTime = pThrd->nextTimeout;
-  tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pRpc->label);
+  tTrace("%s, cli conn timeout, try to remove expire conn from conn pool", pTransInst->label);
 
   SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
   while (p != NULL) {
@@ -249,14 +271,15 @@ static void cliTimeoutCb(uv_timer_t* handle) {
     p = taosHashIterate((SHashObj*)pThrd->pool, p);
   }
 
-  pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
+  pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
-  uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
+  uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0);
 }
-static void* createConnPool(int size) {
+
+void* createConnPool(int size) {
   // thread local, no lock
   return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
 }
-static void* destroyConnPool(void* pool) {
+void* destroyConnPool(void* pool) {
   SConnList* connList = taosHashIterate((SHashObj*)pool, NULL);
   while (connList != NULL) {
     while (!QUEUE_IS_EMPTY(&connList->conn)) {
@@ -301,15 +324,15 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
   tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
   tTrace("cli conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap);
 
-  SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
+  STrans* pTransInst = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
 
-  conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
+  conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
   SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
   // list already create before
   assert(plist != NULL);
   QUEUE_PUSH(&plist->conn, &conn->conn);
 }
-static void cliAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
+static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
   SCliConn*    conn = handle->data;
   SConnBuffer* pBuf = &conn->readBuf;
   transAllocBuffer(pBuf, buf);
@@ -358,6 +381,7 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
 
   QUEUE_INIT(&conn->conn);
   conn->hostThrd = pThrd;
+  conn->persist = false;
   conn->broken = false;
   transRefCliHandle(conn);
   return conn;
@@ -392,19 +416,19 @@ static void cliSendCb(uv_write_t* req, int status) {
     cliHandleExcept(pConn);
     return;
   }
-  uv_read_start((uv_stream_t*)pConn->stream, cliAllocBufferCb, cliRecvCb);
+  uv_read_start((uv_stream_t*)pConn->stream, cliAllocRecvBufferCb, cliRecvCb);
 }
 
-static void cliSend(SCliConn* pConn) {
+void cliSend(SCliConn* pConn) {
   CONN_HANDLE_BROKEN(pConn);
 
   SCliMsg*       pCliMsg = pConn->data;
   STransConnCtx* pCtx = pCliMsg->ctx;
 
   SCliThrdObj* pThrd = pConn->hostThrd;
-  SRpcInfo*    pTransInst = pThrd->pTransInst;
+  STrans*      pTransInst = pThrd->pTransInst;
 
-  SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg);
+  STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
 
   STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
   int            msgLen = transMsgLenFromCont(pMsg->contLen);
@@ -442,7 +466,8 @@ static void cliSend(SCliConn* pConn) {
 _RETURE:
   return;
 }
-static void cliConnCb(uv_connect_t* req, int status) {
+
+void cliConnCb(uv_connect_t* req, int status) {
   // impl later
   SCliConn* pConn = req->data;
   if (status != 0) {
@@ -472,11 +497,11 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
   pThrd->quit = true;
   uv_stop(pThrd->loop);
 }
-static SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+
+SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
   SCliConn* conn = NULL;
   if (pMsg->msg.handle != NULL) {
     conn = (SCliConn*)(pMsg->msg.handle);
-    transUnrefCliHandle(conn);
     if (conn != NULL) {
       tTrace("%s cli conn %p reused", CONN_GET_INST_LABEL(conn), conn);
     }
@@ -487,13 +512,14 @@ static SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
   }
   return conn;
 }
-static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
+
+void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
   uint64_t et = taosGetTimestampUs();
   uint64_t el = et - pMsg->st;
-  tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((SRpcInfo*)pThrd->pTransInst)->label, el);
+  tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el);
 
   STransConnCtx* pCtx = pMsg->ctx;
-  SRpcInfo*      pTransInst = pThrd->pTransInst;
+  STrans*        pTransInst = pThrd->pTransInst;
 
   SCliConn* conn = cliGetConn(pMsg, pThrd);
   if (conn != NULL) {
@@ -514,6 +540,7 @@ static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
     tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port);
     uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb);
   }
+
   conn->hThrdIdx = pCtx->hThrdIdx;
 }
 static void cliAsyncCb(uv_async_t* handle) {
@@ -522,7 +549,7 @@ static void cliAsyncCb(uv_async_t* handle) {
   SCliMsg*     pMsg = NULL;
 
   // batch process to avoid to lock/unlock frequently
-  queue        wq;
+  queue wq;
   pthread_mutex_lock(&item->mtx);
   QUEUE_MOVE(&item->qmsg, &wq);
   pthread_mutex_unlock(&item->mtx);
@@ -551,17 +578,17 @@ static void* cliWorkThread(void* arg) {
   uv_run(pThrd->loop, UV_RUN_DEFAULT);
 }
 
-void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
+void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
   SCliObj* cli = calloc(1, sizeof(SCliObj));
 
-  SRpcInfo* pRpc = shandle;
+  STrans* pTransInst = shandle;
   memcpy(cli->label, label, strlen(label));
   cli->numOfThreads = numOfThreads;
   cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*));
 
   for (int i = 0; i < cli->numOfThreads; i++) {
     SCliThrdObj* pThrd = createThrdObj();
-    pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
+    pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
     pThrd->pTransInst = shandle;
 
     int err = pthread_create(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd));
@@ -573,7 +600,7 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
   return cli;
 }
 
-static void destroyUserdata(SRpcMsg* userdata) {
+static void destroyUserdata(STransMsg* userdata) {
   if (userdata->pCont == NULL) {
     return;
   }
@@ -629,12 +656,20 @@ static void transDestroyConnCtx(STransConnCtx* ctx) {
   free(ctx);
 }
 //
-static void cliSendQuit(SCliThrdObj* thrd) {
+void cliSendQuit(SCliThrdObj* thrd) {
   // cli can stop gracefully
   SCliMsg* msg = calloc(1, sizeof(SCliMsg));
   transSendAsync(thrd->asyncPool, &msg->q);
 }
-void taosCloseClient(void* arg) {
+int cliRBChoseIdx(STrans* pTransInst) {
+  int64_t index = pTransInst->index;
+  if (pTransInst->index++ >= pTransInst->numOfThreads) {
+    pTransInst->index = 0;
+  }
+  return index % pTransInst->numOfThreads;
+}
+
+void transCloseClient(void* arg) {
   SCliObj* cli = arg;
   for (int i = 0; i < cli->numOfThreads; i++) {
     cliSendQuit(cli->pThreadObj[i]);
@@ -643,13 +678,6 @@ void taosCloseClient(void* arg) {
   free(cli->pThreadObj);
   free(cli);
 }
-static int cliRBChoseIdx(SRpcInfo* pTransInst) {
-  int64_t index = pTransInst->index;
-  if (pTransInst->index++ >= pTransInst->numOfThreads) {
-    pTransInst->index = 0;
-  }
-  return index % pTransInst->numOfThreads;
-}
 void transRefCliHandle(void* handle) {
   if (handle == NULL) {
     return;
@@ -665,17 +693,11 @@ void transUnrefCliHandle(void* handle) {
   if (ref == 0) {
     cliDestroyConn((SCliConn*)handle, true);
   }
-
-  // unref cli handle
 }
-void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
-  // impl later
-  char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
-  uint32_t port = pEpSet->eps[pEpSet->inUse].port;
 
-  SRpcInfo* pTransInst = (SRpcInfo*)shandle;
+void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* pMsg) {
-
+  STrans* pTransInst = (STrans*)shandle;
-  int index = CONN_HOST_THREAD_INDEX(pMsg->handle);
+  int     index = CONN_HOST_THREAD_INDEX((SCliConn*)pMsg->handle);
   if (index == -1) {
     index = cliRBChoseIdx(pTransInst);
   }
@@ -683,6 +705,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t*
   if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) {
     // imp later
   }
+  tDebug("send request at thread:%d %p", index, pMsg);
   STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
   pCtx->ahandle = pMsg->ahandle;
   pCtx->msgType = pMsg->msgType;
@@ -701,14 +724,9 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t*
   SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index];
   transSendAsync(thrd->asyncPool, &(cliMsg->q));
 }
-
+void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) {
-void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) {
+  STrans* pTransInst = (STrans*)shandle;
-  char*    ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn);
+  int     index = CONN_HOST_THREAD_INDEX(pReq->handle);
-  uint32_t port = pEpSet->eps[pEpSet->inUse].port;
-
-  SRpcInfo* pTransInst = (SRpcInfo*)shandle;
-
-  int index = CONN_HOST_THREAD_INDEX(pReq->handle);
   if (index == -1) {
     index = cliRBChoseIdx(pTransInst);
   }
@@ -734,7 +752,6 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) {
   tsem_wait(pSem);
   tsem_destroy(pSem);
   free(pSem);
-
-  return;
 }
+
 #endif

source/libs/transport/src/transComm.c

@@ -16,20 +16,6 @@
 
 #include "transComm.h"
 
-int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
-  T_MD5_CTX context;
-  int       ret = -1;
-
-  tMD5Init(&context);
-  tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
-  tMD5Update(&context, (uint8_t*)pMsg, msgLen);
-  tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
-  tMD5Final(&context);
-
-  if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0;
-
-  return ret;
-}
 int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
   T_MD5_CTX context;
   int       ret = -1;
@@ -44,17 +30,7 @@ int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
 
   return ret;
 }
-void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) {
-  T_MD5_CTX context;
 
-  tMD5Init(&context);
-  tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
-  tMD5Update(&context, (uint8_t*)pMsg, msgLen);
-  tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN);
-  tMD5Final(&context);
-
-  memcpy(pAuth, context.digest, sizeof(context.digest));
-}
 void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) {
   T_MD5_CTX context;
 
@@ -67,45 +43,6 @@ void transBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) {
   memcpy(pAuth, context.digest, sizeof(context.digest));
 }
 
-int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) {
-  SRpcHead* pHead = rpcHeadFromCont(pCont);
-  int32_t   finalLen = 0;
-  int       overhead = sizeof(SRpcComp);
-
-  if (!NEEDTO_COMPRESSS_MSG(contLen)) {
-    return contLen;
-  }
-
-  char* buf = malloc(contLen + overhead + 8);  // 8 extra bytes
-  if (buf == NULL) {
-    tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen);
-    return contLen;
-  }
-
-  int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead);
-  tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead);
-
-  /*
-   * only the compressed size is less than the value of contLen - overhead, the compression is applied
-   * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message
-   */
-  if (compLen > 0 && compLen < contLen - overhead) {
-    SRpcComp* pComp = (SRpcComp*)pCont;
-    pComp->reserved = 0;
-    pComp->contLen = htonl(contLen);
-    memcpy(pCont + overhead, buf, compLen);
-
-    pHead->comp = 1;
-    tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen);
-    finalLen = compLen + overhead;
-  } else {
-    finalLen = contLen;
-  }
-
-  free(buf);
-  return finalLen;
-}
-
 bool transCompressMsg(char* msg, int32_t len, int32_t* flen) {
   return false;
   // SRpcHead* pHead = rpcHeadFromCont(pCont);
@@ -154,39 +91,6 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) {
   return false;
 }
 
-SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) {
-  int       overhead = sizeof(SRpcComp);
-  SRpcHead* pNewHead = NULL;
-  uint8_t*  pCont = pHead->content;
-  SRpcComp* pComp = (SRpcComp*)pHead->content;
-
-  if (pHead->comp) {
-    // decompress the content
-    assert(pComp->reserved == 0);
-    int contLen = htonl(pComp->contLen);
-
-    // prepare the temporary buffer to decompress message
-    char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD);
-    pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext));  // reserve SRpcReqContext
-
-    if (pNewHead) {
-      int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead;
-      int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen);
-      assert(origLen == contLen);
-
-      memcpy(pNewHead, pHead, sizeof(SRpcHead));
-      pNewHead->msgLen = rpcMsgLenFromCont(origLen);
-      /// rpcFreeMsg(pHead);  // free the compressed message buffer
-      pHead = pNewHead;
-      tTrace("decomp malloc mem:%p", temp);
-    } else {
-      tError("failed to allocate memory to decompress msg, contLen:%d", contLen);
-    }
-  }
-
-  return pHead;
-}
-
 void transConnCtxDestroy(STransConnCtx* ctx) {
   free(ctx->ip);
   free(ctx);
@@ -315,7 +219,7 @@ int transSendAsync(SAsyncPool* pool, queue* q) {
   if (el > 50) {
     // tInfo("lock and unlock cost: %d", (int)el);
   }
-
   return uv_async_send(async);
 }
+
 #endif

source/libs/transport/src/transSrv.c

@@ -37,8 +37,7 @@ typedef struct SSrvConn {
 
   struct sockaddr_in addr;
   struct sockaddr_in locaddr;
-  // SRpcMsg sendMsg;
+
-  // del later
   char secured;
   int  spi;
   char info[64];
@@ -49,7 +48,7 @@ typedef struct SSrvConn {
 
 typedef struct SSrvMsg {
   SSrvConn* pConn;
-  SRpcMsg   msg;
+  STransMsg msg;
   queue     q;
 } SSrvMsg;
 
@@ -59,12 +58,13 @@ typedef struct SWorkThrdObj {
   uv_os_fd_t  fd;
   uv_loop_t*  loop;
   SAsyncPool* asyncPool;
-  // uv_async_t*     workerAsync;  //
+
   queue           msg;
-  queue           conn;
   pthread_mutex_t msgMtx;
-  void*           pTransInst;
+
-  bool            quit;
+  queue conn;
+  void* pTransInst;
+  bool  quit;
 } SWorkThrdObj;
 
 typedef struct SServerObj {
@@ -91,7 +91,7 @@ static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen);
 static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen);
 
 static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
-static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
+static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
 static void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
 static void uvOnTimeoutCb(uv_timer_t* handle);
 static void uvOnSendCb(uv_write_t* req, int status);
@@ -121,7 +121,7 @@ static void* acceptThread(void* arg);
 static bool addHandleToWorkloop(void* arg);
 static bool addHandleToAcceptloop(void* arg);
 
-void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
+void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
   SSrvConn*    conn = handle->data;
   SConnBuffer* pBuf = &conn->readBuf;
   transAllocBuffer(pBuf, buf);
@@ -163,7 +163,7 @@ static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) {
       tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta);
       code = TSDB_CODE_RPC_INVALID_TIME_STAMP;
     } else {
-      if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
+      if (transAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
         // tDebug("%s, authentication failed, msg discarded", pConn->info);
         code = TSDB_CODE_RPC_AUTH_FAILURE;
       } else {
@@ -204,37 +204,36 @@ static void uvHandleReq(SSrvConn* pConn) {
     memcpy(pConn->user, uMsg->user, tListLen(uMsg->user));
     memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret));
   }
-
-  pConn->inType = pHead->msgType;
-
-  SRpcInfo* pRpc = (SRpcInfo*)p->shandle;
   pHead->code = htonl(pHead->code);
 
   int32_t dlen = 0;
   if (transDecompressMsg(NULL, 0, NULL)) {
     // add compress later
-    // pHead = rpcDecompressRpcMsg(pHead);
+    // pHead = rpcDecompresSTransMsg(pHead);
   } else {
     pHead->msgLen = htonl(pHead->msgLen);
     // impl later
     //
   }
 
-  SRpcMsg rpcMsg;
+  STransMsg transMsg;
-  rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
+  transMsg.contLen = transContLenFromMsg(pHead->msgLen);
-  rpcMsg.pCont = pHead->content;
+  transMsg.pCont = pHead->content;
-  rpcMsg.msgType = pHead->msgType;
+  transMsg.msgType = pHead->msgType;
-  rpcMsg.code = pHead->code;
+  transMsg.code = pHead->code;
-  rpcMsg.ahandle = NULL;
+  transMsg.ahandle = NULL;
-  rpcMsg.handle = pConn;
+  transMsg.handle = pConn;
 
   transClearBuffer(&pConn->readBuf);
-
+  pConn->inType = pHead->msgType;
   transRefSrvHandle(pConn);
-  tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(rpcMsg.msgType),
+
+  tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType),
          inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
-         ntohs(pConn->locaddr.sin_port), rpcMsg.contLen);
+         ntohs(pConn->locaddr.sin_port), transMsg.contLen);
-  (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
+
+  STrans* pTransInst = (STrans*)p->shandle;
+  (*((STrans*)p->shandle)->cfp)(pTransInst->parent, &transMsg, NULL);
   // uv_timer_start(&pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
   // auth
   // validate msg type
@@ -260,7 +259,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
   }
 
   tError("server conn %p read error: %s", conn, uv_err_name(nread));
-  if (nread < 0 || nread == UV_EOF) {
+  if (nread < 0) {
     conn->broken = true;
     transUnrefSrvHandle(conn);
 
@@ -318,8 +317,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) {
   // impl later;
   tTrace("server conn %p prepare to send resp", smsg->pConn);
 
-  SSrvConn* pConn = smsg->pConn;
+  SSrvConn*  pConn = smsg->pConn;
-  SRpcMsg*  pMsg = &smsg->msg;
+  STransMsg* pMsg = &smsg->msg;
   if (pMsg->pCont == 0) {
     pMsg->pCont = (void*)rpcMallocCont(0);
     pMsg->contLen = 0;
@@ -526,7 +525,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
       return;
     }
 
-    uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnRecvCb);
+    uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb);
 
   } else {
     tDebug("failed to create new connection");
@@ -547,7 +546,6 @@ static bool addHandleToWorkloop(void* arg) {
     return false;
   }
 
-  // SRpcInfo* pRpc = pThrd->shandle;
   uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
   uv_pipe_open(pThrd->pipe, pThrd->fd);
 
@@ -668,7 +666,7 @@ static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) {
   return msgLen;
 }
 
-void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
+void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) {
   SServerObj* srv = calloc(1, sizeof(SServerObj));
   srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
   srv->numOfThreads = numOfThreads;
@@ -720,7 +718,7 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
 
   return srv;
 End:
-  taosCloseServer(srv);
+  transCloseServer(srv);
   return NULL;
 }
 
@@ -740,7 +738,7 @@ void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
   transSendAsync(pThrd->asyncPool, &srvMsg->q);
 }
 
-void taosCloseServer(void* arg) {
+void transCloseServer(void* arg) {
   // impl later
   SServerObj* srv = arg;
   for (int i = 0; i < srv->numOfThreads; i++) {
@@ -786,7 +784,7 @@ void transUnrefSrvHandle(void* handle) {
   }
   // unref srv handle
 }
-void rpcSendResponse(const SRpcMsg* pMsg) {
+void transSendResponse(const STransMsg* pMsg) {
   if (pMsg->handle == NULL) {
     return;
   }
@@ -799,14 +797,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) {
   tTrace("server conn %p start to send resp", pConn);
   transSendAsync(pThrd->asyncPool, &srvMsg->q);
 }
-
+int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) {
-int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) {
+  SSrvConn*          pConn = thandle;
-  SSrvConn* pConn = thandle;
-
   struct sockaddr_in addr = pConn->addr;
+
   pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);
   pInfo->clientPort = ntohs(addr.sin_port);
-
   tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
   return 0;
 }

source/libs/transport/test/transUT.cc

@@ -29,24 +29,25 @@ const char *ckey = "ckey";
 class Server;
 int port = 7000;
 // server process
+typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
 // client process;
 static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
 class Client {
  public:
   void Init(int nThread) {
-    memset(&rpcInit, 0, sizeof(rpcInit));
+    memset(&rpcInit_, 0, sizeof(rpcInit_));
-    rpcInit.localPort = 0;
+    rpcInit_.localPort = 0;
-    rpcInit.label = (char *)label;
+    rpcInit_.label = (char *)label;
-    rpcInit.numOfThreads = nThread;
+    rpcInit_.numOfThreads = nThread;
-    rpcInit.cfp = processResp;
+    rpcInit_.cfp = processResp;
-    rpcInit.user = (char *)user;
+    rpcInit_.user = (char *)user;
-    rpcInit.secret = (char *)secret;
+    rpcInit_.secret = (char *)secret;
-    rpcInit.ckey = (char *)ckey;
+    rpcInit_.ckey = (char *)ckey;
-    rpcInit.spi = 1;
+    rpcInit_.spi = 1;
-    rpcInit.parent = this;
+    rpcInit_.parent = this;
-    rpcInit.connType = TAOS_CONN_CLIENT;
+    rpcInit_.connType = TAOS_CONN_CLIENT;
-    this->transCli = rpcOpen(&rpcInit);
+    this->transCli = rpcOpen(&rpcInit_);
     tsem_init(&this->sem, 0, 0);
   }
   void SetResp(SRpcMsg *pMsg) {
@@ -55,9 +56,27 @@ class Client {
   }
   SRpcMsg *Resp() { return &this->resp; }
 
-  void Restart() {
+  void Restart(CB cb) {
     rpcClose(this->transCli);
-    this->transCli = rpcOpen(&rpcInit);
+    rpcInit_.cfp = cb;
+    this->transCli = rpcOpen(&rpcInit_);
+  }
+  void setPersistFP(bool (*pfp)(void *parent, tmsg_t msgType)) {
+    rpcClose(this->transCli);
+    rpcInit_.pfp = pfp;
+    this->transCli = rpcOpen(&rpcInit_);
+  }
+  void setConstructFP(void *(*mfp)(void *parent, tmsg_t msgType)) {
+    rpcClose(this->transCli);
+    rpcInit_.mfp = mfp;
+    this->transCli = rpcOpen(&rpcInit_);
+  }
+  void setPAndMFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
+    rpcClose(this->transCli);
+
+    rpcInit_.pfp = pfp;
+    rpcInit_.mfp = mfp;
+    this->transCli = rpcOpen(&rpcInit_);
   }
 
   void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) {
@@ -79,7 +98,7 @@ class Client {
 
  private:
   tsem_t   sem;
-  SRpcInit rpcInit;
+  SRpcInit rpcInit_;
   void *   transCli;
   SRpcMsg  resp;
 };
@@ -133,39 +152,56 @@ static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
   client->SetResp(pMsg);
   client->SemPost();
 }
+
+static void initEnv() {
+  dDebugFlag = 143;
+  vDebugFlag = 0;
+  mDebugFlag = 143;
+  cDebugFlag = 0;
+  jniDebugFlag = 0;
+  tmrDebugFlag = 143;
+  uDebugFlag = 143;
+  rpcDebugFlag = 143;
+  qDebugFlag = 0;
+  wDebugFlag = 0;
+  sDebugFlag = 0;
+  tsdbDebugFlag = 0;
+  tsLogEmbedded = 1;
+  tsAsyncLog = 0;
+
+  std::string path = "/tmp/transport";
+  taosRemoveDir(path.c_str());
+  taosMkDir(path.c_str());
+
+  tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
+  if (taosInitLog("taosdlog", 1) != 0) {
+    printf("failed to init log file\n");
+  }
+}
 class TransObj {
  public:
   TransObj() {
-    dDebugFlag = 143;
+    initEnv();
-    vDebugFlag = 0;
-    mDebugFlag = 143;
-    cDebugFlag = 0;
-    jniDebugFlag = 0;
-    tmrDebugFlag = 143;
-    uDebugFlag = 143;
-    rpcDebugFlag = 143;
-    qDebugFlag = 0;
-    wDebugFlag = 0;
-    sDebugFlag = 0;
-    tsdbDebugFlag = 0;
-    tsLogEmbedded = 1;
-    tsAsyncLog = 0;
-
-    std::string path = "/tmp/transport";
-    taosRemoveDir(path.c_str());
-    taosMkDir(path.c_str());
-
-    tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
-    if (taosInitLog("taosdlog", 1) != 0) {
-      printf("failed to init log file\n");
-    }
     cli = new Client;
     cli->Init(1);
     srv = new Server;
     srv->Start();
   }
-  void RestartCli() { cli->Restart(); }
+
+  void RestartCli(CB cb) { cli->Restart(cb); }
   void StopSrv() { srv->Stop(); }
+  void SetCliPersistFp(bool (*pfp)(void *parent, tmsg_t msgType)) {
+    // do nothing
+    cli->setPersistFP(pfp);
+  }
+  void SetCliMFp(void *(*mfp)(void *parent, tmsg_t msgType)) {
+    // do nothing
+    cli->setConstructFP(mfp);
+  }
+  void SetMAndPFp(bool (*pfp)(void *parent, tmsg_t msgType), void *(*mfp)(void *parent, tmsg_t msgType)) {
+    // do nothing
+    cli->setPAndMFp(pfp, mfp);
+  }
   void RestartSrv() { srv->Restart(); }
   void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); }
   ~TransObj() {
@@ -191,16 +227,16 @@ class TransEnv : public ::testing::Test {
   TransObj *tr = NULL;
 };
 
-// TEST_F(TransEnv, 01sendAndRec) {
+TEST_F(TransEnv, 01sendAndRec) {
-//  for (int i = 0; i < 1; i++) {
+  for (int i = 0; i < 1; i++) {
-//    SRpcMsg req = {0}, resp = {0};
+    SRpcMsg req = {0}, resp = {0};
-//    req.msgType = 0;
+    req.msgType = 0;
-//    req.pCont = rpcMallocCont(10);
+    req.pCont = rpcMallocCont(10);
-//    req.contLen = 10;
+    req.contLen = 10;
-//    tr->cliSendAndRecv(&req, &resp);
+    tr->cliSendAndRecv(&req, &resp);
-//    assert(resp.code == 0);
+    assert(resp.code == 0);
-//  }
+  }
-//}
+}
 
 TEST_F(TransEnv, 02StopServer) {
   for (int i = 0; i < 1; i++) {
@@ -218,6 +254,31 @@ TEST_F(TransEnv, 02StopServer) {
   tr->StopSrv();
   // tr->RestartSrv();
   tr->cliSendAndRecv(&req, &resp);
-
   assert(resp.code != 0);
 }
+TEST_F(TransEnv, clientUserDefined) {}
+
+TEST_F(TransEnv, cliPersistHandle) {
+  // impl late
+}
+TEST_F(TransEnv, srvPersistHandle) {
+  // impl later
+}
+
+TEST_F(TransEnv, srvPersisHandleExcept) {
+  // conn breken
+  //
+}
+TEST_F(TransEnv, cliPersisHandleExcept) {
+  // conn breken
+}
+
+TEST_F(TransEnv, multiCliPersisHandleExcept) {
+  // conn breken
+}
+TEST_F(TransEnv, multiSrvPersisHandleExcept) {
+  // conn breken
+}
+TEST_F(TransEnv, queryExcept) {
+  // query and conn is broken
+}

source/os/src/osLocale.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define ALLOW_FORBID_FUNC
 #define _DEFAULT_SOURCE
 #include "osLocale.h"
 

source/util/src/terror.c

@@ -349,6 +349,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            "No available disk")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               "TSDB messed message")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL,             "TSDB invalid tag value")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW,        "TSDB no cache last row data")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META,     "No sma index in meta")
 
 // query
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          "Invalid handle")

source/util/src/tuuid.c

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tuuid.h"
+
+static int64_t hashId = 0;
+static int32_t SerialNo = 0;
+
+int32_t tGenIdPI32(void) {
+  if (hashId == 0) {
+    char    uid[64];
+    int32_t code = taosGetSystemUUID(uid, tListLen(uid));
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    } else {
+      hashId = MurmurHash3_32(uid, strlen(uid));
+    }
+  }
+
+  int64_t  ts = taosGetTimestampMs();
+  uint64_t pid = taosGetPId();
+  int32_t  val = atomic_add_fetch_32(&SerialNo, 1);
+
+  int32_t id = ((hashId & 0x1F) << 26) | ((pid & 0x3F) << 20) | ((ts & 0xFFF) << 8) | (val & 0xFF);
+  return id;
+}
+
+int64_t tGenIdPI64(void) {
+  if (hashId == 0) {
+    char    uid[64];
+    int32_t code = taosGetSystemUUID(uid, tListLen(uid));
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    } else {
+      hashId = MurmurHash3_32(uid, strlen(uid));
+    }
+  }
+
+  int64_t  ts = taosGetTimestampMs();
+  uint64_t pid = taosGetPId();
+  int32_t  val = atomic_add_fetch_32(&SerialNo, 1);
+
+  int64_t id = ((hashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
+  return id;
+}

source/util/src/tworker.c

@@ -188,7 +188,7 @@ void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueu
 
 int32_t tWWorkerInit(SWWorkerPool *pool) {
   pool->nextId = 0;
-  pool->workers = calloc(sizeof(SWWorker), pool->max);
+  pool->workers = calloc(pool->max, sizeof(SWWorker));
   if (pool->workers == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;

tests/script/jenkins/basic.txt

@@ -16,4 +16,6 @@
 # ---- dnode
 ./test.sh -f tsim/dnode/basic1.sim
 
+# ---- insert
+./test.sh -f tsim/insert/basic0.sim
 #======================b1-end===============

tests/script/tsim/db/basic6.sim

@@ -14,13 +14,19 @@ $st = $stPrefix . $i
 $tb = $tbPrefix . $i
 
 print =============== step1
-sql create database $db replica 1 days 20 keep 2000 cache 16 vgroups 4
+# quorum  presicion 
+sql create database $db vgroups 8 replica 1 days 20 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
 sql show databases
-print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1 $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+if $rows != 1 then 
+  return -1
+endi
 if $data00 != $db then 
   return -1
 endi
-if $data02 != 4 then 
+if $data02 != 8 then 
   return -1
 endi
 if $data03 != 0 then 
@@ -32,16 +38,23 @@ endi
 if $data06 != 20 then 
   return -1
 endi
-if $data08 != 16 then 
+if $data07 != 3650,3650,3650 then 
+  return -1
+endi
+if $data08 != 32 then 
+  return -1
+endi
+if $data09 != 12 then 
   return -1
 endi
  
 print =============== step2
-sql create database $db
+sql_error create database $db
+sql create database if not exists $db
 sql show databases
 if $rows != 1 then 
   return -1
-endi 
+endi
 
 print =============== step3
 sql drop database $db

tests/script/tsim/insert/basic0.sim

@@ -0,0 +1,334 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+print =============== create database
+sql create database d0
+sql show databases
+if $rows != 1 then 
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+sql use d0
+
+print =============== create super table, include column type for count/sum/min/max/first
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags(1000)
+sql create table ct2 using stb tags(2000)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+
+
+print =============== insert data, mode1: one row one table in sql
+print =============== insert data, mode1: mulit rows one table in sql
+print =============== insert data, mode1: one rows mulit table in sql
+print =============== insert data, mode1: mulit rows mulit table in sql
+sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3)
+sql insert into ct2 values(now+0s, 10, 2.0, 3.0)
+sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3)
+# after fix bug, modify sql_error to sql
+sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4)
+sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6)
+
+#===================================================================
+#===================================================================
+print =============== query data from child table
+sql select * from ct1
+if $rows != 4 then         # after fix bug, modify 4 to 7
+  return -1
+endi
+if $data01 != 10 then 
+  return -1
+endi
+if $data02 != 2.00000 then 
+  return -1
+endi
+if $data03 != 3.000000000 then 
+  return -1
+endi
+#if $data41 != -14 then 
+#  return -1
+#endi
+#if $data42 != -2.40000 then 
+#  return -1
+#endi
+#if $data43 != -3.400000000 then 
+#  return -1
+#endi
+
+
+print =============== select count(*) from child table
+sql select count(*) from ct1
+if $rows != 1 then 
+  return -1
+endi
+
+print $data00 $data01 $data02
+if $data00 != 4 then 
+  return -1
+endi
+
+print =============== select count(column) from child table
+sql select count(ts), count(c1), count(c2), count(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $data00 != 4 then 
+  return -1
+endi
+if $data01 != 4 then 
+  return -1
+endi
+if $data02 != 4 then 
+  return -1
+endi
+if $data03 != 4 then 
+  return -1
+endi
+
+#print =============== select first(*)/first(column) from child table
+#sql select first(*) from ct1
+#sql select first(ts), first(c1), first(c2), first(c3) from ct1
+
+print =============== select min(column) from child table
+sql select min(c1), min(c2), min(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 10 then 
+  return -1
+endi
+if $data01 != 2.00000 then 
+  return -1
+endi
+if $data02 != 3.000000000 then 
+  return -1
+endi
+
+print =============== select max(column) from child table
+sql select max(c1), max(c2), max(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 13 then 
+  return -1
+endi
+if $data01 != 2.30000 then 
+  return -1
+endi
+if $data02 != 3.300000000 then 
+  return -1
+endi
+
+print =============== select sum(column) from child table
+sql select sum(c1), sum(c2), sum(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 46 then 
+  return -1
+endi
+if $data01 != 8.599999905 then 
+  return -1
+endi
+if $data02 != 12.600000000 then 
+  return -1
+endi
+
+print =============== select column, from child table
+sql select c1, c2, c3 from ct1
+print $data00 $data01 $data02
+#if $rows != 4 then 
+#  return -1
+#endi
+#if $data00 != 10 then 
+#  return -1
+#endi
+#if $data01 != 2.00000 then 
+#  return -1
+#endi
+#if $data02 != 3.000000000 then 
+#  return -1
+#endi
+#if $data10 != 11 then 
+#  return -1
+#endi
+#if $data11 != 2.10000 then 
+#  return -1
+#endi
+#if $data12 != 3.100000000 then 
+#  return -1
+#endi
+#if $data30 != 13 then 
+#  return -1
+#endi
+#if $data31 != 2.30000 then 
+#  return -1
+#endi
+#if $data32 != 3.300000000 then 
+#  return -1
+#endi
+#===================================================================
+#===================================================================
+
+#print =============== query data from stb
+#sql select * from stb
+#if $rows != 4 then 
+#  return -1
+#endi
+#print =============== select count(*) from supter table
+#sql select count(*) from stb
+#if $rows != 1 then 
+#  return -1
+#endi
+#
+#print $data00 $data01 $data02
+#if $data00 != 8 then 
+#  return -1
+#endi
+#
+#print =============== select count(column) from supter table
+#sql select count(ts), count(c1), count(c2), count(c3) from stb
+#print $data00 $data01 $data02  $data03
+#if $data00 != 8 then 
+#  return -1
+#endi
+#if $data01 != 8 then 
+#  return -1
+#endi
+#if $data02 != 8 then 
+#  return -1
+#endi
+#if $data03 != 8 then 
+#  return -1
+#endi
+
+
+#===================================================================
+#===================================================================
+
+print =============== stop and restart taosd, then again do query above
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+sleep 2000
+sql select * from ct1
+if $rows != 4 then         # after fix bug, modify 4 to 7
+  return -1
+endi
+if $data01 != 10 then 
+  return -1
+endi
+if $data02 != 2.00000 then 
+  return -1
+endi
+if $data03 != 3.000000000 then 
+  return -1
+endi
+#if $data41 != -14 then 
+#  return -1
+#endi
+#if $data42 != -2.40000 then 
+#  return -1
+#endi
+#if $data43 != -3.400000000 then 
+#  return -1
+#endi
+
+
+print =============== select count(*) from child table
+sql select count(*) from ct1
+if $rows != 1 then 
+  return -1
+endi
+
+print $data00 $data01 $data02
+if $data00 != 4 then 
+  return -1
+endi
+
+print =============== select count(column) from child table
+sql select count(ts), count(c1), count(c2), count(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $data00 != 4 then 
+  return -1
+endi
+if $data01 != 4 then 
+  return -1
+endi
+if $data02 != 4 then 
+  return -1
+endi
+if $data03 != 4 then 
+  return -1
+endi
+
+#print =============== select first(*)/first(column) from child table
+#sql select first(*) from ct1
+#sql select first(ts), first(c1), first(c2), first(c3) from ct1
+
+print =============== select min(column) from child table
+sql select min(c1), min(c2), min(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 10 then 
+  return -1
+endi
+if $data01 != 2.00000 then 
+  return -1
+endi
+if $data02 != 3.000000000 then 
+  return -1
+endi
+
+print =============== select max(column) from child table
+sql select max(c1), max(c2), max(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 13 then 
+  return -1
+endi
+if $data01 != 2.30000 then 
+  return -1
+endi
+if $data02 != 3.300000000 then 
+  return -1
+endi
+
+print =============== select sum(column) from child table
+sql select sum(c1), sum(c2), sum(c3) from ct1
+print $data00 $data01 $data02  $data03
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 46 then 
+  return -1
+endi
+if $data01 != 8.599999905 then 
+  return -1
+endi
+if $data02 != 12.600000000 then 
+  return -1
+endi
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/table/basic1.sim

@@ -61,10 +61,50 @@ if $rows != 7 then
 endi
 
 print $data00 $data01 $data02
-print $data10 $data11 $data22
+print $data10 $data11 $data12
-print $data20 $data11 $data22
+print $data20 $data21 $data22
+
+print =============== create normal table
+sql create database ndb
+sql use ndb
+sql create table nt0 (ts timestamp, i int)
+sql create table if not exists nt0 (ts timestamp, i int)
+sql create table nt1 (ts timestamp, i int)
+sql create table if not exists nt1 (ts timestamp, i int)
+sql create table if not exists nt3 (ts timestamp, i int)
+
+sql show tables
+if $rows != 3 then 
+  return -1
+endi
+
+sql insert into nt0 values(now+1s, 1)(now+2s, 2)(now+3s, 3)
+sql insert into nt1 values(now+1s, 1)(now+2s, 2)(now+3s, 3)
+
+sql select * from nt1
+if $rows != 3 then 
+  return -1
+endi
+
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+
+if $data01 != 1 then 
+  return -1
+endi
+
+if $data11 != 2 then 
+  return -1
+endi
+
+if $data21 != 3 then 
+  return -1
+endi
+
 
 print =============== insert data
+sql use d1
 sql insert into c1 values(now+1s, 1)
 sql insert into c1 values(now+2s, 2)
 sql insert into c1 values(now+3s, 3)
@@ -95,7 +135,7 @@ endi
 
 print $data00 $data01
 print $data10 $data11
-print $data20 $data11
+print $data20 $data21
 
 if $data01 != 1 then 
   return -1
@@ -160,7 +200,7 @@ endi
 
 print $data00 $data01
 print $data10 $data11
-print $data20 $data11
+print $data20 $data21
 
 if $data01 != 1 then 
   return -1
@@ -210,4 +250,27 @@ if $rows != 21 then
   return -1
 endi
 
+print =============== query data from normal table after restart dnode
+sql use ndb
+sql select * from nt1
+if $rows != 3 then 
+  return -1
+endi
+
+print $data00 $data01
+print $data10 $data11
+print $data20 $data21
+
+if $data01 != 1 then 
+  return -1
+endi
+
+if $data11 != 2 then 
+  return -1
+endi
+
+if $data21 != 3 then 
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT