From 264c30e5db9b348d06135bc9638d7e1cd9b66069 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 10:52:18 +0800
Subject: [PATCH 01/29] sync refactor

---
 source/libs/sync/inc/syncInt.h  |  9 ++++---
 source/libs/sync/inc/syncUtil.h |  1 +
 source/libs/sync/src/syncMain.c | 45 ++++++++++++++++++++++++++++++---
 source/libs/sync/src/syncUtil.c |  4 ++-
 4 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index b8c7eb60e7..79412febd9 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -99,8 +99,8 @@ typedef struct SRaftStore SRaftStore;
 struct SVotesGranted;
 typedef struct SVotesGranted SVotesGranted;
 
-struct SVotesResponded;
-typedef struct SVotesResponded SVotesResponded;
+struct SVotesRespond;
+typedef struct SVotesRespond SVotesRespond;
 
 typedef struct SRaftId {
   SyncNodeId  addr;  // typedef uint64_t SyncNodeId;
@@ -112,6 +112,7 @@ typedef struct SSyncNode {
   SyncGroupId vgId;
   SSyncCfg    syncCfg;
   char        path[TSDB_FILENAME_LEN];
+  char        walPath[TSDB_FILENAME_LEN];
   void*       rpcClient;
   int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
   void* queue;
@@ -142,8 +143,8 @@ typedef struct SSyncNode {
   SRaftStore* pRaftStore;
 
   // tla+ candidate vars
-  SVotesGranted*   pVotesGranted;
-  SVotesResponded* pVotesResponded;
+  SVotesGranted* pVotesGranted;
+  SVotesRespond* pVotesRespond;
 
   // tla+ leader vars
   SHashObj* pNextIndex;
diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h
index 9b481e82d9..eb1e888254 100644
--- a/source/libs/sync/inc/syncUtil.h
+++ b/source/libs/sync/inc/syncUtil.h
@@ -44,6 +44,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest);
 // ---- misc ----
 int32_t syncUtilRand(int32_t max);
 int32_t syncUtilElectRandomMS();
+int32_t syncUtilQuorum(int32_t replicaNum);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index db34d16690..836f1b0e83 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -20,10 +20,12 @@
 #include "syncEnv.h"
 #include "syncInt.h"
 #include "syncRaft.h"
+#include "syncRaftStore.h"
 #include "syncRequestVote.h"
 #include "syncRequestVoteReply.h"
 #include "syncTimeout.h"
 #include "syncUtil.h"
+#include "syncVoteMgr.h"
 
 static int32_t tsNodeRefId = -1;
 
@@ -35,6 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId);
 static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
 static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
 
+static void UpdateTerm(SyncTerm term);
 static void syncNodeBecomeFollower(SSyncNode* pSyncNode);
 static void syncNodeBecomeLeader(SSyncNode* pSyncNode);
 static void syncNodeFollower2Candidate(SSyncNode* pSyncNode);
@@ -71,19 +74,22 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   assert(pSyncNode != NULL);
   memset(pSyncNode, 0, sizeof(SSyncNode));
 
+  // init by SSyncInfo
   pSyncNode->vgId = pSyncInfo->vgId;
   pSyncNode->syncCfg = pSyncInfo->syncCfg;
   memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
-  pSyncNode->pFsm = pSyncInfo->pFsm;
-
+  memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath));
   pSyncNode->rpcClient = pSyncInfo->rpcClient;
   pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg;
   pSyncNode->queue = pSyncInfo->queue;
   pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg;
 
+  // init internal
   pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
-  pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
+  syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncInfo->vgId, &pSyncNode->raftId);
 
+  // init peersNum, peers, peersId
+  pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
   int j = 0;
   for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
     if (i != pSyncInfo->syncCfg.myIndex) {
@@ -91,9 +97,37 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
       j++;
     }
   }
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncInfo->vgId, &pSyncNode->peersId[i]);
+  }
 
+  // init replicaNum, replicasId
+  pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum;
+  for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
+    syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]);
+  }
+
+  // raft algorithm
+  pSyncNode->pFsm = pSyncInfo->pFsm;
+  pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum);
+  pSyncNode->leaderCache = EMPTY_RAFT_ID;
+
+  // life cycle
+
+  // init server vars
   pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
-  syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId);
+  pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath);
+  assert(pSyncNode->pRaftStore != NULL);
+
+  // init candidate vars
+  pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode);
+  assert(pSyncNode->pVotesGranted != NULL);
+  pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode);
+  assert(pSyncNode->pVotesRespond != NULL);
+
+  // init leader vars
+  pSyncNode->pNextIndex = NULL;
+  pSyncNode->pMatchIndex = NULL;
 
   // init ping timer
   pSyncNode->pPingTimer = NULL;
@@ -119,6 +153,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer;
   pSyncNode->heartbeatTimerCounter = 0;
 
+  // init callback
   pSyncNode->FpOnPing = syncNodeOnPingCb;
   pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb;
   pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
@@ -353,6 +388,8 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
 
 static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {}
 
+static void UpdateTerm(SyncTerm term) {}
+
 static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
     pSyncNode->leaderCache = EMPTY_RAFT_ID;
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index c70e490025..1e62301814 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -97,4 +97,6 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) {
 
 int32_t syncUtilRand(int32_t max) { return rand() % max; }
 
-int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
\ No newline at end of file
+int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
+
+int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
\ No newline at end of file

From 4540739c45866641c3fd1a7a0f9f1db809ca9b72 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Mar 2022 11:34:38 +0800
Subject: [PATCH 02/29] minor changes

---
 source/libs/monitor/CMakeLists.txt   |  1 -
 source/libs/monitor/inc/monInt.h     |  1 -
 source/libs/transport/CMakeLists.txt |  1 +
 source/libs/transport/inc/thttp.h    | 33 ++++++++++++++++++++++++++++
 source/util/CMakeLists.txt           |  2 +-
 5 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 source/libs/transport/inc/thttp.h

diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt
index 050372fab3..58f1c08039 100644
--- a/source/libs/monitor/CMakeLists.txt
+++ b/source/libs/monitor/CMakeLists.txt
@@ -3,7 +3,6 @@ add_library(monitor STATIC ${MONITOR_SRC})
 target_include_directories(
   monitor
   PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/monitor"
-  PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport"
   PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h
index c3b6569555..6ef901410b 100644
--- a/source/libs/monitor/inc/monInt.h
+++ b/source/libs/monitor/inc/monInt.h
@@ -19,7 +19,6 @@
 #include "monitor.h"
 
 #include "tarray.h"
-#include "tlockfree.h"
 #include "tjson.h"
 
 typedef struct {
diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt
index 465646ac95..5cc436cf32 100644
--- a/source/libs/transport/CMakeLists.txt
+++ b/source/libs/transport/CMakeLists.txt
@@ -12,6 +12,7 @@ target_link_libraries(
     PUBLIC os
     PUBLIC util
     PUBLIC common
+    PUBLIC zlib
 )
 if (${BUILD_WITH_UV_TRANS}) 
 if (${BUILD_WITH_UV})
diff --git a/source/libs/transport/inc/thttp.h b/source/libs/transport/inc/thttp.h
new file mode 100644
index 0000000000..a9e3953f57
--- /dev/null
+++ b/source/libs/transport/inc/thttp.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TRANSPORT_HTTP_H_
+#define _TD_TRANSPORT_HTTP_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
+
+int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_UTIL_H_*/
diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt
index 4b28800c28..7a47639e75 100644
--- a/source/util/CMakeLists.txt
+++ b/source/util/CMakeLists.txt
@@ -10,7 +10,7 @@ target_link_libraries(
     util
     PRIVATE os
     PUBLIC lz4_static
-    PUBLIC api cjson zlib
+    PUBLIC api cjson
 )
 
 if(${BUILD_TEST})

From 18a22e6517d6d032ea8361583584a777ed8b587e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Tue, 8 Mar 2022 13:41:03 +0800
Subject: [PATCH 03/29] minor changes

---
 source/libs/transport/inc/thttp.h | 33 -------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100644 source/libs/transport/inc/thttp.h

diff --git a/source/libs/transport/inc/thttp.h b/source/libs/transport/inc/thttp.h
deleted file mode 100644
index a9e3953f57..0000000000
--- a/source/libs/transport/inc/thttp.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TRANSPORT_HTTP_H_
-#define _TD_TRANSPORT_HTTP_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
-
-int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_UTIL_H_*/

From c9c48e870d618573bd0d096727b2f4199187a43f Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 13:43:54 +0800
Subject: [PATCH 04/29] sync refactor

---
 include/libs/sync/sync.h               |  7 +--
 source/libs/sync/inc/syncInt.h         |  2 +
 source/libs/sync/inc/syncUtil.h        |  9 ++--
 source/libs/sync/src/syncMain.c        | 65 ++++++++++++++++++++++
 source/libs/sync/src/syncUtil.c        | 44 ++++++++++++++-
 source/libs/sync/test/CMakeLists.txt   | 14 +++++
 source/libs/sync/test/syncInitTest.cpp | 74 ++++++++++++++++++++++++++
 7 files changed, 208 insertions(+), 7 deletions(-)
 create mode 100644 source/libs/sync/test/syncInitTest.cpp

diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index fddf18c571..ccbeb00bfd 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -31,9 +31,9 @@ typedef int64_t  SyncIndex;
 typedef uint64_t SyncTerm;
 
 typedef enum {
-  TAOS_SYNC_STATE_FOLLOWER = 0,
-  TAOS_SYNC_STATE_CANDIDATE = 1,
-  TAOS_SYNC_STATE_LEADER = 2,
+  TAOS_SYNC_STATE_FOLLOWER = 100,
+  TAOS_SYNC_STATE_CANDIDATE = 101,
+  TAOS_SYNC_STATE_LEADER = 102,
 } ESyncState;
 
 typedef struct SSyncBuffer {
@@ -134,6 +134,7 @@ typedef struct SSyncInfo {
   SyncGroupId vgId;
   SSyncCfg    syncCfg;
   char        path[TSDB_FILENAME_LEN];
+  char        walPath[TSDB_FILENAME_LEN];
   SSyncFSM*   pFsm;
 
   void* rpcClient;
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 79412febd9..fdacb87481 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -23,6 +23,7 @@ extern "C" {
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include "cJSON.h"
 #include "sync.h"
 #include "taosdef.h"
 #include "tglobal.h"
@@ -189,6 +190,7 @@ typedef struct SSyncNode {
 
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
 void       syncNodeClose(SSyncNode* pSyncNode);
+cJSON*     syncNode2Json(const SSyncNode* pSyncNode);
 
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
 int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h
index eb1e888254..6cb1105b2c 100644
--- a/source/libs/sync/inc/syncUtil.h
+++ b/source/libs/sync/inc/syncUtil.h
@@ -42,9 +42,12 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest);
 void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest);
 
 // ---- misc ----
-int32_t syncUtilRand(int32_t max);
-int32_t syncUtilElectRandomMS();
-int32_t syncUtilQuorum(int32_t replicaNum);
+int32_t     syncUtilRand(int32_t max);
+int32_t     syncUtilElectRandomMS();
+int32_t     syncUtilQuorum(int32_t replicaNum);
+cJSON*      syncUtilNodeInfo2Json(const SNodeInfo* p);
+cJSON*      syncUtilRaftId2Json(const SRaftId* p);
+const char* syncUtilState2String(ESyncState state);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 836f1b0e83..aebbd4d337 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -170,6 +170,71 @@ void syncNodeClose(SSyncNode* pSyncNode) {
   free(pSyncNode);
 }
 
+cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  // init by SSyncInfo
+  cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId);
+  cJSON_AddStringToObject(pRoot, "path", pSyncNode->path);
+  cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath);
+
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient);
+  cJSON_AddStringToObject(pRoot, "rpcClient", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg);
+  cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf);
+
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue);
+  cJSON_AddStringToObject(pRoot, "queue", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg);
+  cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf);
+
+  // init internal
+  cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->me);
+  cJSON_AddItemToObject(pRoot, "me", pMe);
+  cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->raftId);
+  cJSON_AddItemToObject(pRoot, "raftId", pRaftId);
+
+  cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum);
+  cJSON* pPeers = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "peers", pPeers);
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peers[i]));
+  }
+  cJSON* pPeersId = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "peersId", pPeersId);
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i]));
+  }
+
+  cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum);
+  cJSON* pReplicasId = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId);
+  for (int i = 0; i < pSyncNode->replicaNum; ++i) {
+    cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i]));
+  }
+
+  // raft algorithm
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm);
+  cJSON_AddStringToObject(pRoot, "pFsm", u64buf);
+  cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum);
+  cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache);
+  cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache);
+
+  // tla+ server vars
+  cJSON_AddStringToObject(pRoot, "state", syncUtilState2String(pSyncNode->state));
+
+  // tla+ candidate vars
+
+  // tla+ leader vars
+
+  // tla+ log vars
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SSyncNode", pRoot);
+  return pJson;
+}
+
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;
   syncUtilraftId2EpSet(destRaftId, &epSet);
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index 1e62301814..04be3f33ac 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -99,4 +99,46 @@ int32_t syncUtilRand(int32_t max) { return rand() % max; }
 
 int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
 
-int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
\ No newline at end of file
+int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
+
+cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn);
+  cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort);
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot);
+  return pJson;
+}
+
+cJSON* syncUtilRaftId2Json(const SRaftId* p) {
+  char   u64buf[128];
+  cJSON* pRoot = cJSON_CreateObject();
+
+  snprintf(u64buf, sizeof(u64buf), "%lu", p->addr);
+  cJSON_AddStringToObject(pRoot, "addr", u64buf);
+  char     host[128];
+  uint16_t port;
+  syncUtilU642Addr(p->addr, host, sizeof(host), &port);
+  cJSON_AddStringToObject(pRoot, "host", host);
+  cJSON_AddNumberToObject(pRoot, "port", port);
+  cJSON_AddNumberToObject(pRoot, "vgId", p->vgId);
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot);
+  return pJson;
+}
+
+const char* syncUtilState2String(ESyncState state) {
+  if (state == TAOS_SYNC_STATE_FOLLOWER) {
+    return "TAOS_SYNC_STATE_FOLLOWER";
+  } else if (state == TAOS_SYNC_STATE_CANDIDATE) {
+    return "TAOS_SYNC_STATE_CANDIDATE";
+  } else if (state == TAOS_SYNC_STATE_LEADER) {
+    return "TAOS_SYNC_STATE_LEADER";
+  } else {
+    return "TAOS_SYNC_STATE_UNKNOWN";
+  }
+}
\ No newline at end of file
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 770d1d1bd8..3a8eea53d4 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -10,6 +10,7 @@ add_executable(syncIOSendMsgServerTest "")
 add_executable(syncRaftStoreTest "")
 add_executable(syncEnqTest "")
 add_executable(syncIndexTest "")
+add_executable(syncInitTest "")
 
 
 target_sources(syncTest
@@ -60,6 +61,10 @@ target_sources(syncIndexTest
     PRIVATE
     "syncIndexTest.cpp"
 )
+target_sources(syncInitTest
+    PRIVATE
+    "syncInitTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -122,6 +127,11 @@ target_include_directories(syncIndexTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncInitTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -172,6 +182,10 @@ target_link_libraries(syncIndexTest
     sync
     gtest_main
 )
+target_link_libraries(syncInitTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp
new file mode 100644
index 0000000000..602297b2a6
--- /dev/null
+++ b/source/libs/sync/test/syncInitTest.cpp
@@ -0,0 +1,74 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t ports[3] = {7010, 7110, 7210};
+
+SSyncNode* syncInitTest() {
+  SSyncFSM* pFsm;
+
+  SSyncInfo syncInfo;
+  syncInfo.vgId = 1;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = pFsm;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
+  snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+  pCfg->myIndex = 0;
+  pCfg->replicaNum = 3;
+
+  pCfg->nodeInfo[0].nodePort = ports[0];
+  snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  pCfg->nodeInfo[1].nodePort = ports[1];
+  snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
+
+  pCfg->nodeInfo[2].nodePort = ports[2];
+  snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+
+  SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
+  assert(pSyncNode != NULL);
+
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->pSyncNode = pSyncNode;
+
+  return pSyncNode;
+}
+
+int main() {
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
+  tsAsyncLog = 0;
+  sDebugFlag = 143 + 64;
+
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[0]);
+  assert(ret == 0);
+
+  SSyncNode* pSyncNode = syncInitTest();
+  assert(pSyncNode != NULL);
+
+  cJSON* pJson = syncNode2Json(pSyncNode);
+  char*  serialized = cJSON_Print(pJson);
+  printf("%s\n", serialized);
+  free(serialized);
+  cJSON_Delete(pJson);
+
+  return 0;
+}

From f556d98159308abf0a15b4b522dae7a3b4fdd712 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 13:55:13 +0800
Subject: [PATCH 05/29] sync refactor

---
 source/libs/sync/inc/syncInt.h        |  6 ++---
 source/libs/sync/src/syncMain.c       | 35 ++++++++++++++-------------
 source/libs/sync/test/syncEnqTest.cpp |  2 +-
 3 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index fdacb87481..2c8e36fd08 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -120,11 +120,11 @@ typedef struct SSyncNode {
   int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
 
   // init internal
-  SNodeInfo me;
-  SRaftId   raftId;
+  SNodeInfo myNodeInfo;
+  SRaftId   myRaftId;
 
   int32_t   peersNum;
-  SNodeInfo peers[TSDB_MAX_REPLICA];
+  SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA];
   SRaftId   peersId[TSDB_MAX_REPLICA];
 
   int32_t replicaNum;
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index aebbd4d337..44d2e4c160 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -85,20 +85,20 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
   pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg;
 
   // init internal
-  pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
-  syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncInfo->vgId, &pSyncNode->raftId);
+  pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
+  syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId);
 
   // init peersNum, peers, peersId
   pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
   int j = 0;
   for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
     if (i != pSyncInfo->syncCfg.myIndex) {
-      pSyncNode->peers[j] = pSyncInfo->syncCfg.nodeInfo[i];
+      pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i];
       j++;
     }
   }
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncInfo->vgId, &pSyncNode->peersId[i]);
+    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]);
   }
 
   // init replicaNum, replicasId
@@ -190,16 +190,16 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf);
 
   // init internal
-  cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->me);
-  cJSON_AddItemToObject(pRoot, "me", pMe);
-  cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->raftId);
-  cJSON_AddItemToObject(pRoot, "raftId", pRaftId);
+  cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo);
+  cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe);
+  cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId);
+  cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId);
 
   cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum);
   cJSON* pPeers = cJSON_CreateArray();
-  cJSON_AddItemToObject(pRoot, "peers", pPeers);
+  cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers);
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peers[i]));
+    cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i]));
   }
   cJSON* pPeersId = cJSON_CreateArray();
   cJSON_AddItemToObject(pRoot, "peersId", pPeersId);
@@ -222,7 +222,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache);
 
   // tla+ server vars
-  cJSON_AddStringToObject(pRoot, "state", syncUtilState2String(pSyncNode->state));
+  cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state);
+  cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state));
 
   // tla+ candidate vars
 
@@ -283,7 +284,7 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) {
   for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) {
     SRaftId destId;
     syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
     ret = syncNodePing(pSyncNode, &destId, pMsg);
     assert(ret == 0);
     syncPingDestroy(pMsg);
@@ -294,8 +295,8 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
   int32_t ret = 0;
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
     SRaftId destId;
-    syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &destId);
-    SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId);
+    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId);
+    SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
     ret = syncNodePing(pSyncNode, &destId, pMsg);
     assert(ret == 0);
     syncPingDestroy(pMsg);
@@ -304,7 +305,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
 
 int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
   int32_t   ret;
-  SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &pSyncNode->raftId);
+  SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
   ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg);
   assert(ret == 0);
   syncPingDestroy(pMsg);
@@ -385,7 +386,7 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
     cJSON_Delete(pJson);
   }
 
-  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->raftId, &pMsg->srcId);
+  SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId);
   SRpcMsg        rpcMsg;
   syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
   syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
@@ -485,7 +486,7 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
 //
 static void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
   pSyncNode->state = TAOS_SYNC_STATE_LEADER;
-  pSyncNode->leaderCache = pSyncNode->raftId;
+  pSyncNode->leaderCache = pSyncNode->myRaftId;
 
   // next Index +=1
   // match Index = 0;
diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp
index 0bf43f933e..e2bc9a73ae 100644
--- a/source/libs/sync/test/syncEnqTest.cpp
+++ b/source/libs/sync/test/syncEnqTest.cpp
@@ -84,7 +84,7 @@ int main(int argc, char** argv) {
   gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
 
   for (int i = 0; i < 10; ++i) {
-    SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->raftId, &pSyncNode->raftId);
+    SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
     SRpcMsg        rpcMsg;
     syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
     pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);

From ffe442301c532ca5d0139ac4e23d8097d2abf34e Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 14:19:50 +0800
Subject: [PATCH 06/29] sync refactor

---
 source/libs/sync/inc/syncInt.h         |  4 ++-
 source/libs/sync/inc/syncReplication.h |  1 +
 source/libs/sync/src/syncElection.c    |  8 ++++--
 source/libs/sync/src/syncMain.c        | 35 +++++++++++++++++++++++---
 source/libs/sync/src/syncReplication.c | 11 +++++++-
 source/libs/sync/src/syncTimeout.c     |  2 +-
 source/libs/sync/src/syncUtil.c        |  2 +-
 7 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 2c8e36fd08..874763cd45 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -155,7 +155,7 @@ typedef struct SSyncNode {
   SSyncLogStore* pLogStore;
   SyncIndex      commitIndex;
 
-  // timer
+  // ping timer
   tmr_h             pPingTimer;
   int32_t           pingTimerMS;
   uint64_t          pingTimerLogicClock;
@@ -163,6 +163,7 @@ typedef struct SSyncNode {
   TAOS_TMR_CALLBACK FpPingTimer;  // Timer Fp
   uint64_t          pingTimerCounter;
 
+  // elect timer
   tmr_h             pElectTimer;
   int32_t           electTimerMS;
   uint64_t          electTimerLogicClock;
@@ -170,6 +171,7 @@ typedef struct SSyncNode {
   TAOS_TMR_CALLBACK FpElectTimer;  // Timer Fp
   uint64_t          electTimerCounter;
 
+  // heartbeat timer
   tmr_h             pHeartbeatTimer;
   int32_t           heartbeatTimerMS;
   uint64_t          heartbeatTimerLogicClock;
diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h
index 467cfdde5c..aca6205b9d 100644
--- a/source/libs/sync/inc/syncReplication.h
+++ b/source/libs/sync/inc/syncReplication.h
@@ -53,6 +53,7 @@ extern "C" {
 //
 int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode);
 
+int32_t syncNodeReplicate(SSyncNode* pSyncNode);
 int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg);
 
 #ifdef __cplusplus
diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c
index 87017b718d..24f41a0e69 100644
--- a/source/libs/sync/src/syncElection.c
+++ b/source/libs/sync/src/syncElection.c
@@ -28,11 +28,15 @@
 //             mdest         |-> j])
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
-int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {}
+int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  return ret;
+}
 
 int32_t syncNodeElect(SSyncNode* pSyncNode) {
   // start election
-  syncNodeRequestVotePeers(pSyncNode);
+  int32_t ret = syncNodeRequestVotePeers(pSyncNode);
+  return ret;
 }
 
 int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) {
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 44d2e4c160..f2939df60c 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -37,7 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId);
 static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
 static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
 
-static void UpdateTerm(SyncTerm term);
+static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
 static void syncNodeBecomeFollower(SSyncNode* pSyncNode);
 static void syncNodeBecomeLeader(SSyncNode* pSyncNode);
 static void syncNodeFollower2Candidate(SSyncNode* pSyncNode);
@@ -452,9 +452,38 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
   }
 }
 
-static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {}
+static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
+  SSyncNode* pSyncNode = (SSyncNode*)param;
+  if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <=
+      atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) {
+    SyncTimeout* pSyncMsg =
+        syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock),
+                          pSyncNode->heartbeatTimerMS, pSyncNode);
 
-static void UpdateTerm(SyncTerm term) {}
+    SRpcMsg rpcMsg;
+    syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
+    pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
+    syncTimeoutDestroy(pSyncMsg);
+
+    // reset timer ms
+    // pSyncNode->heartbeatTimerMS += 100;
+
+    taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager,
+                 &pSyncNode->pHeartbeatTimer);
+  } else {
+    sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu",
+           pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
+  }
+}
+
+static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) {
+  if (term > pSyncNode->pRaftStore->currentTerm) {
+    pSyncNode->pRaftStore->currentTerm = term;
+    pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID;
+    raftStorePersist(pSyncNode->pRaftStore);
+    syncNodeBecomeFollower(pSyncNode);
+  }
+}
 
 static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
   if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c
index 37e8959ff3..dfbe5db0ed 100644
--- a/source/libs/sync/src/syncReplication.c
+++ b/source/libs/sync/src/syncReplication.c
@@ -41,7 +41,16 @@
 //                mdest          |-> j])
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
-int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {}
+int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
+  int32_t ret = 0;
+  return ret;
+}
+
+int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
+  // start replicate
+  int32_t ret = syncNodeAppendEntriesPeers(pSyncNode);
+  return ret;
+}
 
 int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) {
   sTrace("syncNodeAppendEntries pSyncNode:%p ", pSyncNode);
diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c
index df9b9d27b4..7cbfd6d40a 100644
--- a/source/libs/sync/src/syncTimeout.c
+++ b/source/libs/sync/src/syncTimeout.c
@@ -44,7 +44,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) {
   } else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) {
     if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) {
       ++(ths->heartbeatTimerCounter);
-      syncNodeAppendEntriesPeers(ths);
+      syncNodeReplicate(ths);
     }
   } else {
     sTrace("unknown timeoutType:%d", pMsg->timeoutType);
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index 04be3f33ac..fa6c245fd8 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -127,7 +127,7 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) {
   cJSON_AddNumberToObject(pRoot, "vgId", p->vgId);
 
   cJSON* pJson = cJSON_CreateObject();
-  cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot);
+  cJSON_AddItemToObject(pJson, "SRaftId", pRoot);
   return pJson;
 }
 

From bf0e4b782824e5f398d1f53b42ffcc9d8d9be56e Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 8 Mar 2022 14:31:02 +0800
Subject: [PATCH 07/29] Test/sangshuduo/td 13408 move tests in for3.0 (#10613)

* restore .gitmodules

* Revert "[TD-13408]<test>: move tests out"

This reverts commit f80a4ca49ff37431bc0bc0dafb5ccf5858b00beb.

* revert f80a4ca49ff37431bc0bc0dafb5ccf5858b00beb

* immigrate file change from stand-alone repo to TDengine

for 3.0

* remove tests repository for Jenkinsfile2

* remove tests/test from .gitignore

* add examples/rust back

for master

* fix typo

Co-authored-by: tangfangzhi <fztang@taosdata.com>
---
 .gitmodules   | 3 +++
 examples/rust | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 examples/rust

diff --git a/.gitmodules b/.gitmodules
index e9af594edc..07e4bb2b9c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "deps/TSZ"]
 	path = deps/TSZ
 	url = https://github.com/taosdata/TSZ.git
+[submodule "examples/rust"]
+	path = examples/rust
+	url = https://github.com/songtianyi/tdengine-rust-bindings.git
diff --git a/examples/rust b/examples/rust
new file mode 160000
index 0000000000..1c8924dc66
--- /dev/null
+++ b/examples/rust
@@ -0,0 +1 @@
+Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df

From c3119245fbe7c47ac0036d0eed21b6e7bf56e807 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 8 Mar 2022 14:37:37 +0800
Subject: [PATCH 08/29] [TD-13894]<fix>: test-all script for 3.0 (#10623)

---
 tests/test-all.sh | 129 ++++++++++++++++++++++++----------------------
 1 file changed, 68 insertions(+), 61 deletions(-)

diff --git a/tests/test-all.sh b/tests/test-all.sh
index c2bc305de0..aa7c4240bc 100755
--- a/tests/test-all.sh
+++ b/tests/test-all.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# set -x
+
 # Color setting
 RED='\033[0;31m'
 GREEN='\033[1;32m'
@@ -12,19 +14,19 @@ tests_dir=`pwd`
 IN_TDINTERNAL="community"
 
 function stopTaosd {
-	echo "Stop taosd"
+  echo "Stop taosd"
   sudo systemctl stop taosd || echo 'no sudo or systemctl or stop fail'
   PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
-	while [ -n "$PID" ]
-	do
+  while [ -n "$PID" ]
+  do
     pkill -TERM -x taosd
     sleep 1
-  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
-	done
+    PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+  done
 }
 
 function dohavecore(){
-  corefile=`find $corepath -mmin 1`  
+  [ -d $corepath ] && corefile=`find $corepath -mmin 1` || return 1
   if [ -n "$corefile" ];then
     core_file=`echo $corefile|cut -d " " -f2`
     proc=`file $core_file|awk -F "execfn:"  '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,`
@@ -39,9 +41,9 @@ function dohavecore(){
         cd community
         cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
       fi
-    else 
+    else
       cd ../../
-      if [[ $1 == 1 ]];then 
+      if [[ $1 == 1 ]];then
         #tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug
         cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
       fi
@@ -56,20 +58,20 @@ function dohavecore(){
 function runSimCaseOneByOne {
   while read -r line; do
     if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
-			case=`echo $line | grep sim$ |awk '{print $NF}'`    
-      start_time=`date +%s`      
+      case=`echo $line | grep sim$ |awk '{print $NF}'`
+      start_time=`date +%s`
       date +%F\ %T | tee -a out.log
       if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
         echo -n $case
-        ./test.sh -f $case > /dev/null 2>&1 && \
-        ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
-        ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
+        ./test.sh -f $case > /dev/null 2>&1 \
+            && ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log ) \
+            || ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         echo -e "${RED} failed${NC}" | tee -a out.log
       else
         echo -n $case
         ./test.sh -f $case > /dev/null 2>&1 && \
-        ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
-        ( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
+        ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
+        ([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         echo -e "${RED} failed${NC}" | tee -a out.log
       fi
       out_log=`tail -1 out.log  `
@@ -85,35 +87,35 @@ function runSimCaseOneByOne {
 function runSimCaseOneByOnefq {
 
   start=`sed -n "/$1-start/=" jenkins/basic.txt`
-  end=`sed -n "/$1-end/=" jenkins/basic.txt`  
+  end=`sed -n "/$1-end/=" jenkins/basic.txt`
   for ((i=$start;i<=$end;i++)) ; do
     line=`sed -n "$i"p jenkins/basic.txt`
     if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
-			case=`echo $line | grep sim$ |awk '{print $NF}'`
+      case=`echo $line | grep sim$ |awk '{print $NF}'`
 
-      start_time=`date +%s`    
+      start_time=`date +%s`
       date +%F\ %T | tee -a out.log
       if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
         echo -n $case
-        ./test.sh -f $case > case.log 2>&1 && \
-        ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
-        ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
+        ./test.sh -f $case > case.log 2>&1 \
+            && \
+        ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
+        ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
       else
-	      pwd
         echo -n $case
         ./test.sh -f $case > ../../sim/case.log 2>&1 && \
-        ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
-        ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
+        ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
+        ([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
         ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' &&  pwd && cat ../../sim/case.log )
       fi
-      
+
       out_log=`tail -1 out.log  `
       if [[ $out_log =~ 'failed' ]];then
         rm case.log
         if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
           cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
-        else 
+        else
           cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
         fi
         dohavecore $2 1
@@ -125,7 +127,7 @@ function runSimCaseOneByOnefq {
       echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
       dohavecore $2 1
     fi
-  done 
+  done
   rm -rf ../../../sim/case.log
   rm -rf ../../sim/case.log
 }
@@ -134,7 +136,6 @@ function runPyCaseOneByOne {
   while read -r line; do
     if [[ $line =~ ^python.* ]]; then
       if [[ $line != *sleep* ]]; then
-        
         if [[ $line =~ '-r' ]];then
           case=`echo $line|awk '{print $4}'`
         else
@@ -172,7 +173,6 @@ function runPyCaseOneByOnefq() {
     line=`sed -n "$i"p fulltest.sh`
     if [[ $line =~ ^python.* ]]; then
       if [[ $line != *sleep* ]]; then
-        
         if [[ $line =~ '-r' ]];then
           case=`echo $line|awk '{print $4}'`
         else
@@ -186,7 +186,7 @@ function runPyCaseOneByOnefq() {
         fi
         $line > case.log 2>&1 && \
           echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \
-          echo -e "${RED} failed${NC}" | tee -a pytest-out.log 
+          echo -e "${RED} failed${NC}" | tee -a pytest-out.log
         end_time=`date +%s`
         out_log=`tail -1 pytest-out.log  `
         if [[ $out_log =~ 'failed' ]];then
@@ -205,7 +205,7 @@ function runPyCaseOneByOnefq() {
       fi
       dohavecore $2 2
     fi
-  done 
+  done
   rm -rf ../../sim/case.log
 }
 
@@ -239,7 +239,9 @@ totalExampleFailed=0
 if [ "${OS}" == "Linux" ]; then
     corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern`
     if [ -z "$corepath" ];then
-      echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied"
+        [ -d /coredump ] || mkdir /coredump || echo -e "failed to mdkir /coredump"
+        [ -d /coredump ] \
+            && echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied"
       corepath="/coredump/"
     fi
 fi
@@ -248,6 +250,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]  && [ "$2" !
   echo "### run TSIM test case ###"
   cd $tests_dir/script
 
+  [ -d ../../sim ] || mkdir ../../sim || echo -e "failed to mkdir ../../sim"
   [ -f out.log ] && rm -f out.log
   if [ "$1" == "cron" ]; then
     echo "### run TSIM regression test ###"
@@ -358,7 +361,7 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]  && [ "$2" != "
     runPyCaseOneByOnefq p2 1
   elif [ "$1" == "p3" ]; then
     echo "### run Python_3 test ###"
-    runPyCaseOneByOnefq p3 1 
+    runPyCaseOneByOnefq p3 1
   elif [ "$1" == "p4" ]; then
     echo "### run Python_4 test ###"
     runPyCaseOneByOnefq p4 1
@@ -368,16 +371,20 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]  && [ "$2" != "
     echo "### run Python smoke test ###"
     runPyCaseOneByOne smoketest.sh
   fi
-  totalPySuccess=`grep 'success' pytest-out.log | wc -l`
 
-  if [ "$totalPySuccess" -gt "0" ]; then
-    echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
-  fi
+  if [ -f pytest-out.log ]; then
+    totalPySuccess=`grep 'success' pytest-out.log | wc -l`
+    if [ "$totalPySuccess" -gt "0" ]; then
+        echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
+    fi
 
-  totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
-  if [ "$totalPyFailed" -ne "0" ]; then
-    echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
-#  exit $totalPyFailed
+    totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
+    if [ "$totalPyFailed" -ne "0" ]; then
+        echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
+        #  exit $totalPyFailed
+    fi
+  else
+    echo -e "pytest-out.log not found!"
   fi
 fi
 
@@ -395,14 +402,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]  && [ "$2" !=
 
   pwd
   cd debug/
-  
+
   stopTaosd
   rm -rf /var/lib/taos/*
   nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 &
   sleep 30
-  
-  cd $tests_dir/../src/connector/jdbc  
-  
+
+  cd $tests_dir/../src/connector/jdbc
+
   mvn test > jdbc-out.log 2>&1
   tail -n 20 jdbc-out.log
 
@@ -412,14 +419,14 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]  && [ "$2" !=
   JDBCFailed=`echo ${failed/%,}`
   error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'`
   JDBCError=`echo ${error/%,}`
-  
+
   totalJDBCFailed=`expr $JDBCFailed + $JDBCError`
   totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed`
 
   if [ "$totalJDBCSuccess" -gt "0" ]; then
     echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}"
   fi
-  
+
   if [ "$totalJDBCFailed" -ne "0" ]; then
     echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}"
   fi
@@ -427,7 +434,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]  && [ "$2" !=
 fi
 
 if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ]  && [ "$2" != "example" ]  && [ "$1" == "full" ]; then
-  echo "### run Unit tests ###"  
+  echo "### run Unit tests ###"
 
   stopTaosd
   cd $tests_dir
@@ -443,19 +450,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ]  && [ "$2" !=
   rm -rf /var/lib/taos/*
   nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 &
   sleep 30
-  
+
   pwd
   ./queryTest > unittest-out.log 2>&1
   tail -n 20 unittest-out.log
 
-  totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'`  
+  totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'`
   totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'`
   totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess`
 
   if [ "$totalUnitSuccess" -gt "0" ]; then
     echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}"
   fi
-  
+
   if [ "$totalUnitFailed" -ne "0" ]; then
     echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}"
   fi
@@ -463,7 +470,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ]  && [ "$2" !=
 fi
 
 if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then
-  echo "### run Example tests ###"  
+  echo "### run Example tests ###"
 
   stopTaosd
   cd $tests_dir
@@ -480,7 +487,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
   nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 &
   echo "sleeping for 30 seconds"
   #sleep 30
-  
+
   cd $tests_dir
   echo "current dir: "
   pwd
@@ -493,16 +500,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
   ./apitest > /dev/null 2>&1
   if [ $? != "0" ]; then
     echo "apitest failed"
-    totalExampleFailed=`expr $totalExampleFailed + 1`    
+    totalExampleFailed=`expr $totalExampleFailed + 1`
   else
     echo "apitest pass"
     totalExamplePass=`expr $totalExamplePass + 1`
-  fi 
+  fi
 
   ./prepare 127.0.0.1 > /dev/null 2>&1
   if [ $? != "0" ]; then
     echo "prepare failed"
-    totalExampleFailed=`expr $totalExampleFailed + 1`    
+    totalExampleFailed=`expr $totalExampleFailed + 1`
   else
     echo "prepare pass"
     totalExamplePass=`expr $totalExamplePass + 1`
@@ -511,7 +518,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
   ./subscribe -test > /dev/null 2>&1
   if [ $? != "0" ]; then
     echo "subscribe failed"
-    totalExampleFailed=`expr $totalExampleFailed + 1`    
+    totalExampleFailed=`expr $totalExampleFailed + 1`
   else
     echo "subscribe pass"
     totalExamplePass=`expr $totalExamplePass + 1`
@@ -520,7 +527,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
   yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1
   if [ $? != "0" ]; then
     echo "asyncdemo failed"
-    totalExampleFailed=`expr $totalExampleFailed + 1`    
+    totalExampleFailed=`expr $totalExampleFailed + 1`
   else
     echo "asyncdemo pass"
     totalExamplePass=`expr $totalExamplePass + 1`
@@ -529,16 +536,16 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
   ./demo 127.0.0.1 > /dev/null 2>&1
   if [ $? != "0" ]; then
     echo "demo failed"
-    totalExampleFailed=`expr $totalExampleFailed + 1`    
+    totalExampleFailed=`expr $totalExampleFailed + 1`
   else
     echo "demo pass"
     totalExamplePass=`expr $totalExamplePass + 1`
   fi
-  
+
   if [ "$totalExamplePass" -gt "0" ]; then
     echo -e "\n${GREEN} ### Total $totalExamplePass examples succeed! ### ${NC}"
   fi
-  
+
   if [ "$totalExampleFailed" -ne "0" ]; then
     echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}"
   fi

From 54c260738fa01676431144acbfbc84e00b0c219a Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 14:45:03 +0800
Subject: [PATCH 09/29] sync refactor

---
 source/libs/sync/src/syncMain.c        | 55 ++++++++++++++++++++++++++
 source/libs/sync/src/syncUtil.c        |  2 +-
 source/libs/sync/test/CMakeLists.txt   | 14 +++++++
 source/libs/sync/test/syncUtilTest.cpp | 32 +++++++++++++++
 4 files changed, 102 insertions(+), 1 deletion(-)
 create mode 100644 source/libs/sync/test/syncUtilTest.cpp

diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index f2939df60c..5338b5bd7f 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -231,6 +231,61 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
 
   // tla+ log vars
 
+  // ping timer
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer);
+  cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf);
+  cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClock);
+  cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser);
+  cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer);
+  cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter);
+  cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf);
+
+  // elect timer
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer);
+  cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf);
+  cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClock);
+  cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser);
+  cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer);
+  cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter);
+  cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf);
+
+  // heartbeat timer
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer);
+  cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf);
+  cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClock);
+  cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser);
+  cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer);
+  cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter);
+  cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf);
+
+  // callback
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing);
+  cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply);
+  cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote);
+  cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply);
+  cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries);
+  cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply);
+  cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout);
+  cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf);
+
   cJSON* pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "SSyncNode", pRoot);
   return pJson;
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index fa6c245fd8..d2b413bdaa 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -97,7 +97,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) {
 
 int32_t syncUtilRand(int32_t max) { return rand() % max; }
 
-int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
+int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
 
 int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
 
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 3a8eea53d4..f7f0a0795c 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -11,6 +11,7 @@ add_executable(syncRaftStoreTest "")
 add_executable(syncEnqTest "")
 add_executable(syncIndexTest "")
 add_executable(syncInitTest "")
+add_executable(syncUtilTest "")
 
 
 target_sources(syncTest
@@ -65,6 +66,10 @@ target_sources(syncInitTest
     PRIVATE
     "syncInitTest.cpp"
 )
+target_sources(syncUtilTest
+    PRIVATE
+    "syncUtilTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -132,6 +137,11 @@ target_include_directories(syncInitTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncUtilTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -186,6 +196,10 @@ target_link_libraries(syncInitTest
     sync
     gtest_main
 )
+target_link_libraries(syncUtilTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp
new file mode 100644
index 0000000000..9a1c113620
--- /dev/null
+++ b/source/libs/sync/test/syncUtilTest.cpp
@@ -0,0 +1,32 @@
+#include "syncUtil.h"
+//#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+void electRandomMSTest() {
+  for (int i = 0; i < 10; ++i) {
+    int32_t ms = syncUtilElectRandomMS();
+    printf("syncUtilElectRandomMS: %d \n", ms);
+  }
+}
+
+int main() {
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
+  tsAsyncLog = 0;
+  sDebugFlag = 143 + 64;
+  logTest();
+  electRandomMSTest();
+
+  return 0;
+}

From 2ae6f747f918cecad4d6eeebbaaf35ad4c47ffea Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 17:07:29 +0800
Subject: [PATCH 10/29] sync refactor

---
 source/libs/sync/inc/syncVoteMgr.h            | 10 ++-
 source/libs/sync/src/syncElection.c           | 17 ++++
 source/libs/sync/src/syncMain.c               |  4 +-
 source/libs/sync/src/syncVoteMgr.c            | 70 +++++++++++++++-
 source/libs/sync/test/CMakeLists.txt          | 14 ++++
 .../libs/sync/test/syncVotesGrantedTest.cpp   | 80 +++++++++++++++++++
 6 files changed, 188 insertions(+), 7 deletions(-)
 create mode 100644 source/libs/sync/test/syncVotesGrantedTest.cpp

diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h
index e2307e9e66..a769bfbccd 100644
--- a/source/libs/sync/inc/syncVoteMgr.h
+++ b/source/libs/sync/inc/syncVoteMgr.h
@@ -28,10 +28,14 @@ extern "C" {
 #include "syncUtil.h"
 #include "taosdef.h"
 
+// SVotesGranted -----------------------------
 typedef struct SVotesGranted {
+  SRaftId (*replicas)[TSDB_MAX_REPLICA];
+  int32_t    replicaNum;
+  bool       isGranted[TSDB_MAX_REPLICA];
+  int32_t    votes;
   SyncTerm   term;
   int32_t    quorum;
-  int32_t    votes;
   bool       toLeader;
   SSyncNode *pSyncNode;
 } SVotesGranted;
@@ -41,7 +45,10 @@ void           voteGrantedDestroy(SVotesGranted *pVotesGranted);
 bool           voteGrantedMajority(SVotesGranted *pVotesGranted);
 void           voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg);
 void           voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term);
+cJSON *        voteGranted2Json(SVotesGranted *pVotesGranted);
+char *         voteGranted2Str(SVotesGranted *pVotesGranted);
 
+// SVotesRespond -----------------------------
 typedef struct SVotesRespond {
   SRaftId (*replicas)[TSDB_MAX_REPLICA];
   bool       isRespond[TSDB_MAX_REPLICA];
@@ -51,6 +58,7 @@ typedef struct SVotesRespond {
 } SVotesRespond;
 
 SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode);
+void           votesRespondDestory(SVotesRespond *pVotesRespond);
 bool           votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId);
 void           votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg);
 void           Reset(SVotesRespond *pVotesRespond, SyncTerm term);
diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c
index 24f41a0e69..223431336e 100644
--- a/source/libs/sync/src/syncElection.c
+++ b/source/libs/sync/src/syncElection.c
@@ -15,6 +15,7 @@
 
 #include "syncElection.h"
 #include "syncMessage.h"
+#include "syncRaftStore.h"
 
 // TLA+ Spec
 // RequestVote(i, j) ==
@@ -29,11 +30,27 @@
 //    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
 //
 int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+
   int32_t ret = 0;
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    SyncRequestVote* pMsg = syncRequestVoteBuild();
+    pMsg->srcId = pSyncNode->myRaftId;
+    pMsg->destId = pSyncNode->peersId[i];
+    pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm;
+    pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore);
+    pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore);
+
+    ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg);
+    assert(ret == 0);
+    syncRequestVoteDestroy(pMsg);
+  }
   return ret;
 }
 
 int32_t syncNodeElect(SSyncNode* pSyncNode) {
+  assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
+
   // start election
   int32_t ret = syncNodeRequestVotePeers(pSyncNode);
   return ret;
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 5338b5bd7f..ad424b9353 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -107,12 +107,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
     syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]);
   }
 
-  // raft algorithm
+  // init raft algorithm
   pSyncNode->pFsm = pSyncInfo->pFsm;
   pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum);
   pSyncNode->leaderCache = EMPTY_RAFT_ID;
 
-  // life cycle
+  // init life cycle
 
   // init server vars
   pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c
index c9f0ceab57..407b7b1941 100644
--- a/source/libs/sync/src/syncVoteMgr.c
+++ b/source/libs/sync/src/syncVoteMgr.c
@@ -14,15 +14,25 @@
  */
 
 #include "syncVoteMgr.h"
+#include "syncUtil.h"
+
+// SVotesGranted -----------------------------
+static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) {
+  memset(pVotesGranted->isGranted, 0, sizeof(pVotesGranted->isGranted));
+  pVotesGranted->votes = 0;
+}
 
 SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) {
   SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted));
   assert(pVotesGranted != NULL);
   memset(pVotesGranted, 0, sizeof(SVotesGranted));
 
-  pVotesGranted->quorum = pSyncNode->quorum;
+  pVotesGranted->replicas = &(pSyncNode->replicasId);
+  pVotesGranted->replicaNum = pSyncNode->replicaNum;
+  voteGrantedClearVotes(pVotesGranted);
+
   pVotesGranted->term = 0;
-  pVotesGranted->votes = 0;
+  pVotesGranted->quorum = pSyncNode->quorum;
   pVotesGranted->toLeader = false;
   pVotesGranted->pSyncNode = pSyncNode;
 
@@ -43,15 +53,61 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) {
 void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) {
   assert(pMsg->voteGranted == true);
   assert(pMsg->term == pVotesGranted->term);
-  pVotesGranted->votes++;
+  assert(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId));
+
+  int j = -1;
+  for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
+    if (syncUtilSameId(&((*(pVotesGranted->replicas))[i]), &(pMsg->srcId))) {
+      j = i;
+      break;
+    }
+  }
+  assert(j != -1);
+
+  if (pVotesGranted->isGranted[j] != true) {
+    ++(pVotesGranted->votes);
+    pVotesGranted->isGranted[j] = true;
+  }
+  assert(pVotesGranted->votes <= pVotesGranted->replicaNum);
 }
 
 void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) {
   pVotesGranted->term = term;
-  pVotesGranted->votes = 0;
+  voteGrantedClearVotes(pVotesGranted);
   pVotesGranted->toLeader = false;
 }
 
+cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum);
+  cJSON *pReplicas = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
+  for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
+    cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i])));
+  }
+  cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes);
+  snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term);
+  cJSON_AddStringToObject(pRoot, "term", u64buf);
+  cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum);
+  cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader);
+  snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode);
+  cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot);
+  return pJson;
+}
+
+char *voteGranted2Str(SVotesGranted *pVotesGranted) {
+  cJSON *pJson = voteGranted2Json(pVotesGranted);
+  char * serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+// SVotesRespond -----------------------------
 SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) {
   SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond));
   assert(pVotesRespond != NULL);
@@ -65,6 +121,12 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) {
   return pVotesRespond;
 }
 
+void votesRespondDestory(SVotesRespond *pVotesRespond) {
+  if (pVotesRespond != NULL) {
+    free(pVotesRespond);
+  }
+}
+
 bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) {
   bool ret = false;
   for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index f7f0a0795c..18b7748105 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -12,6 +12,7 @@ add_executable(syncEnqTest "")
 add_executable(syncIndexTest "")
 add_executable(syncInitTest "")
 add_executable(syncUtilTest "")
+add_executable(syncVotesGrantedTest "")
 
 
 target_sources(syncTest
@@ -70,6 +71,10 @@ target_sources(syncUtilTest
     PRIVATE
     "syncUtilTest.cpp"
 )
+target_sources(syncVotesGrantedTest
+    PRIVATE
+    "syncVotesGrantedTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -142,6 +147,11 @@ target_include_directories(syncUtilTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncVotesGrantedTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -200,6 +210,10 @@ target_link_libraries(syncUtilTest
     sync
     gtest_main
 )
+target_link_libraries(syncVotesGrantedTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp
new file mode 100644
index 0000000000..97b807736b
--- /dev/null
+++ b/source/libs/sync/test/syncVotesGrantedTest.cpp
@@ -0,0 +1,80 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+#include "syncVoteMgr.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t ports[3] = {7010, 7110, 7210};
+
+SSyncNode* doSync(int myIndex) {
+  SSyncFSM* pFsm;
+
+  SSyncInfo syncInfo;
+  syncInfo.vgId = 1;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = pFsm;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+  pCfg->myIndex = myIndex;
+  pCfg->replicaNum = 3;
+
+  pCfg->nodeInfo[0].nodePort = ports[0];
+  snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  pCfg->nodeInfo[1].nodePort = ports[1];
+  snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
+
+  pCfg->nodeInfo[2].nodePort = ports[2];
+  snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
+  // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+
+  SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
+  assert(pSyncNode != NULL);
+
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->pSyncNode = pSyncNode;
+
+  return pSyncNode;
+}
+
+int main() {
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
+  tsAsyncLog = 0;
+  sDebugFlag = 143 + 64;
+  logTest();
+
+  int     myIndex = 0;
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  assert(ret == 0);
+
+  ret = syncEnvStart();
+  assert(ret == 0);
+
+  SSyncNode*     pSyncNode = doSync(myIndex);
+  SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
+  assert(pVotesGranted != NULL);
+
+  char* serialized = voteGranted2Str(pVotesGranted);
+  assert(serialized != NULL);
+  printf("%s\n", serialized);
+  free(serialized);
+
+  return 0;
+}

From 255d79544295df25703b18a8d03b436859cf7ec9 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 8 Mar 2022 17:22:21 +0800
Subject: [PATCH 11/29] add tq push

---
 include/common/tcommon.h                      |   6 +-
 include/common/tmsg.h                         |  12 +-
 include/libs/qcom/query.h                     | 145 +++++++++++-------
 source/client/src/clientImpl.c                |   4 +-
 source/client/src/tmq.c                       |  30 ++--
 source/common/src/tmsg.c                      |   4 +-
 source/dnode/mnode/impl/src/mndDb.c           |   6 +-
 source/dnode/mnode/impl/src/mndScheduler.c    |  31 ++--
 source/dnode/mnode/impl/src/mndSubscribe.c    |  15 +-
 source/dnode/mnode/impl/src/mnode.c           |   8 +-
 source/dnode/mnode/impl/test/db/db.cpp        |  12 +-
 source/dnode/vnode/inc/tq.h                   |  14 +-
 source/dnode/vnode/src/inc/tqInt.h            |  10 +-
 source/dnode/vnode/src/inc/tqPush.h           |  77 ++++++++++
 source/dnode/vnode/src/tq/tq.c                |  63 ++++----
 source/dnode/vnode/src/tq/tqPush.c            |  84 ++++++++++
 source/dnode/vnode/src/vnd/vnodeQuery.c       |  12 +-
 source/libs/catalog/src/catalog.c             |   2 +-
 source/libs/catalog/test/catalogTests.cpp     |  48 +++---
 source/libs/executor/src/executorimpl.c       |   4 +-
 source/libs/parser/src/dCDAstProcess.c        |   4 +-
 .../libs/parser/test/mockCatalogService.cpp   |  22 +--
 source/libs/planner/src/physicalPlan.c        |   4 +-
 source/libs/planner/src/physicalPlanJson.c    |  10 +-
 source/libs/scheduler/src/scheduler.c         |   6 +-
 source/libs/scheduler/test/schedulerTests.cpp |  14 +-
 tests/test/c/tmqDemo.c                        |   6 +-
 27 files changed, 438 insertions(+), 215 deletions(-)
 create mode 100644 source/dnode/vnode/src/inc/tqPush.h
 create mode 100644 source/dnode/vnode/src/tq/tqPush.c

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index d0ec5c9296..1d3ab4f340 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -135,7 +135,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
   return (void*)buf;
 }
 
-static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) {
+static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
   int32_t tlen = 0;
   int32_t sz = 0;
   tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
@@ -156,7 +156,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
   return tlen;
 }
 
-static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
+static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
   int32_t sz;
   buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
   buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
@@ -194,7 +194,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
   // tfree(pBlock);
 }
 
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) {
+static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
   if (pRsp->schemas) {
     if (pRsp->schemas->nCols) {
       tfree(pRsp->schemas->pSchema);
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index d56db2046e..238753e5b7 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -795,7 +795,7 @@ typedef struct SVgroupInfo {
   int32_t  vgId;
   uint32_t hashBegin;
   uint32_t hashEnd;
-  SEpSet   epset;
+  SEpSet   epSet;
 } SVgroupInfo;
 
 typedef struct {
@@ -1876,8 +1876,8 @@ typedef struct {
 } SVCreateTSmaReq;
 
 typedef struct {
-  int8_t      type;  // 0 status report, 1 update data
-  char        indexName[TSDB_INDEX_NAME_LEN + 1]; // 
+  int8_t      type;                                // 0 status report, 1 update data
+  char        indexName[TSDB_INDEX_NAME_LEN + 1];  //
   STimeWindow windows;
 } STSmaMsg;
 
@@ -2073,7 +2073,7 @@ typedef struct {
   int32_t         skipLogNum;
   int32_t         numOfTopics;
   SArray*         pBlockData;  // SArray<SSDataBlock>
-} SMqConsumeRsp;
+} SMqPollRsp;
 
 // one req for one vg+topic
 typedef struct {
@@ -2086,7 +2086,7 @@ typedef struct {
 
   int64_t currentOffset;
   char    topic[TSDB_TOPIC_FNAME_LEN];
-} SMqConsumeReq;
+} SMqPollReq;
 
 typedef struct {
   int32_t vgId;
@@ -2108,7 +2108,7 @@ typedef struct {
 struct tmq_message_t {
   SMqRspHead head;
   union {
-    SMqConsumeRsp    consumeRsp;
+    SMqPollRsp       consumeRsp;
     SMqCMGetSubEpRsp getEpRsp;
   };
   void* extra;
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index 1f56254476..5d5ab74ba9 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -44,10 +44,10 @@ enum {
 };
 
 typedef struct STableComInfo {
-  uint8_t numOfTags;      // the number of tags in schema
-  uint8_t precision;      // the number of precision
-  int16_t numOfColumns;   // the number of columns
-  int32_t rowSize;        // row size of the schema
+  uint8_t numOfTags;     // the number of tags in schema
+  uint8_t precision;     // the number of precision
+  int16_t numOfColumns;  // the number of columns
+  int32_t rowSize;       // row size of the schema
 } STableComInfo;
 
 /*
@@ -56,49 +56,45 @@ typedef struct STableComInfo {
  * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
  */
 typedef struct SCTableMeta {
-  int32_t  vgId:24;
+  int32_t  vgId : 24;
   int8_t   tableType;
   uint64_t uid;
   uint64_t suid;
 } SCTableMeta;
 
 /*
- * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
+ * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a
+ * SCTableMeta.
  */
 typedef struct STableMeta {
-  //BEGIN: KEEP THIS PART SAME WITH SCTableMeta
-  int32_t        vgId:24;
-  int8_t         tableType;
-  uint64_t       uid;
-  uint64_t       suid;
-  //END: KEEP THIS PART SAME WITH SCTableMeta
-  
-  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
-  int16_t        sversion;
-  int16_t        tversion;
-  STableComInfo  tableInfo;
-  SSchema        schema[];
+  // BEGIN: KEEP THIS PART SAME WITH SCTableMeta
+  int32_t  vgId : 24;
+  int8_t   tableType;
+  uint64_t uid;
+  uint64_t suid;
+  // END: KEEP THIS PART SAME WITH SCTableMeta
+
+  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
+  // info
+  int16_t       sversion;
+  int16_t       tversion;
+  STableComInfo tableInfo;
+  SSchema       schema[];
 } STableMeta;
 
 typedef struct SDBVgInfo {
-  int32_t   vgVersion;  
+  int32_t   vgVersion;
   int8_t    hashMethod;
-  SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
+  SHashObj* vgHash;  // key:vgId, value:SVgroupInfo
 } SDBVgInfo;
 
 typedef struct SUseDbOutput {
-  char           db[TSDB_DB_FNAME_LEN];
-  uint64_t       dbId;
-  SDBVgInfo     *dbVgroup;
+  char       db[TSDB_DB_FNAME_LEN];
+  uint64_t   dbId;
+  SDBVgInfo* dbVgroup;
 } SUseDbOutput;
 
-enum {
-  META_TYPE_NULL_TABLE = 1,
-  META_TYPE_CTABLE,
-  META_TYPE_TABLE,
-  META_TYPE_BOTH_TABLE
-};
-
+enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE };
 
 typedef struct STableMetaOutput {
   int32_t     metaType;
@@ -107,30 +103,30 @@ typedef struct STableMetaOutput {
   char        ctbName[TSDB_TABLE_NAME_LEN];
   char        tbName[TSDB_TABLE_NAME_LEN];
   SCTableMeta ctbMeta;
-  STableMeta *tbMeta;
+  STableMeta* tbMeta;
 } STableMetaOutput;
 
 typedef struct SDataBuf {
-  void     *pData;
-  uint32_t  len;
-  void     *handle;
+  void*    pData;
+  uint32_t len;
+  void*    handle;
 } SDataBuf;
 
 typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code);
 typedef int32_t (*__async_exec_fn_t)(void* param);
 
 typedef struct SMsgSendInfo {
-  __async_send_cb_fn_t fp;        //async callback function
-  void     *param;
-  uint64_t  requestId;
-  uint64_t  requestObjRefId;
-  int32_t   msgType;
-  SDataBuf  msgInfo;
+  __async_send_cb_fn_t fp;  // async callback function
+  void*                param;
+  uint64_t             requestId;
+  uint64_t             requestObjRefId;
+  int32_t              msgType;
+  SDataBuf             msgInfo;
 } SMsgSendInfo;
 
 typedef struct SQueryNodeAddr {
   int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epset;
+  SEpSet  epSet;
 } SQueryNodeAddr;
 
 int32_t initTaskQueue();
@@ -154,32 +150,67 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
  * @param pInfo
  * @return
  */
-int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
+int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
 
-int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp);
+int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp);
 
 void initQueryModuleMsgHandle();
 
 const SSchema* tGetTbnameColumnSchema();
-bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
+bool           tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
-int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta);
+int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
 
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
-extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen);
+extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
 
-#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
-#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
-#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
+#define SET_META_TYPE_NULL(t)       (t) = META_TYPE_NULL_TABLE
+#define SET_META_TYPE_CTABLE(t)     (t) = META_TYPE_CTABLE
+#define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define qFatal(...)  do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0)
-#define qError(...)  do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0)
-#define qWarn(...)   do { if (qDebugFlag & DEBUG_WARN)  { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }}  while(0)
-#define qInfo(...)   do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0)
-#define qDebug(...)  do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
-#define qTrace(...)  do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0)
-#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
+#define qFatal(...)                                                     \
+  do {                                                                  \
+    if (qDebugFlag & DEBUG_FATAL) {                                     \
+      taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
+    }                                                                   \
+  } while (0)
+#define qError(...)                                                     \
+  do {                                                                  \
+    if (qDebugFlag & DEBUG_ERROR) {                                     \
+      taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
+    }                                                                   \
+  } while (0)
+#define qWarn(...)                                                    \
+  do {                                                                \
+    if (qDebugFlag & DEBUG_WARN) {                                    \
+      taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
+    }                                                                 \
+  } while (0)
+#define qInfo(...)                                               \
+  do {                                                           \
+    if (qDebugFlag & DEBUG_INFO) {                               \
+      taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
+    }                                                            \
+  } while (0)
+#define qDebug(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_DEBUG) {                               \
+      taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qTrace(...)                                               \
+  do {                                                            \
+    if (qDebugFlag & DEBUG_TRACE) {                               \
+      taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
+    }                                                             \
+  } while (0)
+#define qDebugL(...)                                                     \
+  do {                                                                   \
+    if (qDebugFlag & DEBUG_DEBUG) {                                      \
+      taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
+    }                                                                    \
+  } while (0)
 
 #ifdef __cplusplus
 }
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 259d0e5799..e9febef7e2 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -517,7 +517,7 @@ void* doFetchRow(SRequestObj* pRequest) {
       SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
       SVgroupInfo*  pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex);
 
-      epSet = pVgroupInfo->epset;
+      epSet = pVgroupInfo->epSet;
     } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) {
       pRequest->type = TDMT_VND_SHOW_TABLES;
       SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
@@ -534,7 +534,7 @@ void* doFetchRow(SRequestObj* pRequest) {
       pRequest->body.requestMsg.pData = pShowReq;
 
       SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
-      epSet = pVgroupInfo->epset;
+      epSet = pVgroupInfo->epSet;
 
       int64_t  transporterId = 0;
       STscObj* pTscObj = pRequest->pTscObj;
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 10dc378518..60103cc9c5 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -13,8 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
-
 #include "clientInt.h"
 #include "clientLog.h"
 #include "parser.h"
@@ -606,17 +604,17 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
-  SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
   return pRsp->skipLogNum;
 }
 
 void tmqShowMsg(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;
 
-  static bool    noPrintSchema;
-  char           pBuf[128];
-  SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
-  int32_t        colNum = pRsp->schemas->nCols;
+  static bool noPrintSchema;
+  char        pBuf[128];
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
+  int32_t     colNum = pRsp->schemas->nCols;
   if (!noPrintSchema) {
     printf("|");
     for (int32_t i = 0; i < colNum; i++) {
@@ -703,7 +701,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
     goto WRITE_QUEUE_FAIL;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
-  tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
+  tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
   if (pRsp->consumeRsp.numOfTopics == 0) {
     /*printf("no data\n");*/
@@ -874,7 +872,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -886,7 +884,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
+  SMqPollReq* pReq = malloc(sizeof(SMqPollReq));
   if (pReq == NULL) {
     return NULL;
   }
@@ -900,7 +898,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien
   pReq->currentOffset = reqOffset;
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
+  pReq->head.contLen = htonl(sizeof(SMqPollReq));
   return pReq;
 }
 
@@ -914,7 +912,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
       /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
       /*continue;*/
       /*}*/
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         // TODO: out of mem
@@ -941,7 +939,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqConsumeReq),
+          .len = sizeof(SMqPollReq),
           .handle = NULL,
       };
       sendInfo->requestId = generateRequestId();
@@ -982,7 +980,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
         continue;
       }
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
         tsem_post(&tmq->rspSem);
@@ -1011,7 +1009,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqConsumeReq),
+          .len = sizeof(SMqPollReq),
           .handle = NULL,
       };
       sendInfo->requestId = generateRequestId();
@@ -1271,7 +1269,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 
 void tmq_message_destroy(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;
-  SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
+  SMqPollRsp* pRsp = &tmq_message->consumeRsp;
   tDeleteSMqConsumeRsp(pRsp);
   /*free(tmq_message);*/
   taosFreeQitem(tmq_message);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index e7309eacc8..97b19c1c79 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -1481,7 +1481,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
     if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
     if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
     if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
-    if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1;
+    if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
   }
 
   return 0;
@@ -1541,7 +1541,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
     if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1;
     if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1;
     if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1;
-    if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1;
+    if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1;
     taosArrayPush(pRsp->pVgroupInfos, &vgInfo);
   }
 
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 974f5fc982..9cde9201ed 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -900,10 +900,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
       vgInfo.vgId = pVgroup->vgId;
       vgInfo.hashBegin = pVgroup->hashBegin;
       vgInfo.hashEnd = pVgroup->hashEnd;
-      vgInfo.epset.numOfEps = pVgroup->replica;
+      vgInfo.epSet.numOfEps = pVgroup->replica;
       for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
         SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
-        SEp       *pEp = &vgInfo.epset.eps[gid];
+        SEp       *pEp = &vgInfo.epSet.eps[gid];
         SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
         if (pDnode != NULL) {
           memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
@@ -911,7 +911,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
         }
         mndReleaseDnode(pMnode, pDnode);
         if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
-          vgInfo.epset.inUse = gid;
+          vgInfo.epSet.inUse = gid;
         }
       }
       vindex++;
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 5308c0c0f6..dfde6a9258 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -33,23 +33,29 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   SSdb*      pSdb = pMnode->pSdb;
   SVgObj*    pVgroup = NULL;
   SQueryDag* pDag = qStringToDag(pTopic->physicalPlan);
-  SArray*    pAray = NULL;
-  SArray*    unassignedVg = pSub->unassignedVg;
+  if (pDag == NULL) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return -1;
+  }
 
   ASSERT(pSub->vgNum == 0);
 
   int32_t levelNum = taosArrayGetSize(pDag->pSubplans);
   if (levelNum != 1) {
+    qDestroyQueryDag(pDag);
+    terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
     return -1;
   }
 
-  SArray* inner = taosArrayGet(pDag->pSubplans, 0);
+  SArray* plans = taosArrayGet(pDag->pSubplans, 0);
 
-  int32_t opNum = taosArrayGetSize(inner);
+  int32_t opNum = taosArrayGetSize(plans);
   if (opNum != 1) {
+    qDestroyQueryDag(pDag);
+    terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
     return -1;
   }
-  SSubplan* plan = taosArrayGetP(inner, 0);
+  SSubplan* plan = taosArrayGetP(plans, 0);
 
   void* pIter = NULL;
   while (1) {
@@ -62,17 +68,24 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
 
     pSub->vgNum++;
     plan->execNode.nodeId = pVgroup->vgId;
-    plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
+    plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
     SMqConsumerEp consumerEp = {0};
     consumerEp.status = 0;
     consumerEp.consumerId = -1;
-    consumerEp.epSet = plan->execNode.epset;
+    consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
     int32_t msgLen;
-    int32_t code = qSubPlanToString(plan, &consumerEp.qmsg, &msgLen);
-    taosArrayPush(unassignedVg, &consumerEp);
+    if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
+      sdbRelease(pSdb, pVgroup);
+      qDestroyQueryDag(pDag);
+      terrno = TSDB_CODE_QRY_INVALID_INPUT;
+      return -1;
+    }
+    taosArrayPush(pSub->unassignedVg, &consumerEp);
   }
 
+  qDestroyQueryDag(pDag);
+
   return 0;
 }
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 9acd881897..1d964c4383 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -93,7 +93,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
   strcpy(pSub->key, key);
 
   if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
-    terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
     tDeleteSMqSubscribeObj(pSub);
     free(pSub);
     return NULL;
@@ -295,7 +294,11 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
           for (int32_t k = 0; k < vgsz; k++) {
             char           offsetKey[TSDB_PARTITION_KEY_LEN];
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
-            SMqSubVgEp     vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
+            SMqSubVgEp     vgEp = {
+                    .epSet = pConsumerEp->epSet,
+                    .vgId = pConsumerEp->vgId,
+                    .offset = -1,
+            };
             mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
             SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
             if (pOffsetObj != NULL) {
@@ -345,7 +348,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   if (pRebSub == NULL) {
     pRebSub = tNewSMqRebSubscribe(key);
     if (pRebSub == NULL) {
-      // TODO
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
     }
     taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe));
@@ -412,7 +415,11 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
   }
   if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
     mInfo("mq rebalance will be triggered");
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg)};
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_DO_REBALANCE,
+        .pCont = pRebMsg,
+        .contLen = sizeof(SMqDoRebalanceMsg),
+    };
     pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
   } else {
     taosHashCleanup(pRebMsg->rebSubHash);
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index 64c7a66bf9..d3642f4204 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -96,7 +96,11 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
   if (mndIsMaster(pMnode)) {
     int32_t contLen = 0;
     void   *pReq = mndBuildTimerMsg(&contLen);
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
+    SRpcMsg rpcMsg = {
+        .msgType = TDMT_MND_MQ_TIMER,
+        .pCont = pReq,
+        .contLen = contLen,
+    };
     pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
   }
 
@@ -631,4 +635,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
   }
 
   return 0;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp
index 17fda48cd7..9dbc1be4e9 100644
--- a/source/dnode/mnode/impl/test/db/db.cpp
+++ b/source/dnode/mnode/impl/test/db/db.cpp
@@ -292,9 +292,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
       EXPECT_GT(pInfo->vgId, 0);
       EXPECT_EQ(pInfo->hashBegin, 0);
       EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1);
-      EXPECT_EQ(pInfo->epset.inUse, 0);
-      EXPECT_EQ(pInfo->epset.numOfEps, 1);
-      SEp* pAddr = &pInfo->epset.eps[0];
+      EXPECT_EQ(pInfo->epSet.inUse, 0);
+      EXPECT_EQ(pInfo->epSet.numOfEps, 1);
+      SEp* pAddr = &pInfo->epSet.eps[0];
       EXPECT_EQ(pAddr->port, 9030);
       EXPECT_STREQ(pAddr->fqdn, "localhost");
     }
@@ -307,9 +307,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
       EXPECT_GT(pInfo->vgId, 0);
       EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2);
       EXPECT_EQ(pInfo->hashEnd, UINT32_MAX);
-      EXPECT_EQ(pInfo->epset.inUse, 0);
-      EXPECT_EQ(pInfo->epset.numOfEps, 1);
-      SEp* pAddr = &pInfo->epset.eps[0];
+      EXPECT_EQ(pInfo->epSet.inUse, 0);
+      EXPECT_EQ(pInfo->epSet.numOfEps, 1);
+      SEp* pAddr = &pInfo->epSet.eps[0];
       EXPECT_EQ(pAddr->port, 9030);
       EXPECT_STREQ(pAddr->fqdn, "localhost");
     }
diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h
index 36626514ec..b4d45e83fb 100644
--- a/source/dnode/vnode/inc/tq.h
+++ b/source/dnode/vnode/inc/tq.h
@@ -13,16 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TQ_H_
-#define _TD_TQ_H_
+#ifndef _TQ_H_
+#define _TQ_H_
 
-#include "tcommon.h"
 #include "executor.h"
-#include "tmallocator.h"
 #include "meta.h"
-#include "scheduler.h"
 #include "taoserror.h"
-#include "tlist.h"
+#include "tcommon.h"
+#include "tmallocator.h"
 #include "tmsg.h"
 #include "trpc.h"
 #include "ttimer.h"
@@ -54,7 +52,7 @@ void tqClose(STQ*);
 int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
-int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
 int32_t tqProcessRebReq(STQ* pTq, char* msg);
 
@@ -62,4 +60,4 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg);
 }
 #endif
 
-#endif /*_TD_TQ_H_*/
+#endif /*_TQ_H_*/
diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h
index f416413859..fc6a3699d5 100644
--- a/source/dnode/vnode/src/inc/tqInt.h
+++ b/source/dnode/vnode/src/inc/tqInt.h
@@ -19,7 +19,7 @@
 #include "meta.h"
 #include "tlog.h"
 #include "tq.h"
-#include "trpc.h"
+#include "tqPush.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -31,30 +31,35 @@ extern "C" {
       taosPrintLog("TQ  FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
     }                                                            \
   }
+
 #define tqError(...)                                             \
   {                                                              \
     if (tqDebugFlag & DEBUG_ERROR) {                             \
       taosPrintLog("TQ  ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
     }                                                            \
   }
+
 #define tqWarn(...)                                            \
   {                                                            \
     if (tqDebugFlag & DEBUG_WARN) {                            \
       taosPrintLog("TQ  WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
     }                                                          \
   }
+
 #define tqInfo(...)                                       \
   {                                                       \
     if (tqDebugFlag & DEBUG_INFO) {                       \
       taosPrintLog("TQ  ", DEBUG_INFO, 255, __VA_ARGS__); \
     }                                                     \
   }
+
 #define tqDebug(...)                                               \
   {                                                                \
     if (tqDebugFlag & DEBUG_DEBUG) {                               \
       taosPrintLog("TQ  ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
     }                                                              \
   }
+
 #define tqTrace(...)                                               \
   {                                                                \
     if (tqDebugFlag & DEBUG_TRACE) {                               \
@@ -138,9 +143,7 @@ typedef struct {
   // topics that are not connectted
   STqMetaList* unconnectTopic;
 
-  // TODO:temporaral use, to be replaced by unified tfile
   TdFilePtr pFile;
-  // TODO:temporaral use, to be replaced by unified tfile
   TdFilePtr pIdxFile;
 
   char*          dirPath;
@@ -157,6 +160,7 @@ struct STQ {
   STqCfg*       tqConfig;
   STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
+  STqPushMgr*   tqPushMgr;
   SWal*         pWal;
   SMeta*        pVnodeMeta;
 };
diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h
new file mode 100644
index 0000000000..32fd7c3ddf
--- /dev/null
+++ b/source/dnode/vnode/src/inc/tqPush.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TQ_PUSH_H_
+#define _TQ_PUSH_H_
+
+#include "thash.h"
+#include "trpc.h"
+#include "ttimer.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+  TQ_PUSHER_TYPE__CLIENT = 1,
+  TQ_PUSHER_TYPE__STREAM,
+};
+
+typedef struct {
+  int8_t   type;
+  int8_t   reserved[3];
+  int32_t  ttl;
+  int64_t  consumerId;
+  SRpcMsg* pMsg;
+  // SMqPollRsp* rsp;
+} STqClientPusher;
+
+typedef struct {
+  int8_t  type;
+  int8_t  nodeType;
+  int8_t  reserved[6];
+  int64_t streamId;
+  SEpSet  epSet;
+} STqStreamPusher;
+
+typedef struct {
+  int8_t type;  // mq or stream
+} STqPusher;
+
+typedef struct {
+  SHashObj* pHash;  // <id, STqPush*>
+} STqPushMgr;
+
+typedef struct {
+  int8_t inited;
+  tmr_h  timer;
+} STqPushMgmt;
+
+static STqPushMgmt tqPushMgmt;
+
+int32_t tqPushMgrInit();
+void    tqPushMgrCleanUp();
+
+STqPushMgr* tqPushMgrOpen();
+void        tqPushMgrClose(STqPushMgr* pushMgr);
+
+STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
+STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TQ_PUSH_H_*/
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index d540b589b7..16809f1527 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -12,28 +12,16 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
 
 #include "tcompare.h"
 #include "tqInt.h"
 #include "tqMetaStore.h"
 
-int tqInit() {
-  int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1);
-  if (old == 1) return 0;
+int32_t tqInit() { return tqPushMgrInit(); }
 
-  tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
-  return 0;
-}
+void tqCleanUp() { tqPushMgrCleanUp(); }
 
-void tqCleanUp() {
-  int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0);
-  if (old == 0) return;
-  taosTmrStop(tqMgmt.timer);
-  taosTmrCleanUp(tqMgmt.timer);
-}
-
-STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
+STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
   STQ* pTq = malloc(sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -42,7 +30,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
   pTq->path = strdup(path);
   pTq->tqConfig = tqConfig;
   pTq->pWal = pWal;
-  pTq->pVnodeMeta = pMeta;
+  pTq->pVnodeMeta = pVnodeMeta;
 #if 0
   pTq->tqMemRef.pAllocatorFactory = allocFac;
   pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
@@ -60,6 +48,13 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
     return NULL;
   }
 
+  pTq->tqPushMgr = tqPushMgrOpen();
+  if (pTq->tqPushMgr == NULL) {
+    // free store
+    free(pTq);
+    return NULL;
+  }
+
   return pTq;
 }
 
@@ -72,6 +67,8 @@ void tqClose(STQ* pTq) {
 }
 
 int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
+  // iterate hash
+  // process all msg
   // if waiting
   // memcpy and send msg to fetch thread
   // TODO: add reference
@@ -199,7 +196,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
     for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
       pTopic->buffer.output[j].status = 0;
       STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-      SReadHandle    handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
+      SReadHandle    handle = {
+             .reader = pReadHandle,
+             .meta = pTq->pVnodeMeta,
+      };
       pTopic->buffer.output[j].pReadHandle = pReadHandle;
       pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
     }
@@ -208,11 +208,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
   return 0;
 }
 
-int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
-  SMqConsumeReq* pReq = pMsg->pCont;
-  int64_t        consumerId = pReq->consumerId;
-  int64_t        fetchOffset;
-  int64_t        blockingTime = pReq->blockingTime;
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
+  SMqPollReq* pReq = pMsg->pCont;
+  int64_t     consumerId = pReq->consumerId;
+  int64_t     fetchOffset;
+  int64_t     blockingTime = pReq->blockingTime;
 
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
     fetchOffset = 0;
@@ -222,7 +222,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  SMqConsumeRsp rsp = {
+  SMqPollRsp rsp = {
       .consumerId = consumerId,
       .numOfTopics = 0,
       .pBlockData = NULL,
@@ -236,6 +236,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     rpcSendResponse(pMsg);
     return 0;
   }
+
   int sz = taosArrayGetSize(pConsumer->topics);
   ASSERT(sz == 1);
   STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
@@ -247,13 +248,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
 
   SWalHead* pHead;
   while (1) {
-    int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
+    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
     if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
       break;
     }
+    int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
     pHead = pTopic->pReadhandle->pHead;
     if (pHead->head.msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body;
@@ -280,7 +282,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
         rsp.numOfTopics = 1;
         rsp.pBlockData = pRes;
 
-        int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
+        int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
         void*   buf = rpcMallocCont(tlen);
         if (buf == NULL) {
           pMsg->code = -1;
@@ -290,7 +292,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
         ((SMqRspHead*)buf)->epoch = pReq->epoch;
 
         void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-        tEncodeSMqConsumeRsp(&abuf, &rsp);
+        tEncodeSMqPollRsp(&abuf, &rsp);
         taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
         pMsg->pCont = buf;
         pMsg->contLen = tlen;
@@ -304,7 +306,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
     }
   }
 
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
+  /*if (blockingTime != 0) {*/
+  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
+  /*} else {*/
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
@@ -314,12 +319,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqConsumeRsp(&abuf, &rsp);
+  tEncodeSMqPollRsp(&abuf, &rsp);
   rsp.pBlockData = NULL;
   pMsg->pCont = buf;
   pMsg->contLen = tlen;
   pMsg->code = 0;
   rpcSendResponse(pMsg);
+  /*}*/
+
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
new file mode 100644
index 0000000000..fea65846be
--- /dev/null
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tqPush.h"
+
+int32_t tqPushMgrInit() {
+  //
+  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
+  if (old == 1) return 0;
+
+  tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
+  return 0;
+}
+
+void tqPushMgrCleanUp() {
+  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
+  if (old == 0) return;
+  taosTmrStop(tqPushMgmt.timer);
+  taosTmrCleanUp(tqPushMgmt.timer);
+}
+
+STqPushMgr* tqPushMgrOpen() {
+  STqPushMgr* mgr = malloc(sizeof(STqPushMgr));
+  if (mgr == NULL) {
+    return NULL;
+  }
+  mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  return mgr;
+}
+
+void tqPushMgrClose(STqPushMgr* pushMgr) {
+  taosHashCleanup(pushMgr->pHash);
+  free(pushMgr);
+}
+
+STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
+  STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher));
+  if (clientPusher == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
+  clientPusher->pMsg = pMsg;
+  clientPusher->consumerId = consumerId;
+  clientPusher->ttl = ttl;
+  if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(clientPusher);
+    // TODO send rsp back
+    return NULL;
+  }
+  return clientPusher;
+}
+
+STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
+  STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher));
+  if (streamPusher == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  streamPusher->type = TQ_PUSHER_TYPE__STREAM;
+  streamPusher->nodeType = 0;
+  streamPusher->streamId = streamId;
+  memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));
+
+  if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(streamPusher);
+    return NULL;
+  }
+  return streamPusher;
+}
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 4b47413715..ccb59c26de 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -29,7 +29,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta};
 
   switch (pMsg->msgType) {
-    case TDMT_VND_QUERY:{
+    case TDMT_VND_QUERY: {
       return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg);
     }
     case TDMT_VND_QUERY_CONTINUE:
@@ -63,7 +63,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
     case TDMT_VND_TABLE_META:
       return vnodeGetTableMeta(pVnode, pMsg);
     case TDMT_VND_CONSUME:
-      return tqProcessConsumeReq(pVnode->pTq, pMsg);
+      return tqProcessPollReq(pVnode->pTq, pMsg);
     default:
       vError("unknown msg type:%d in fetch queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;
@@ -71,8 +71,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 }
 
 static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
-  STbCfg *        pTbCfg = NULL;
-  STbCfg *        pStbCfg = NULL;
+  STbCfg         *pTbCfg = NULL;
+  STbCfg         *pStbCfg = NULL;
   tb_uid_t        uid;
   int32_t         nCols;
   int32_t         nTagCols;
@@ -204,9 +204,9 @@ static void freeItemHelper(void *pItem) {
  */
 static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
   SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta);
-  SArray *    pArray = taosArrayInit(10, POINTER_BYTES);
+  SArray     *pArray = taosArrayInit(10, POINTER_BYTES);
 
-  char *  name = NULL;
+  char   *name = NULL;
   int32_t totalLen = 0;
   int32_t numOfTables = 0;
   while ((name = metaTbCursorNext(pCur)) != NULL) {
diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c
index 56c0ec1130..77d25fa164 100644
--- a/source/libs/catalog/src/catalog.c
+++ b/source/libs/catalog/src/catalog.c
@@ -845,7 +845,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg
   };
 
   SRpcMsg rpcRsp = {0};
-  rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp);
+  rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
   
   if (TSDB_CODE_SUCCESS != rpcRsp.code) {
     if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp
index aeac6b3eee..c7867c4da5 100644
--- a/source/libs/catalog/test/catalogTests.cpp
+++ b/source/libs/catalog/test/catalogTests.cpp
@@ -228,10 +228,10 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
     vgInfo.vgId = i + 1;
     vgInfo.hashBegin = i * hashUnit;
     vgInfo.hashEnd = hashUnit * (i + 1) - 1;
-    vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
-    vgInfo.epset.inUse = i % vgInfo.epset.numOfEps;
-    for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) {
-      SEp *addr = &vgInfo.epset.eps[n];
+    vgInfo.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1;
+    vgInfo.epSet.inUse = i % vgInfo.epSet.numOfEps;
+    for (int32_t n = 0; n < vgInfo.epSet.numOfEps; ++n) {
+      SEp *addr = &vgInfo.epSet.eps[n];
       strcpy(addr->fqdn, "a0");
       addr->port = n + 22;
     }
@@ -301,10 +301,10 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
       vg.hashEnd = htonl(UINT32_MAX);
     }
 
-    vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
-    vg.epset.inUse = i % vg.epset.numOfEps;
-    for (int32_t n = 0; n < vg.epset.numOfEps; ++n) {
-      SEp *addr = &vg.epset.eps[n];
+    vg.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1;
+    vg.epSet.inUse = i % vg.epSet.numOfEps;
+    for (int32_t n = 0; n < vg.epSet.numOfEps; ++n) {
+      SEp *addr = &vg.epSet.eps[n];
       strcpy(addr->fqdn, "a0");
       addr->port = n + 22;
     }
@@ -877,7 +877,7 @@ TEST(tableMeta, normalTable) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
     usleep(50000);
@@ -1384,7 +1384,7 @@ TEST(refreshGetMeta, normal2normal) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1463,7 +1463,7 @@ TEST(refreshGetMeta, normal2notexist) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1537,7 +1537,7 @@ TEST(refreshGetMeta, normal2child) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1621,7 +1621,7 @@ TEST(refreshGetMeta, stable2child) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1706,7 +1706,7 @@ TEST(refreshGetMeta, stable2stable) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1794,7 +1794,7 @@ TEST(refreshGetMeta, child2stable) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   while (true) {
     uint64_t n = 0;
@@ -1879,7 +1879,7 @@ TEST(tableDistVgroup, normalTable) {
   ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
   vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
   ASSERT_EQ(vgInfo->vgId, 8);
-  ASSERT_EQ(vgInfo->epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -1921,7 +1921,7 @@ TEST(tableDistVgroup, childTableCase) {
   ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
   vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
   ASSERT_EQ(vgInfo->vgId, 9);
-  ASSERT_EQ(vgInfo->epset.numOfEps, 4);
+  ASSERT_EQ(vgInfo->epSet.numOfEps, 4);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -1964,13 +1964,13 @@ TEST(tableDistVgroup, superTableCase) {
   ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10);
   vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
   ASSERT_EQ(vgInfo->vgId, 1);
-  ASSERT_EQ(vgInfo->epset.numOfEps, 1);
+  ASSERT_EQ(vgInfo->epSet.numOfEps, 1);
   vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1);
   ASSERT_EQ(vgInfo->vgId, 2);
-  ASSERT_EQ(vgInfo->epset.numOfEps, 2);
+  ASSERT_EQ(vgInfo->epSet.numOfEps, 2);
   vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2);
   ASSERT_EQ(vgInfo->vgId, 3);
-  ASSERT_EQ(vgInfo->epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -2025,14 +2025,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 3);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
   code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
   pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
   ASSERT_EQ(pvgInfo->vgId, 8);
-  ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
+  ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
   taosArrayDestroy(vgList);
 
   ctgTestBuildDBVgroup(&dbVgroup);
@@ -2053,14 +2053,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 7);
-  ASSERT_EQ(vgInfo.epset.numOfEps, 2);
+  ASSERT_EQ(vgInfo.epSet.numOfEps, 2);
 
   code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
   pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
   ASSERT_EQ(pvgInfo->vgId, 8);
-  ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
+  ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
   taosArrayDestroy(vgList);
 
   catalogDestroy();
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 3ebad151fd..eeb48a1f3d 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4977,7 +4977,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf
   SSourceDataInfo   *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex);
 
   qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
-         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);
+         GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);
 
   pMsg->header.vgId = htonl(pSource->addr.nodeId);
   pMsg->sId = htobe64(pSource->schedId);
@@ -5000,7 +5000,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf
   pMsgSendInfo->fp = loadRemoteDataCallback;
 
   int64_t transporterId = 0;
-  int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo);
+  int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c
index 713847807d..7ec2d05d44 100644
--- a/source/libs/parser/src/dCDAstProcess.c
+++ b/source/libs/parser/src/dCDAstProcess.c
@@ -58,7 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
 
     SVgroupInfo* info = taosArrayGet(array, 0);
     pShowReq->head.vgId = htonl(info->vgId);
-    *pEpSet = info->epset;
+    *pEpSet = info->epSet;
 
     *outputLen  = sizeof(SVShowTablesReq);
     *output     = pShowReq;
@@ -902,4 +902,4 @@ SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseCon
   }
 
   return pModifSqlStmt;
-}
\ No newline at end of file
+}
diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp
index cdb547ee1b..cd3ffcdce9 100644
--- a/source/libs/parser/test/mockCatalogService.cpp
+++ b/source/libs/parser/test/mockCatalogService.cpp
@@ -43,11 +43,11 @@ public:
 
     SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, };
 
-    vgroup.epset.eps[0] = (SEp){"dnode_1", 6030};
-    vgroup.epset.eps[1] = (SEp){"dnode_2", 6030};
-    vgroup.epset.eps[2] = (SEp){"dnode_3", 6030};
-    vgroup.epset.inUse = 0;
-    vgroup.epset.numOfEps = 3;
+    vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030};
+    vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030};
+    vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030};
+    vgroup.epSet.inUse = 0;
+    vgroup.epSet.numOfEps = 3;
 
     meta_->vgs.emplace_back(vgroup);
     return *this;
@@ -122,7 +122,7 @@ public:
   int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
     // todo
     vgInfo->vgId = 1;
-    addEpIntoEpSet(&vgInfo->epset, "node1", 6030);
+    addEpIntoEpSet(&vgInfo->epSet, "node1", 6030);
     return 0;
   }
 
@@ -143,10 +143,10 @@ public:
     meta_[db][tbname]->schema->uid = id_++;
 
     SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,};
-    addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030);
-    addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030);
-    addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030);
-    vgroup.epset.inUse = 0;
+    addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030);
+    addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030);
+    addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030);
+    vgroup.epSet.inUse = 0;
 
     meta_[db][tbname]->vgs.emplace_back(vgroup);
     // super table
@@ -313,4 +313,4 @@ int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableM
 
 int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
   return impl_->catalogGetTableHashVgroup(pTableName, vgInfo);
-}
\ No newline at end of file
+}
diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c
index 748be8fcd8..dbc0340b34 100644
--- a/source/libs/planner/src/physicalPlan.c
+++ b/source/libs/planner/src/physicalPlan.c
@@ -254,7 +254,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
 
 static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) {
   pNodeAddr->nodeId = vg->vgId;
-  pNodeAddr->epset  = vg->epset;
+  pNodeAddr->epSet  = vg->epSet;
 }
 
 static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) {
@@ -363,7 +363,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan
     SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY);
     SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i);
 
-    subplan->execNode.epset = blocks->vg.epset;
+    subplan->execNode.epSet = blocks->vg.epSet;
     subplan->pDataSink  = createDataInserter(pCxt, blocks, NULL);
     subplan->pNode      = NULL;
     subplan->type       = QUERY_TYPE_MODIFY;
diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c
index b2109c0a4f..f61102a66d 100644
--- a/source/libs/planner/src/physicalPlanJson.c
+++ b/source/libs/planner/src/physicalPlanJson.c
@@ -762,11 +762,11 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) {
   bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
 
   if (res) {
-    res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse);
+    res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epSet.inUse);
   }
 
   if (res) {
-    res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps);
+    res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epSet.eps, sizeof(SEp), pAddr->epSet.numOfEps);
   }
   return res;
 }
@@ -775,11 +775,11 @@ static bool queryNodeAddrFromJson(const cJSON* json, void* obj) {
   SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj;
 
   pAddr->nodeId = getNumber(json, jkNodeAddrId);
-  pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse);
+  pAddr->epSet.inUse = getNumber(json, jkNodeAddrInUse);
 
   int32_t numOfEps = 0;
-  bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps);
-  pAddr->epset.numOfEps = numOfEps;
+  bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps);
+  pAddr->epSet.numOfEps = numOfEps;
   return res;
 }
 
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index f1ed0cef7d..8ed39eb0b7 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -423,13 +423,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
     SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
-  if (pTask->plan->execNode.epset.numOfEps > 0) {
+  if (pTask->plan->execNode.epSet.numOfEps > 0) {
     if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) {
       SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno);
       SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-    SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps);
+    SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
 
     return TSDB_CODE_SUCCESS;
   }
@@ -1061,7 +1061,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
     isCandidateAddr = true;
   }
 
-  SEpSet epSet = addr->epset;
+  SEpSet epSet = addr->epSet;
 
   switch (msgType) {
     case TDMT_VND_CREATE_TABLE:
diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp
index daf65cf251..8ed963d875 100644
--- a/source/libs/scheduler/test/schedulerTests.cpp
+++ b/source/libs/scheduler/test/schedulerTests.cpp
@@ -104,8 +104,8 @@ void schtBuildQueryDag(SQueryDag *dag) {
   scanPlan->type = QUERY_TYPE_SCAN;
 
   scanPlan->execNode.nodeId = 1;
-  scanPlan->execNode.epset.inUse = 0;
-  addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030);
+  scanPlan->execNode.epSet.inUse = 0;
+  addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030);
 
   scanPlan->pChildren = NULL;
   scanPlan->level = 1;
@@ -118,7 +118,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
   mergePlan->id.subplanId = 0x5555555555;
   mergePlan->type = QUERY_TYPE_MERGE;
   mergePlan->level = 0;
-  mergePlan->execNode.epset.numOfEps = 0;
+  mergePlan->execNode.epSet.numOfEps = 0;
 
   mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES);
   mergePlan->pParents = NULL;
@@ -157,8 +157,8 @@ void schtBuildInsertDag(SQueryDag *dag) {
   insertPlan[0].level = 0;
 
   insertPlan[0].execNode.nodeId = 1;
-  insertPlan[0].execNode.epset.inUse = 0;
-  addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030);
+  insertPlan[0].execNode.epSet.inUse = 0;
+  addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030);
 
   insertPlan[0].pChildren = NULL;
   insertPlan[0].pParents = NULL;
@@ -173,8 +173,8 @@ void schtBuildInsertDag(SQueryDag *dag) {
   insertPlan[1].level = 0;
 
   insertPlan[1].execNode.nodeId = 1;
-  insertPlan[1].execNode.epset.inUse = 0;
-  addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030);
+  insertPlan[1].execNode.epSet.inUse = 0;
+  addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030);
 
   insertPlan[1].pChildren = NULL;
   insertPlan[1].pParents = NULL;
diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c
index 3fe0425c91..3eb8e60d56 100644
--- a/tests/test/c/tmqDemo.c
+++ b/tests/test/c/tmqDemo.c
@@ -352,7 +352,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
     if (tmqmessage) {
       cnt++;
       msg_process(tmqmessage);
@@ -383,7 +383,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   }
 
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
     if (tmqmessage) {
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
@@ -411,7 +411,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog
   int32_t skipLogNum = 0;
   int64_t startTime = taosGetTimestampUs();
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
+    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
     if (tmqmessage) {
       batchCnt++;
       skipLogNum += tmqGetSkipLogNum(tmqmessage);

From 8447cea613ed90a6f63016ff96ba864b641da2b6 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 8 Mar 2022 17:33:27 +0800
Subject: [PATCH 12/29] disable destory

---
 source/dnode/mnode/impl/src/mndScheduler.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index dfde6a9258..6b81852545 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -85,7 +85,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     taosArrayPush(pSub->unassignedVg, &consumerEp);
   }
 
-  qDestroyQueryDag(pDag);
+  /*qDestroyQueryDag(pDag);*/
 
   return 0;
 }

From cc24ae769ca6ddd4a55c9db5aeb3084be07cfe38 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Tue, 8 Mar 2022 17:38:10 +0800
Subject: [PATCH 13/29] fix crash

---
 source/dnode/mnode/impl/src/mndScheduler.c |   4 +-
 source/libs/planner/src/physicalPlanJson.c | 209 +++++++++++----------
 2 files changed, 115 insertions(+), 98 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 6b81852545..ce288d33f7 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -47,7 +47,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     return -1;
   }
 
-  SArray* plans = taosArrayGet(pDag->pSubplans, 0);
+  SArray* plans = taosArrayGetP(pDag->pSubplans, 0);
 
   int32_t opNum = taosArrayGetSize(plans);
   if (opNum != 1) {
@@ -85,7 +85,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     taosArrayPush(pSub->unassignedVg, &consumerEp);
   }
 
-  /*qDestroyQueryDag(pDag);*/
+  qDestroyQueryDag(pDag);
 
   return 0;
 }
diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c
index f61102a66d..4018741091 100644
--- a/source/libs/planner/src/physicalPlanJson.c
+++ b/source/libs/planner/src/physicalPlanJson.c
@@ -13,12 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "plannerInt.h"
-#include "parser.h"
 #include "cJSON.h"
+#include "parser.h"
+#include "plannerInt.h"
 
-typedef bool (*FToJson)(const void* obj, cJSON* json); 
-typedef bool (*FFromJson)(const cJSON* json, void* obj); 
+typedef bool (*FToJson)(const void* obj, cJSON* json);
+typedef bool (*FFromJson)(const cJSON* json, void* obj);
 
 static char* getString(const cJSON* json, const char* name) {
   char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name));
@@ -30,7 +30,7 @@ static void copyString(const cJSON* json, const char* name, char* dst) {
 }
 
 static uint64_t getBigintFromString(const cJSON* json, const char* name) {
-  char* val = getString(json, name);
+  char*    val = getString(json, name);
   uint64_t intVal = strtoul(val, NULL, 10);
   tfree(val);
 
@@ -39,7 +39,7 @@ static uint64_t getBigintFromString(const cJSON* json, const char* name) {
 
 static int64_t getNumber(const cJSON* json, const char* name) {
   double d = cJSON_GetNumberValue(cJSON_GetObjectItem(json, name));
-  return (int64_t) d;
+  return (int64_t)d;
 }
 
 static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) {
@@ -72,7 +72,8 @@ static bool fromObject(const cJSON* json, const char* name, FFromJson func, void
   return func(jObj, obj);
 }
 
-static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) {
+static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size,
+                                bool required) {
   cJSON* jObj = cJSON_GetObjectItem(json, name);
   if (NULL == jObj) {
     return !required;
@@ -85,7 +86,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f
 }
 
 static const char* jkPnodeType = "Type";
-static int32_t getPnodeTypeSize(cJSON* json) {
+static int32_t     getPnodeTypeSize(cJSON* json) {
   switch (getNumber(json, jkPnodeType)) {
     case OP_StreamScan:
     case OP_TableScan:
@@ -119,7 +120,7 @@ static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void*
 
 static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func, SArray** array) {
   const cJSON* jArray = cJSON_GetObjectItem(json, name);
-  int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
+  int32_t      size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
   if (size > 0) {
     *array = taosArrayInit(size, POINTER_BYTES);
     if (NULL == *array) {
@@ -128,7 +129,7 @@ static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func,
   }
   for (int32_t i = 0; i < size; ++i) {
     cJSON* jItem = cJSON_GetArrayItem(jArray, i);
-    void* item = calloc(1, getPnodeTypeSize(jItem));
+    void*  item = calloc(1, getPnodeTypeSize(jItem));
     if (NULL == item || !func(jItem, item)) {
       return false;
     }
@@ -161,9 +162,10 @@ static bool addArray(cJSON* json, const char* name, FToJson func, const SArray*
   return addTarray(json, name, func, array, true);
 }
 
-static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize, bool isPoint) {
+static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize,
+                       bool isPoint) {
   const cJSON* jArray = cJSON_GetObjectItem(json, name);
-  int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
+  int32_t      size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
   if (size > 0) {
     *array = taosArrayInit(size, isPoint ? POINTER_BYTES : itemSize);
     if (NULL == *array) {
@@ -188,7 +190,8 @@ static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArra
   return fromTarray(json, name, func, array, itemSize, true);
 }
 
-static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) {
+static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize,
+                        int32_t size) {
   if (size > 0) {
     cJSON* jArray = cJSON_AddArrayToObject(json, name);
     if (NULL == jArray) {
@@ -218,7 +221,8 @@ static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t i
   return true;
 }
 
-static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) {
+static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize,
+                                  int32_t* size) {
   const cJSON* jArray = getArray(json, name, size);
   if (*size > 0) {
     *array = calloc(1, itemSize * (*size));
@@ -229,7 +233,8 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson
   return fromItem(jArray, func, *array, itemSize, *size);
 }
 
-static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) {
+static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize,
+                         int32_t* size) {
   const cJSON* jArray = getArray(json, name, size);
   return fromItem(jArray, func, array, itemSize, *size);
 }
@@ -240,7 +245,7 @@ static const char* jkSchemaBytes = "Bytes";
 // The 'name' field do not need to be serialized.
 static bool schemaToJson(const void* obj, cJSON* jSchema) {
   const SSlotSchema* schema = (const SSlotSchema*)obj;
-  bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type);
+  bool               res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type);
   if (res) {
     res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId);
   }
@@ -264,7 +269,8 @@ static const char* jkDataBlockSchemaPrecision = "Precision";
 
 static bool dataBlockSchemaToJson(const void* obj, cJSON* json) {
   const SDataBlockSchema* schema = (const SDataBlockSchema*)obj;
-  bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema), schema->numOfCols);
+  bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema),
+                         schema->numOfCols);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkDataBlockSchemaResultRowSize, schema->resultRowSize);
   }
@@ -279,7 +285,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) {
   schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize);
   schema->precision = getNumber(json, jkDataBlockSchemaPrecision);
 
-  return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols);
+  return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema),
+                               sizeof(SSlotSchema), &schema->numOfCols);
 }
 
 static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr";
@@ -290,7 +297,7 @@ static const char* jkColumnFilterInfoUpperBnd = "UpperBnd";
 
 static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) {
   const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj;
-  bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr);
+  bool                     res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr);
   if (res) {
     res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr);
   }
@@ -323,7 +330,7 @@ static const char* jkColumnInfoFilterList = "FilterList";
 
 static bool columnInfoToJson(const void* obj, cJSON* jCol) {
   const SColumnInfo* col = (const SColumnInfo*)obj;
-  bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId);
+  bool               res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId);
   if (res) {
     res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type);
   }
@@ -331,8 +338,9 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) {
     res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes);
   }
 
-  if (res) { // TODO: temporarily disable it
-//    res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters);
+  if (res) {  // TODO: temporarily disable it
+              //    res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo,
+              //    sizeof(SColumnFilterInfo), col->flist.numOfFilters);
   }
 
   return res;
@@ -341,10 +349,11 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) {
 static bool columnInfoFromJson(const cJSON* json, void* obj) {
   SColumnInfo* col = (SColumnInfo*)obj;
   col->colId = getNumber(json, jkColumnInfoColId);
-  col->type  = getNumber(json, jkColumnInfoType);
+  col->type = getNumber(json, jkColumnInfoType);
   col->bytes = getNumber(json, jkColumnInfoBytes);
   int32_t size = 0;
-  bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size);
+  bool    res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson,
+                                      (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size);
   col->flist.numOfFilters = size;
   return res;
 }
@@ -355,7 +364,7 @@ static const char* jkColumnInfo = "Info";
 
 static bool columnToJson(const void* obj, cJSON* jCol) {
   const SColumn* col = (const SColumn*)obj;
-  bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid);
+  bool           res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid);
   if (res) {
     res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag);
   }
@@ -381,7 +390,7 @@ static const char* jkExprNodeRight = "Right";
 
 static bool operatorToJson(const void* obj, cJSON* jOper) {
   const tExprNode* exprInfo = (const tExprNode*)obj;
-  bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr);
+  bool             res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr);
   if (res) {
     res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft);
   }
@@ -406,9 +415,10 @@ static const char* jkFunctionChild = "Child";
 
 static bool functionToJson(const void* obj, cJSON* jFunc) {
   const tExprNode* exprInfo = (const tExprNode*)obj;
-  bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName);
+  bool             res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName);
   if (res && NULL != exprInfo->_function.pChild) {
-      res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num);
+    res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*),
+                      exprInfo->_function.num);
   }
   return res;
 }
@@ -420,7 +430,8 @@ static bool functionFromJson(const cJSON* json, void* obj) {
   if (NULL == exprInfo->_function.pChild) {
     return false;
   }
-  return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num);
+  return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild,
+                               sizeof(tExprNode*), &exprInfo->_function.num);
 }
 
 static const char* jkVariantType = "Type";
@@ -430,12 +441,12 @@ static const char* jkVariantValue = "Value";
 
 static bool variantToJson(const void* obj, cJSON* jVar) {
   const SVariant* var = (const SVariant*)obj;
-  bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType);
+  bool            res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType);
   if (res) {
     res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen);
   }
   if (res) {
-    if (0/* in */) {
+    if (0 /* in */) {
       res = addArray(jVar, jkVariantvalues, variantToJson, var->arr);
     } else if (IS_NUMERIC_TYPE(var->nType)) {
       res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d);
@@ -450,7 +461,7 @@ static bool variantFromJson(const cJSON* json, void* obj) {
   SVariant* var = (SVariant*)obj;
   var->nType = getNumber(json, jkVariantType);
   var->nLen = getNumber(json, jkVariantLen);
-  if (0/* in */) {
+  if (0 /* in */) {
     return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant));
   } else if (IS_NUMERIC_TYPE(var->nType)) {
     var->d = getNumber(json, jkVariantValue);
@@ -468,7 +479,7 @@ static const char* jkExprNodeValue = "Value";
 
 static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) {
   const tExprNode* exprInfo = *(const tExprNode**)obj;
-  bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType);
+  bool             res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType);
   if (res) {
     switch (exprInfo->nodeType) {
       case TEXPR_BINARYEXPR_NODE:
@@ -502,7 +513,8 @@ static bool exprNodeFromJson(const cJSON* json, void* obj) {
     case TEXPR_FUNCTION_NODE:
       return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false);
     case TEXPR_COL_NODE:
-      return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema), false);
+      return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema),
+                                 false);
     case TEXPR_VALUE_NODE:
       return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false);
     default:
@@ -518,7 +530,7 @@ static const char* jkSqlExprParams = "Params";
 // token does not need to be serialized.
 static bool sqlExprToJson(const void* obj, cJSON* jExpr) {
   const SSqlExpr* expr = (const SSqlExpr*)obj;
-  bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema);
+  bool            res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema);
   if (res) {
     res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols);
   }
@@ -533,9 +545,10 @@ static bool sqlExprToJson(const void* obj, cJSON* jExpr) {
 
 static bool sqlExprFromJson(const cJSON* json, void* obj) {
   SSqlExpr* expr = (SSqlExpr*)obj;
-  bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false);
+  bool      res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false);
   if (res) {
-    res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols);
+    res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn),
+                                &expr->numOfCols);
   }
   if (res) {
     expr->interBytes = getNumber(json, jkSqlExprInterBytes);
@@ -553,7 +566,7 @@ static const char* jkExprInfoExpr = "Expr";
 
 static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) {
   const SExprInfo* exprInfo = (const SExprInfo*)obj;
-  bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base);
+  bool             res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base);
   if (res) {
     res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, &exprInfo->pExpr);
   }
@@ -562,9 +575,10 @@ static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) {
 
 static bool exprInfoFromJson(const cJSON* json, void* obj) {
   SExprInfo* exprInfo = (SExprInfo*)obj;
-  bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true);
+  bool       res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true);
   if (res) {
-    res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true);
+    res =
+        fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true);
   }
   return res;
 }
@@ -576,12 +590,12 @@ static bool timeWindowToJson(const void* obj, cJSON* json) {
   const STimeWindow* win = (const STimeWindow*)obj;
 
   char tmp[40] = {0};
-  snprintf(tmp, tListLen(tmp),"%"PRId64, win->skey);
+  snprintf(tmp, tListLen(tmp), "%" PRId64, win->skey);
 
   bool res = cJSON_AddStringToObject(json, jkTimeWindowStartKey, tmp);
   if (res) {
     memset(tmp, 0, tListLen(tmp));
-    snprintf(tmp, tListLen(tmp),"%"PRId64, win->ekey);
+    snprintf(tmp, tListLen(tmp), "%" PRId64, win->ekey);
     res = cJSON_AddStringToObject(json, jkTimeWindowEndKey, tmp);
   }
   return res;
@@ -604,7 +618,7 @@ static bool scanNodeToJson(const void* obj, cJSON* json) {
   const SScanPhyNode* pNode = (const SScanPhyNode*)obj;
 
   char uid[40] = {0};
-  snprintf(uid, tListLen(uid), "%"PRIu64, pNode->uid);
+  snprintf(uid, tListLen(uid), "%" PRIu64, pNode->uid);
   bool res = cJSON_AddStringToObject(json, jkScanNodeTableId, uid);
 
   if (res) {
@@ -629,11 +643,11 @@ static bool scanNodeToJson(const void* obj, cJSON* json) {
 static bool scanNodeFromJson(const cJSON* json, void* obj) {
   SScanPhyNode* pNode = (SScanPhyNode*)obj;
 
-  pNode->uid       = getBigintFromString(json, jkScanNodeTableId);
+  pNode->uid = getBigintFromString(json, jkScanNodeTableId);
   pNode->tableType = getNumber(json, jkScanNodeTableType);
-  pNode->count     = getNumber(json, jkScanNodeTableCount);
-  pNode->order     = getNumber(json, jkScanNodeTableOrder);
-  pNode->reverse   = getNumber(json, jkScanNodeTableRevCount);
+  pNode->count = getNumber(json, jkScanNodeTableCount);
+  pNode->order = getNumber(json, jkScanNodeTableOrder);
+  pNode->reverse = getNumber(json, jkScanNodeTableRevCount);
   return true;
 }
 
@@ -644,7 +658,7 @@ static const char* jkColIndexName = "Name";
 
 static bool colIndexToJson(const void* obj, cJSON* json) {
   const SColIndex* col = (const SColIndex*)obj;
-  bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId);
+  bool             res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex);
   }
@@ -673,7 +687,7 @@ static const char* jkAggNodeGroupByList = "GroupByList";
 
 static bool aggNodeToJson(const void* obj, cJSON* json) {
   const SAggPhyNode* agg = (const SAggPhyNode*)obj;
-  bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo);
+  bool               res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit);
   }
@@ -703,7 +717,7 @@ static const char* jkTableScanNodeTagsConditions = "TagsConditions";
 
 static bool tableScanNodeToJson(const void* obj, cJSON* json) {
   const STableScanPhyNode* scan = (const STableScanPhyNode*)obj;
-  bool res = scanNodeToJson(obj, json);
+  bool                     res = scanNodeToJson(obj, json);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag);
   }
@@ -718,7 +732,7 @@ static bool tableScanNodeToJson(const void* obj, cJSON* json) {
 
 static bool tableScanNodeFromJson(const cJSON* json, void* obj) {
   STableScanPhyNode* scan = (STableScanPhyNode*)obj;
-  bool res = scanNodeFromJson(json, obj);
+  bool               res = scanNodeFromJson(json, obj);
   if (res) {
     scan->scanFlag = getNumber(json, jkTableScanNodeFlag);
   }
@@ -736,7 +750,7 @@ static const char* jkEpAddrPort = "Port";
 
 static bool epAddrToJson(const void* obj, cJSON* json) {
   const SEp* ep = (const SEp*)obj;
-  bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
+  bool       res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port);
   }
@@ -750,16 +764,16 @@ static bool epAddrFromJson(const cJSON* json, void* obj) {
   return true;
 }
 
-static const char* jkNodeAddrId      = "NodeId";
-static const char* jkNodeAddrInUse   = "InUse";
+static const char* jkNodeAddrId = "NodeId";
+static const char* jkNodeAddrInUse = "InUse";
 static const char* jkNodeAddrEpAddrs = "Ep";
-static const char* jkNodeAddr        = "NodeAddr";
-static const char* jkNodeTaskId      = "TaskId";
+static const char* jkNodeAddr = "NodeAddr";
+static const char* jkNodeTaskId = "TaskId";
 static const char* jkNodeTaskSchedId = "SchedId";
 
 static bool queryNodeAddrToJson(const void* obj, cJSON* json) {
-  const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*) obj;
-  bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
+  const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*)obj;
+  bool                  res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
 
   if (res) {
     res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epSet.inUse);
@@ -772,24 +786,24 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) {
 }
 
 static bool queryNodeAddrFromJson(const cJSON* json, void* obj) {
-  SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj;
+  SQueryNodeAddr* pAddr = (SQueryNodeAddr*)obj;
 
   pAddr->nodeId = getNumber(json, jkNodeAddrId);
   pAddr->epSet.inUse = getNumber(json, jkNodeAddrInUse);
 
   int32_t numOfEps = 0;
-  bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps);
+  bool    res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps);
   pAddr->epSet.numOfEps = numOfEps;
   return res;
 }
 
 static bool nodeAddrToJson(const void* obj, cJSON* json) {
-  const SDownstreamSource* pSource = (const SDownstreamSource*) obj;
-  bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId);
+  const SDownstreamSource* pSource = (const SDownstreamSource*)obj;
+  bool                     res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId);
 
   if (res) {
     char t[30] = {0};
-    snprintf(t, tListLen(t), "%"PRIu64, pSource->schedId);
+    snprintf(t, tListLen(t), "%" PRIu64, pSource->schedId);
     res = cJSON_AddStringToObject(json, jkNodeTaskSchedId, t);
   }
 
@@ -813,9 +827,10 @@ static const char* jkExchangeNodeSrcEndPoints = "SrcAddrs";
 
 static bool exchangeNodeToJson(const void* obj, cJSON* json) {
   const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj;
-  bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId);
+  bool                    res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId);
   if (res) {
-    res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData, sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints));
+    res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData,
+                      sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints));
   }
   return res;
 }
@@ -823,7 +838,8 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) {
 static bool exchangeNodeFromJson(const cJSON* json, void* obj) {
   SExchangePhyNode* exchange = (SExchangePhyNode*)obj;
   exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId);
-  return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SDownstreamSource));
+  return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints,
+                         sizeof(SDownstreamSource));
 }
 
 static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
@@ -855,7 +871,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
     case OP_AllTimeWindow:
     case OP_AllMultiTableTimeInterval:
     case OP_Order:
-      break; // todo
+      break;  // todo
     case OP_Exchange:
       return exchangeNodeToJson(obj, json);
     default:
@@ -893,7 +909,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
     case OP_AllTimeWindow:
     case OP_AllMultiTableTimeInterval:
     case OP_Order:
-      break; // todo
+      break;  // todo
     case OP_Exchange:
       return exchangeNodeFromJson(json, obj);
     default:
@@ -910,7 +926,7 @@ static const char* jkPnodeChildren = "Children";
 // The 'pParent' field do not need to be serialized.
 static bool phyNodeToJson(const void* obj, cJSON* jNode) {
   const SPhyNode* phyNode = (const SPhyNode*)obj;
-  bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type);
+  bool            res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type);
   if (res) {
     res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name);
   }
@@ -933,7 +949,7 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) {
 }
 
 static bool phyNodeFromJson(const cJSON* json, void* obj) {
-  SPhyNode* node = (SPhyNode*) obj;
+  SPhyNode* node = (SPhyNode*)obj;
 
   node->info.type = getNumber(json, jkPnodeType);
   node->info.name = opTypeToOpName(node->info.type);
@@ -959,7 +975,7 @@ static const char* jkInserterDataSize = "DataSize";
 
 static bool inserterToJson(const void* obj, cJSON* json) {
   const SDataInserter* inserter = (const SDataInserter*)obj;
-  bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables);
+  bool                 res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables);
   if (res) {
     res = cJSON_AddNumberToObject(json, jkInserterDataSize, inserter->size);
   }
@@ -1005,7 +1021,7 @@ static const char* jkDataSinkSchema = "Schema";
 
 static bool dataSinkToJson(const void* obj, cJSON* json) {
   const SDataSink* dsink = (const SDataSink*)obj;
-  bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name);
+  bool             res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name);
   if (res) {
     res = addObject(json, dsink->info.name, specificDataSinkToJson, dsink);
   }
@@ -1034,7 +1050,7 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) {
   const SSubplanId* id = (const SSubplanId*)obj;
 
   char ids[40] = {0};
-  snprintf(ids, tListLen(ids), "%"PRIu64, id->queryId);
+  snprintf(ids, tListLen(ids), "%" PRIu64, id->queryId);
 
   bool res = cJSON_AddStringToObject(jId, jkIdQueryId, ids);
   if (res) {
@@ -1049,9 +1065,9 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) {
 static bool subplanIdFromJson(const cJSON* json, void* obj) {
   SSubplanId* id = (SSubplanId*)obj;
 
-  id->queryId    = getBigintFromString(json, jkIdQueryId);
+  id->queryId = getBigintFromString(json, jkIdQueryId);
   id->templateId = getNumber(json, jkIdTemplateId);
-  id->subplanId  = getNumber(json, jkIdSubplanId);
+  id->subplanId = getNumber(json, jkIdSubplanId);
   return true;
 }
 
@@ -1094,9 +1110,10 @@ static SSubplan* subplanFromJson(const cJSON* json) {
   }
 
   if (res) {
-    res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false);
+    res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink),
+                              false);
   }
-  
+
   if (!res) {
     qDestroySubplan(subplan);
     return NULL;
@@ -1137,15 +1154,15 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) {
 
 cJSON* qDagToJson(const SQueryDag* pDag) {
   cJSON* pRoot = cJSON_CreateObject();
-  if(pRoot == NULL) {
+  if (pRoot == NULL) {
     return NULL;
   }
 
   cJSON_AddNumberToObject(pRoot, "Number", pDag->numOfSubplans);
   cJSON_AddNumberToObject(pRoot, "QueryId", pDag->queryId);
 
-  cJSON *pLevels = cJSON_CreateArray();
-  if(pLevels == NULL) {
+  cJSON* pLevels = cJSON_CreateArray();
+  if (pLevels == NULL) {
     cJSON_Delete(pRoot);
     return NULL;
   }
@@ -1153,19 +1170,19 @@ cJSON* qDagToJson(const SQueryDag* pDag) {
   cJSON_AddItemToObject(pRoot, "Subplans", pLevels);
 
   size_t level = taosArrayGetSize(pDag->pSubplans);
-  for(size_t i = 0; i < level; i++) {
+  for (size_t i = 0; i < level; i++) {
     const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i);
-    size_t num = taosArrayGetSize(pSubplans);
-    cJSON* plansOneLevel = cJSON_CreateArray();
-    if(plansOneLevel == NULL) {
+    size_t        num = taosArrayGetSize(pSubplans);
+    cJSON*        plansOneLevel = cJSON_CreateArray();
+    if (plansOneLevel == NULL) {
       cJSON_Delete(pRoot);
       return NULL;
     }
 
     cJSON_AddItemToArray(pLevels, plansOneLevel);
-    for(size_t j = 0; j < num; j++) {
+    for (size_t j = 0; j < num; j++) {
       cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j));
-      if(pSubplan == NULL) {
+      if (pSubplan == NULL) {
         cJSON_Delete(pRoot);
         return NULL;
       }
@@ -1183,22 +1200,22 @@ char* qDagToString(const SQueryDag* pDag) {
 
 SQueryDag* qJsonToDag(const cJSON* pRoot) {
   SQueryDag* pDag = malloc(sizeof(SQueryDag));
-  if(pDag == NULL) {
+  if (pDag == NULL) {
     return NULL;
   }
   pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number"));
   pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId"));
-  pDag->pSubplans = taosArrayInit(0, sizeof(SArray));
+  pDag->pSubplans = taosArrayInit(0, sizeof(void*));
   if (pDag->pSubplans == NULL) {
     free(pDag);
     return NULL;
   }
   cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans");
-  int level = cJSON_GetArraySize(pLevels);
-  for(int i = 0; i < level; i++) {
+  int    level = cJSON_GetArraySize(pLevels);
+  for (int i = 0; i < level; i++) {
     SArray* plansOneLevel = taosArrayInit(0, sizeof(void*));
-    if(plansOneLevel == NULL) {
-      for(int j = 0; j < i; j++) {
+    if (plansOneLevel == NULL) {
+      for (int j = 0; j < i; j++) {
         taosArrayDestroy(taosArrayGetP(pDag->pSubplans, j));
       }
       taosArrayDestroy(pDag->pSubplans);
@@ -1206,13 +1223,13 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) {
       return NULL;
     }
     cJSON* pItem = cJSON_GetArrayItem(pLevels, i);
-    int sz = cJSON_GetArraySize(pItem);
-    for(int j = 0; j < sz; j++) {
-      cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j);
+    int    sz = cJSON_GetArraySize(pItem);
+    for (int j = 0; j < sz; j++) {
+      cJSON*    pSubplanJson = cJSON_GetArrayItem(pItem, j);
       SSubplan* pSubplan = subplanFromJson(pSubplanJson);
       taosArrayPush(plansOneLevel, &pSubplan);
     }
-    taosArrayPush(pDag->pSubplans, plansOneLevel);
+    taosArrayPush(pDag->pSubplans, &plansOneLevel);
   }
   return pDag;
 }

From d958655dd3221c14356b26cae1c3b8e3e6826186 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 20:22:31 +0800
Subject: [PATCH 14/29] sync refactor

---
 source/libs/sync/inc/syncInt.h                |   1 +
 source/libs/sync/inc/syncUtil.h               |   1 +
 source/libs/sync/src/syncMain.c               |   7 +
 source/libs/sync/src/syncUtil.c               |   7 +
 source/libs/sync/src/syncVoteMgr.c            |  14 +-
 source/libs/sync/test/syncInitTest.cpp        |  71 ++++++----
 source/libs/sync/test/syncPingTest.cpp        |   3 +-
 .../libs/sync/test/syncVotesGrantedTest.cpp   | 130 ++++++++++++++----
 8 files changed, 182 insertions(+), 52 deletions(-)

diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 874763cd45..447b75a5e8 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -193,6 +193,7 @@ typedef struct SSyncNode {
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
 void       syncNodeClose(SSyncNode* pSyncNode);
 cJSON*     syncNode2Json(const SSyncNode* pSyncNode);
+char*      syncNode2Str(const SSyncNode* pSyncNode);
 
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
 int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h
index 6cb1105b2c..f2c979da99 100644
--- a/source/libs/sync/inc/syncUtil.h
+++ b/source/libs/sync/inc/syncUtil.h
@@ -47,6 +47,7 @@ int32_t     syncUtilElectRandomMS();
 int32_t     syncUtilQuorum(int32_t replicaNum);
 cJSON*      syncUtilNodeInfo2Json(const SNodeInfo* p);
 cJSON*      syncUtilRaftId2Json(const SRaftId* p);
+char*       syncUtilRaftId2Str(const SRaftId* p);
 const char* syncUtilState2String(ESyncState state);
 
 #ifdef __cplusplus
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index ad424b9353..3b8d716dbe 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -291,6 +291,13 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
   return pJson;
 }
 
+char* syncNode2Str(const SSyncNode* pSyncNode) {
+  cJSON* pJson = syncNode2Json(pSyncNode);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
 int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
   SEpSet epSet;
   syncUtilraftId2EpSet(destRaftId, &epSet);
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index d2b413bdaa..3fb430a714 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -131,6 +131,13 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) {
   return pJson;
 }
 
+char* syncUtilRaftId2Str(const SRaftId* p) {
+  cJSON* pJson = syncUtilRaftId2Json(p);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
 const char* syncUtilState2String(ESyncState state) {
   if (state == TAOS_SYNC_STATE_FOLLOWER) {
     return "TAOS_SYNC_STATE_FOLLOWER";
diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c
index 407b7b1941..3b6105ea0a 100644
--- a/source/libs/sync/src/syncVoteMgr.c
+++ b/source/libs/sync/src/syncVoteMgr.c
@@ -63,6 +63,7 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) {
     }
   }
   assert(j != -1);
+  assert(j >= 0 && j < pVotesGranted->replicaNum);
 
   if (pVotesGranted->isGranted[j] != true) {
     ++(pVotesGranted->votes);
@@ -87,6 +88,14 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
   for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
     cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i])));
   }
+  int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum);
+  for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
+    arr[i] = pVotesGranted->isGranted[i];
+  }
+  cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum);
+  free(arr);
+  cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted);
+
   cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes);
   snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term);
   cJSON_AddStringToObject(pRoot, "term", u64buf);
@@ -95,6 +104,9 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
   snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode);
   cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
 
+  bool majority = voteGrantedMajority(pVotesGranted);
+  cJSON_AddNumberToObject(pRoot, "majority", majority);
+
   cJSON *pJson = cJSON_CreateObject();
   cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot);
   return pJson;
@@ -102,7 +114,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
 
 char *voteGranted2Str(SVotesGranted *pVotesGranted) {
   cJSON *pJson = voteGranted2Json(pVotesGranted);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp
index 602297b2a6..669c4e68a5 100644
--- a/source/libs/sync/test/syncInitTest.cpp
+++ b/source/libs/sync/test/syncInitTest.cpp
@@ -1,8 +1,10 @@
 #include <gtest/gtest.h>
 #include <stdio.h>
+#include "syncEnv.h"
 #include "syncIO.h"
 #include "syncInt.h"
 #include "syncRaftStore.h"
+#include "syncUtil.h"
 
 void logTest() {
   sTrace("--- sync log test: trace");
@@ -13,13 +15,16 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-uint16_t ports[3] = {7010, 7110, 7210};
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 5;
+int32_t  myIndex = 0;
 
-SSyncNode* syncInitTest() {
-  SSyncFSM* pFsm;
+SRaftId   ids[TSDB_MAX_REPLICA];
+SSyncInfo syncInfo;
+SSyncFSM* pFsm;
 
-  SSyncInfo syncInfo;
-  syncInfo.vgId = 1;
+SSyncNode* syncNodeInit() {
+  syncInfo.vgId = 1234;
   syncInfo.rpcClient = gSyncIO->clientRpc;
   syncInfo.FpSendMsg = syncIOSendMsg;
   syncInfo.queue = gSyncIO->pMsgQ;
@@ -29,46 +34,66 @@ SSyncNode* syncInitTest() {
   snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
 
   SSyncCfg* pCfg = &syncInfo.syncCfg;
-  pCfg->myIndex = 0;
-  pCfg->replicaNum = 3;
+  pCfg->myIndex = myIndex;
+  pCfg->replicaNum = replicaNum;
 
-  pCfg->nodeInfo[0].nodePort = ports[0];
-  snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
-
-  pCfg->nodeInfo[1].nodePort = ports[1];
-  snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
-
-  pCfg->nodeInfo[2].nodePort = ports[2];
-  snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+  for (int i = 0; i < replicaNum; ++i) {
+    pCfg->nodeInfo[i].nodePort = ports[i];
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  }
 
   SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
   assert(pSyncNode != NULL);
 
   gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
   gSyncIO->pSyncNode = pSyncNode;
 
   return pSyncNode;
 }
 
-int main() {
+SSyncNode* syncInitTest() { return syncNodeInit(); }
+
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
+}
+
+int main(int argc, char** argv) {
   // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
   sDebugFlag = 143 + 64;
 
-  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[0]);
+  myIndex = 0;
+  if (argc >= 2) {
+    myIndex = atoi(argv[1]);
+  }
+
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  assert(ret == 0);
+
+  ret = syncEnvStart();
   assert(ret == 0);
 
   SSyncNode* pSyncNode = syncInitTest();
   assert(pSyncNode != NULL);
 
-  cJSON* pJson = syncNode2Json(pSyncNode);
-  char*  serialized = cJSON_Print(pJson);
+  char* serialized = syncNode2Str(pSyncNode);
   printf("%s\n", serialized);
   free(serialized);
-  cJSON_Delete(pJson);
+
+  initRaftId(pSyncNode);
 
   return 0;
 }
diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp
index 4e5f7b78b5..450e097cc8 100644
--- a/source/libs/sync/test/syncPingTest.cpp
+++ b/source/libs/sync/test/syncPingTest.cpp
@@ -25,7 +25,8 @@ SSyncNode* doSync(int myIndex) {
   syncInfo.queue = gSyncIO->pMsgQ;
   syncInfo.FpEqMsg = syncIOEqMsg;
   syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path");
+  snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path");
 
   SSyncCfg* pCfg = &syncInfo.syncCfg;
   pCfg->myIndex = myIndex;
diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp
index 97b807736b..3edde509f8 100644
--- a/source/libs/sync/test/syncVotesGrantedTest.cpp
+++ b/source/libs/sync/test/syncVotesGrantedTest.cpp
@@ -4,6 +4,7 @@
 #include "syncIO.h"
 #include "syncInt.h"
 #include "syncRaftStore.h"
+#include "syncUtil.h"
 #include "syncVoteMgr.h"
 
 void logTest() {
@@ -15,66 +16,141 @@ void logTest() {
   sFatal("--- sync log test: fatal");
 }
 
-uint16_t ports[3] = {7010, 7110, 7210};
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 3;
+int32_t  myIndex = 0;
 
-SSyncNode* doSync(int myIndex) {
-  SSyncFSM* pFsm;
+SRaftId    ids[TSDB_MAX_REPLICA];
+SSyncInfo  syncInfo;
+SSyncFSM*  pFsm;
+SSyncNode* pSyncNode;
 
-  SSyncInfo syncInfo;
-  syncInfo.vgId = 1;
+SSyncNode* syncNodeInit() {
+  syncInfo.vgId = 1234;
   syncInfo.rpcClient = gSyncIO->clientRpc;
   syncInfo.FpSendMsg = syncIOSendMsg;
   syncInfo.queue = gSyncIO->pMsgQ;
   syncInfo.FpEqMsg = syncIOEqMsg;
   syncInfo.pFsm = pFsm;
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
+  snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
 
   SSyncCfg* pCfg = &syncInfo.syncCfg;
   pCfg->myIndex = myIndex;
-  pCfg->replicaNum = 3;
+  pCfg->replicaNum = replicaNum;
 
-  pCfg->nodeInfo[0].nodePort = ports[0];
-  snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  for (int i = 0; i < replicaNum; ++i) {
+    pCfg->nodeInfo[i].nodePort = ports[i];
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  }
 
-  pCfg->nodeInfo[1].nodePort = ports[1];
-  snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
-
-  pCfg->nodeInfo[2].nodePort = ports[2];
-  snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
-  // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
-
-  SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
+  pSyncNode = syncNodeOpen(&syncInfo);
   assert(pSyncNode != NULL);
 
   gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
   gSyncIO->pSyncNode = pSyncNode;
 
   return pSyncNode;
 }
 
-int main() {
+SSyncNode* syncInitTest() { return syncNodeInit(); }
+
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
+}
+
+int main(int argc, char** argv) {
   // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
   sDebugFlag = 143 + 64;
-  logTest();
 
-  int     myIndex = 0;
+  myIndex = 0;
+  if (argc >= 2) {
+    myIndex = atoi(argv[1]);
+  }
+
   int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
   assert(ret == 0);
 
   ret = syncEnvStart();
   assert(ret == 0);
 
-  SSyncNode*     pSyncNode = doSync(myIndex);
-  SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
-  assert(pVotesGranted != NULL);
+  SSyncNode* pSyncNode = syncInitTest();
+  assert(pSyncNode != NULL);
 
-  char* serialized = voteGranted2Str(pVotesGranted);
-  assert(serialized != NULL);
+  char* serialized = syncNode2Str(pSyncNode);
   printf("%s\n", serialized);
   free(serialized);
 
+  initRaftId(pSyncNode);
+
+  SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
+  assert(pVotesGranted != NULL);
+
+  printf("---------------------------------------\n");
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  SyncTerm term = 1234;
+  printf("---------------------------------------\n");
+  voteGrantedReset(pVotesGranted, term);
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  for (int i = 0; i < replicaNum; ++i) {
+    SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
+    reply->destId = pSyncNode->myRaftId;
+    reply->srcId = ids[i];
+    reply->term = term;
+    reply->voteGranted = true;
+
+    voteGrantedVote(pVotesGranted, reply);
+    {
+      char* serialized = voteGranted2Str(pVotesGranted);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+
+    voteGrantedVote(pVotesGranted, reply);
+    {
+      char* serialized = voteGranted2Str(pVotesGranted);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+  }
+
+  printf("---------------------------------------\n");
+  voteGrantedReset(pVotesGranted, 123456789);
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  voteGrantedDestroy(pVotesGranted);
   return 0;
 }

From 3fb4b86248f284308d438004c368feec49c2de93 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Tue, 8 Mar 2022 20:43:12 +0800
Subject: [PATCH 15/29] sync refactor

---
 source/libs/sync/src/syncVoteMgr.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c
index 3b6105ea0a..893a4e3a55 100644
--- a/source/libs/sync/src/syncVoteMgr.c
+++ b/source/libs/sync/src/syncVoteMgr.c
@@ -86,7 +86,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
   cJSON *pReplicas = cJSON_CreateArray();
   cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
   for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
-    cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas)[i])));
+    cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i]));
   }
   int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum);
   for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
@@ -114,7 +114,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
 
 char *voteGranted2Str(SVotesGranted *pVotesGranted) {
   cJSON *pJson = voteGranted2Json(pVotesGranted);
-  char  *serialized = cJSON_Print(pJson);
+  char * serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }

From 43e4cdc7132c940c3bad738fe136209cb31988e3 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Tue, 8 Mar 2022 22:29:52 +0800
Subject: [PATCH 16/29] [TD-13760]<fix>: redefine socket api.

---
 include/os/osSocket.h              |  85 +--
 source/libs/transport/src/rpcTcp.c | 120 ++--
 source/libs/transport/src/rpcUdp.c |  45 +-
 source/libs/transport/src/thttp.c  |  17 +-
 source/os/src/osSocket.c           | 904 +++++++++++++++++------------
 5 files changed, 654 insertions(+), 517 deletions(-)

diff --git a/include/os/osSocket.h b/include/os/osSocket.h
index da1a9651e9..6544c89548 100644
--- a/include/os/osSocket.h
+++ b/include/os/osSocket.h
@@ -21,7 +21,10 @@
     #define socket SOCKET_FUNC_TAOS_FORBID
     #define bind   BIND_FUNC_TAOS_FORBID
     #define listen LISTEN_FUNC_TAOS_FORBID
-    // #define accept ACCEPT_FUNC_TAOS_FORBID
+    #define accept ACCEPT_FUNC_TAOS_FORBID
+    #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+    #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
+    #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
 #endif
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@@ -38,31 +41,6 @@
 extern "C" {
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
-typedef int32_t SOCKET;
-typedef SOCKET  EpollFd;
-#define EpollClose(pollFd) taosCloseSocket(pollFd)
-
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-typedef SOCKET SocketFd;
-#else
-typedef int32_t SocketFd;
-#endif
-
-int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
-int32_t taosWriteSocket(SocketFd fd, void *msg, int len);
-int32_t taosReadSocket(SocketFd fd, void *msg, int len);
-int32_t taosCloseSocketNoCheck(SocketFd fd);
-int32_t taosCloseSocket(SocketFd fd);
-void    taosShutDownSocketRD(SOCKET fd);
-void    taosShutDownSocketWR(SOCKET fd);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
-int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
-int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
-
-uint32_t    taosInetAddr(const char *ipAddr);
-const char *taosInetNtoa(struct in_addr ipInt);
-
 #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 
   #define htobe64 htonll
   #if defined(_TD_GO_DLL_)
@@ -74,25 +52,54 @@ const char *taosInetNtoa(struct in_addr ipInt);
   #define htobe64 htonll
 #endif
 
-int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
-int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
-int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
-int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len);
-int32_t taosSetNonblocking(SOCKET sock, int32_t on);
+#define TAOS_EPOLL_WAIT_TIME 500
 
-SOCKET  taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
-SOCKET  taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
-SOCKET  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int32_t taosKeepTcpAlive(SOCKET sockFd);
+typedef struct TdSocketServer *TdSocketServerPtr;
+typedef struct TdSocket *TdSocketPtr;
+typedef struct TdEpoll *TdEpollPtr;
 
-void    taosBlockSIGPIPE();
+int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosCloseSocket(TdSocketPtr *ppSocket);
+int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
+int32_t taosShutDownSocketRD(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer);
+int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
+int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
+int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
+int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
+int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
+int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
+int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
+int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+
+TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
+int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+
+int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+
+void     taosBlockSIGPIPE();
 uint32_t taosGetIpv4FromFqdn(const char *);
 int32_t  taosGetFqdn(char *);
 void     tinet_ntoa(char *ipstr, uint32_t ip);
 uint32_t ip2uint(const char *const ip_addr);
-void    taosIgnSIGPIPE();
-void    taosSetMaskSIGPIPE();
+void     taosIgnSIGPIPE();
+void     taosSetMaskSIGPIPE();
+uint32_t taosInetAddr(const char *ipAddr);
+const char *taosInetNtoa(struct in_addr ipInt);
+
+TdEpollPtr taosCreateEpoll(int32_t size);
+int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
+int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
+int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c
index d95ac3d36d..aac38b21e8 100644
--- a/source/libs/transport/src/rpcTcp.c
+++ b/source/libs/transport/src/rpcTcp.c
@@ -24,7 +24,7 @@
 #ifndef USE_UV
 typedef struct SFdObj {
   void *             signature;
-  SOCKET             fd;       // TCP socket FD
+  TdSocketPtr        pSocket;       // TCP socket FD
   void *             thandle;  // handle from upper layer, like TAOS
   uint32_t           ip;
   uint16_t           port;
@@ -40,7 +40,7 @@ typedef struct SThreadObj {
   pthread_mutex_t mutex;
   uint32_t        ip;
   bool            stop;
-  EpollFd         pollFd;
+  TdEpollPtr      pEpoll;
   int             numOfFds;
   int             threadId;
   char            label[TSDB_LABEL_LEN];
@@ -56,20 +56,20 @@ typedef struct {
 } SClientObj;
 
 typedef struct {
-  SOCKET       fd;
-  uint32_t     ip;
-  uint16_t     port;
-  int8_t       stop;
-  int8_t       reserve;
-  char         label[TSDB_LABEL_LEN];
-  int          numOfThreads;
-  void *       shandle;
-  SThreadObj **pThreadObj;
-  pthread_t    thread;
+  TdSocketServerPtr  pSocketServer;
+  uint32_t           ip;
+  uint16_t           port;
+  int8_t             stop;
+  int8_t             reserve;
+  char               label[TSDB_LABEL_LEN];
+  int                numOfThreads;
+  void *             shandle;
+  SThreadObj       **pThreadObj;
+  pthread_t          thread;
 } SServerObj;
 
 static void *  taosProcessTcpData(void *param);
-static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd);
+static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket);
 static void    taosFreeFdObj(SFdObj *pFdObj);
 static void    taosReportBrokenLink(SFdObj *pFdObj);
 static void *  taosAcceptTcpConnection(void *arg);
@@ -85,7 +85,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     return NULL;
   }
 
-  pServerObj->fd = -1;
+  pServerObj->pSocketServer = NULL;
   taosResetPthread(&pServerObj->thread);
   pServerObj->ip = ip;
   pServerObj->port = port;
@@ -118,7 +118,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     }
 
     pServerObj->pThreadObj[i] = pThreadObj;
-    pThreadObj->pollFd = -1;
+    pThreadObj->pEpoll = NULL;
     taosResetPthread(&pThreadObj->thread);
     pThreadObj->processData = fp;
     tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
@@ -135,8 +135,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
       break;
     }
 
-    pThreadObj->pollFd = (EpollFd)epoll_create(10);  // size does not matter
-    if (pThreadObj->pollFd < 0) {
+    pThreadObj->pEpoll = taosCreateEpoll(10);  // size does not matter
+    if (pThreadObj->pEpoll == NULL) {
       tError("%s failed to create TCP epoll", label);
       code = -1;
       break;
@@ -151,8 +151,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     pThreadObj->threadId = i;
   }
 
-  pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
-  if (pServerObj->fd < 0) code = -1;
+  pServerObj->pSocketServer = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
+  if (pServerObj->pSocketServer == NULL) code = -1;
 
   if (code == 0) {
     code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj);
@@ -196,8 +196,8 @@ void taosStopTcpServer(void *handle) {
   if (pServerObj == NULL) return;
   pServerObj->stop = 1;
 
-  if (pServerObj->fd >= 0) {
-    taosShutDownSocketRD(pServerObj->fd);
+  if (pServerObj->pSocketServer != NULL) {
+    taosShutDownSocketServerRD(pServerObj->pSocketServer);
   }
   if (taosCheckPthreadValid(pServerObj->thread)) {
     if (taosComparePthread(pServerObj->thread, pthread_self())) {
@@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) {
 }
 
 static void *taosAcceptTcpConnection(void *arg) {
-  SOCKET             connFd = -1;
+  TdSocketPtr        pSocket = NULL;
   struct sockaddr_in caddr;
   int                threadId = 0;
   SThreadObj *       pThreadObj;
@@ -239,13 +239,13 @@ static void *taosAcceptTcpConnection(void *arg) {
 
   while (1) {
     socklen_t addrlen = sizeof(caddr);
-    connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
+    pSocket = taosAcceptTcpConnectSocket(pServerObj->pSocketServer, (struct sockaddr *)&caddr, &addrlen);
     if (pServerObj->stop) {
       tDebug("%s TCP server stop accepting new connections", pServerObj->label);
       break;
     }
 
-    if (connFd == -1) {
+    if (pSocket == NULL) {
       if (errno == EINVAL) {
         tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label);
         break;
@@ -255,11 +255,11 @@ static void *taosAcceptTcpConnection(void *arg) {
       continue;
     }
 
-    taosKeepTcpAlive(connFd);
+    taosKeepTcpAlive(pSocket);
     struct timeval to = {5, 0};
-    int32_t        ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
+    int32_t        ret = taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
     if (ret != 0) {
-      taosCloseSocket(connFd);
+      taosCloseSocket(&pSocket);
       tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno),
              taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
       continue;
@@ -268,14 +268,14 @@ static void *taosAcceptTcpConnection(void *arg) {
     // pick up the thread to handle this connection
     pThreadObj = pServerObj->pThreadObj[threadId];
 
-    SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
+    SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket);
     if (pFdObj) {
       pFdObj->ip = caddr.sin_addr.s_addr;
       pFdObj->port = htons(caddr.sin_port);
-      tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
-             taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
+      tDebug("%s new TCP connection from %s:%hu, FD:%p numOfFds:%d", pServerObj->label,
+             taosInetNtoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
     } else {
-      taosCloseSocket(connFd);
+      taosCloseSocket(&pSocket);
       tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
              taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
     }
@@ -285,7 +285,7 @@ static void *taosAcceptTcpConnection(void *arg) {
     threadId = threadId % pServerObj->numOfThreads;
   }
 
-  taosCloseSocket(pServerObj->fd);
+  taosCloseSocketServer(&pServerObj->pSocketServer);
   return NULL;
 }
 
@@ -339,8 +339,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
       break;
     }
 
-    pThreadObj->pollFd = (int64_t)epoll_create(10);  // size does not matter
-    if (pThreadObj->pollFd < 0) {
+    pThreadObj->pEpoll = taosCreateEpoll(10);  // size does not matter
+    if (pThreadObj->pEpoll == NULL) {
       tError("%s failed to create TCP epoll", label);
       code = -1;
       break;
@@ -388,21 +388,17 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
   atomic_store_32(&pClientObj->index, index + 1);
   SThreadObj *pThreadObj = pClientObj->pThreadObj[index];
 
-  SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  if (fd == (SOCKET)-1) return NULL;
-#else
-  if (fd <= 0) return NULL;
-#endif
+  TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
+  if (pSocket == NULL) return NULL;
 
   struct sockaddr_in sin;
   uint16_t           localPort = 0;
   unsigned int       addrlen = sizeof(sin);
-  if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
+  if (taosGetSocketName(pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
     localPort = (uint16_t)ntohs(sin.sin_port);
   }
 
-  SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
+  SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket);
 
   if (pFdObj) {
     pFdObj->thandle = thandle;
@@ -415,7 +411,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
            ipport, localPort, pFdObj, pThreadObj->numOfFds);
   } else {
     tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
-    taosCloseSocket(fd);
+    taosCloseSocket(&pSocket);
   }
 
   return pFdObj;
@@ -430,7 +426,7 @@ void taosCloseTcpConnection(void *chandle) {
 
   // pFdObj->thandle = NULL;
   pFdObj->closedByApp = 1;
-  taosShutDownSocketWR(pFdObj->fd);
+  taosShutDownSocketWR(pFdObj->pSocket);
 }
 
 int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
@@ -438,8 +434,8 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
   if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
   SThreadObj *pThreadObj = pFdObj->pThreadObj;
 
-  int ret = taosWriteMsg(pFdObj->fd, data, len);
-  tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret);
+  int ret = taosWriteMsg(pFdObj->pSocket, data, len);
+  tTrace("%s %p TCP data is sent, FD:%p bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, ret);
 
   return ret;
 }
@@ -449,7 +445,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
 
   // notify the upper layer, so it will clean the associated context
   if (pFdObj->closedByApp == 0) {
-    taosShutDownSocketWR(pFdObj->fd);
+    taosShutDownSocketWR(pFdObj->pSocket);
 
     SRecvInfo recvInfo;
     recvInfo.msg = NULL;
@@ -473,7 +469,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
 
   SThreadObj *pThreadObj = pFdObj->pThreadObj;
 
-  headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
+  headLen = taosReadMsg(pFdObj->pSocket, &rpcHead, sizeof(SRpcHead));
   if (headLen != sizeof(SRpcHead)) {
     tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
     return -1;
@@ -486,13 +482,12 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   } else {
-    tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd,
-           buffer);
+    tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer);
   }
 
   msg = buffer + tsRpcOverhead;
   leftLen = msgLen - headLen;
-  retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
+  retLen = taosReadMsg(pFdObj->pSocket, msg + headLen, leftLen);
 
   if (leftLen != retLen) {
     tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
@@ -532,7 +527,7 @@ static void *taosProcessTcpData(void *param) {
   setThreadName(name);
 
   while (1) {
-    int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
+    int fdNum = taosWaitEpoll(pThreadObj->pEpoll, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
     if (pThreadObj->stop) {
       tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label);
       break;
@@ -561,7 +556,7 @@ static void *taosProcessTcpData(void *param) {
       }
 
       if (taosReadTcpData(pFdObj, &recvInfo) < 0) {
-        shutdown(pFdObj->fd, SHUT_WR);
+        taosShutDownSocketWR(pFdObj->pSocket);
         continue;
       }
 
@@ -572,9 +567,9 @@ static void *taosProcessTcpData(void *param) {
     if (pThreadObj->stop) break;
   }
 
-  if (pThreadObj->pollFd >= 0) {
-    EpollClose(pThreadObj->pollFd);
-    pThreadObj->pollFd = -1;
+  if (pThreadObj->pEpoll != NULL) {
+    taosCloseEpoll(&pThreadObj->pEpoll);
+    pThreadObj->pEpoll = NULL;
   }
 
   while (pThreadObj->pHead) {
@@ -590,7 +585,7 @@ static void *taosProcessTcpData(void *param) {
   return NULL;
 }
 
-static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
+static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) {
   struct epoll_event event;
 
   SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1);
@@ -599,13 +594,13 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
   }
 
   pFdObj->closedByApp = 0;
-  pFdObj->fd = fd;
+  pFdObj->pSocket = pSocket;
   pFdObj->pThreadObj = pThreadObj;
   pFdObj->signature = pFdObj;
 
   event.events = EPOLLIN | EPOLLRDHUP;
   event.data.ptr = pFdObj;
-  if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+  if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) {
     tfree(pFdObj);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return NULL;
@@ -635,8 +630,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
   }
 
   pFdObj->signature = NULL;
-  epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
-  taosCloseSocket(pFdObj->fd);
+  taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_DEL, pFdObj->pSocket, NULL);
+  taosCloseSocket(&pFdObj->pSocket);
 
   pThreadObj->numOfFds--;
   if (pThreadObj->numOfFds < 0)
@@ -655,8 +650,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pthread_mutex_unlock(&pThreadObj->mutex);
 
-  tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj,
-         pFdObj->fd, pThreadObj->numOfFds);
+  tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
 
   tfree(pFdObj);
 }
diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c
index 3640414a4c..81c8d0af76 100644
--- a/source/libs/transport/src/rpcUdp.c
+++ b/source/libs/transport/src/rpcUdp.c
@@ -30,17 +30,17 @@
 #define RPC_MAX_UDP_SIZE 65480
 
 typedef struct {
-  int       index;
-  SOCKET    fd;
-  uint16_t  port;                   // peer port
-  uint16_t  localPort;              // local port
-  char      label[TSDB_LABEL_LEN];  // copy from udpConnSet;
-  pthread_t thread;
-  void *    hash;
-  void *    shandle;  // handle passed by upper layer during server initialization
-  void *    pSet;
-  void *(*processData)(SRecvInfo *pRecv);
-  char *buffer;  // buffer to receive data
+  int         index;
+  TdSocketPtr pSocket;
+  uint16_t    port;                   // peer port
+  uint16_t    localPort;              // local port
+  char        label[TSDB_LABEL_LEN];  // copy from udpConnSet;
+  pthread_t   thread;
+  void       *hash;
+  void       *shandle;  // handle passed by upper layer during server initialization
+  void       *pSet;
+  void       *(*processData)(SRecvInfo *pRecv);
+  char       *buffer;  // buffer to receive data
 } SUdpConn;
 
 typedef struct {
@@ -86,8 +86,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   for (i = 0; i < threads; ++i) {
     pConn = pSet->udpConn + i;
     ownPort = (port ? port + i : 0);
-    pConn->fd = taosOpenUdpSocket(ip, ownPort);
-    if (pConn->fd < 0) {
+    pConn->pSocket = taosOpenUdpSocket(ip, ownPort);
+    if (pConn->pSocket == NULL) {
       tError("%s failed to open UDP socket %x:%hu", label, ip, port);
       break;
     }
@@ -100,7 +100,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
 
     struct sockaddr_in sin;
     unsigned int       addrlen = sizeof(sin);
-    if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
+    if (taosGetSocketName(pConn->pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
         addrlen == sizeof(sin)) {
       pConn->localPort = (uint16_t)ntohs(sin.sin_port);
     }
@@ -138,9 +138,9 @@ void taosStopUdpConnection(void *handle) {
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
-    if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR);
-    if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
-    pConn->fd = -1;
+    if (pConn->pSocket != NULL) taosShutDownSocketRDWR(pConn->pSocket);
+    if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket);
+    pConn->pSocket = NULL;
   }
 
   for (int i = 0; i < pSet->threads; ++i) {
@@ -163,7 +163,7 @@ void taosCleanUpUdpConnection(void *handle) {
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
-    if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
+    if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket);
   }
 
   tDebug("%s UDP is cleaned up", pSet->label);
@@ -199,13 +199,12 @@ static void *taosRecvUdpData(void *param) {
   setThreadName("recvUdpData");
 
   while (1) {
-    dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
+    dataLen = taosReadFromSocket(pConn->pSocket, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
     if (dataLen <= 0) {
-      tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen,
-             pConn->fd);
+      tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d", pConn->label, strerror(errno), (int32_t)dataLen);
 
       // for windows usage, remote shutdown also returns - 1 in windows client
-      if (pConn->fd == -1) {
+      if (pConn->pSocket == NULL) {
         break;
       } else {
         continue;
@@ -255,7 +254,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
   destAdd.sin_addr.s_addr = ip;
   destAdd.sin_port = htons(port);
 
-  int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
+  int ret = taosSendto(pConn->pSocket, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
 
   return ret;
 }
diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index 95e67290ac..6d1c691b9b 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -173,7 +173,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
 #else
 int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) {
   int32_t code = -1;
-  SOCKET  fd = 0;
+  TdSocketPtr pSocket = NULL;
 
   uint32_t ip = taosGetIpv4FromFqdn(server);
   if (ip == 0xffffffff) {
@@ -182,8 +182,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
     goto SEND_OVER;
   }
 
-  fd = taosOpenTcpClientSocket(ip, port, 0);
-  if (fd < 0) {
+  pSocket = taosOpenTcpClientSocket(ip, port, 0);
+  if (pSocket == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     uError("failed to create http socket to %s:%u since %s", server, port, terrstr());
     goto SEND_OVER;
@@ -200,21 +200,20 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
 
   char    header[1024] = {0};
   int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag);
-
-  if (taosWriteSocket(fd, header, headLen) < 0) {
+  if (taosWriteMsg(pSocket, header, headLen) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     uError("failed to send http header to %s:%u since %s", server, port, terrstr());
     goto SEND_OVER;
   }
 
-  if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
+  if (taosWriteMsg(pSocket, (void*)pCont, contLen) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     uError("failed to send http content to %s:%u since %s", server, port, terrstr());
     goto SEND_OVER;
   }
 
   // read something to avoid nginx error 499
-  if (taosReadSocket(fd, header, 10) < 0) {
+  if (taosWriteMsg(pSocket, header, 10) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     uError("failed to receive response from %s:%u since %s", server, port, terrstr());
     goto SEND_OVER;
@@ -223,8 +222,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
   code = 0;
 
 SEND_OVER:
-  if (fd != 0) {
-    taosCloseSocket(fd);
+  if (pSocket != NULL) {
+    taosCloseSocket(&pSocket);
   }
 
   return code;
diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 096516a98d..9330cb7b32 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -18,176 +18,206 @@
 #include "os.h"
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  #include "winsock2.h"
-  #include <WS2tcpip.h>
-  #include <winbase.h>
-  #include <Winsock2.h>
-#else
-  #include <arpa/inet.h>
-  #include <fcntl.h>
-  #include <netdb.h>
-  #include <netinet/in.h>
-  #include <netinet/ip.h>
-  #include <netinet/tcp.h>
-  #include <netinet/udp.h>
-  #include <sys/socket.h>
-  #include <unistd.h>
-#endif
-
-// typedef struct TdSocketServer {
-// #if SOCKET_WITH_LOCK
-//   pthread_rwlock_t rwlock;
-// #endif
-//   int        refId;
-//   SocketFd   fd;
-// } * TdSocketServerPtr, TdSocketServer;
-
-// typedef struct TdSocketConnector {
-// #if SOCKET_WITH_LOCK
-//   pthread_rwlock_t rwlock;
-// #endif
-//   int        refId;
-//   SocketFd   fd;
-// } * TdSocketConnectorPtr, TdSocketConnector;
-
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-
-#define taosSend(sockfd, buf, len, flags) send((SOCKET)sockfd, buf, len, flags)
-int32_t taosSendto(SocketFd fd, void *buf, int len, unsigned int flags, const struct sockaddr *to, int tolen) {
-  return sendto((SOCKET)sockfd, buf, len, flags, dest_addr, addrlen);
-}
-int32_t taosWriteSocket(SocketFd fd, void *buf, int len) { return send((SOCKET)fd, buf, len, 0); }
-int32_t taosReadSocket(SocketFd fd, void *buf, int len) { return recv((SOCKET)fd, buf, len, 0)(); }
-int32_t taosCloseSocketNoCheck(SocketFd fd) { return closesocket((SOCKET)fd); }
-int32_t taosCloseSocket(SocketFd fd) { closesocket((SOCKET)fd) }
-
-#else
-
-  #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags)
-  int32_t taosSendto(SocketFd fd, void * buf, int len, unsigned int flags, const struct sockaddr * dest_addr, int addrlen) {
-    return sendto(fd, buf, len, flags, dest_addr, addrlen);
-  }
-
-  int32_t taosWriteSocket(SocketFd fd, void *buf, int len) {
-    return write(fd, buf, len);
-  }
-
-  int32_t taosReadSocket(SocketFd fd, void *buf, int len) {
-    return read(fd, buf, len);
-  }
-
-  int32_t taosCloseSocketNoCheck(SocketFd fd) {
-    return close(fd);
-  }
-
-  int32_t taosCloseSocket(SocketFd fd) {
-    if (fd > -1) {
-      close(fd);
-    }
-  }
-#endif
-
-void taosShutDownSocketRD(SOCKET fd) {
-#ifdef WINDOWS
-  closesocket(fd);
-#elif __APPLE__
-  close(fd);
-#else
-  shutdown(fd, SHUT_RD);
-#endif
-}
-
-void taosShutDownSocketWR(SOCKET fd) {
-#ifdef WINDOWS
-  closesocket(fd);
-#elif __APPLE__
-  close(fd);
-#else
-  shutdown(fd, SHUT_WR);
-#endif
-}
-
-#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32))
-
-int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
-  int32_t flags = 0;
-  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
-    //printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  if (on)
-    flags |= O_NONBLOCK;
-  else
-    flags &= ~O_NONBLOCK;
-
-  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
-    //printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  return 0;
-}
-
-
-
-
-#endif
-
-#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) || defined(_TD_DARWIN_32))
-
-int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
-  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
-}
-
-int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen) {
-  return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen);
-}
-
-#endif
-
-#if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER))
-
-uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); }
-
-const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); }
-
-#endif
-
-#if defined(_TD_DARWIN_64)
-
-/*
- * darwin implementation
- */
-
-int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen) {
-  if (level == SOL_SOCKET && optname == SO_SNDBUF) {
-    return 0;
-  }
-
-  if (level == SOL_SOCKET && optname == SO_RCVBUF) {
-    return 0;
-  }
-
-  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
-}
-#endif
-
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-
-/*
- * windows implementation
- */
-
 #include <IPHlpApi.h>
 #include <WS2tcpip.h>
+#include <Winsock2.h>
 #include <stdio.h>
 #include <string.h>
 #include <tchar.h>
+#include <winbase.h>
 #include <winsock2.h>
 #include <ws2def.h>
+#include "winsock2.h"
+#else
+#include <arpa/inet.h>
+#include <fcntl.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <unistd.h>
+#endif
 
-void taosWinSocketInit() {
+typedef int32_t SocketFd;
+typedef SocketFd  EpollFd;
+
+typedef struct TdSocketServer {
+#if SOCKET_WITH_LOCK
+  pthread_rwlock_t rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} *TdSocketServerPtr, TdSocketServer;
+
+typedef struct TdSocket {
+#if SOCKET_WITH_LOCK
+  pthread_rwlock_t rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} *TdSocketPtr, TdSocket;
+
+typedef struct TdEpoll {
+#if SOCKET_WITH_LOCK
+  pthread_rwlock_t rwlock;
+#endif
+  int      refId;
+  EpollFd  fd;
+} *TdEpollPtr, TdEpoll;
+
+int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr,
+                    int addrlen) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen);
+#else
+  return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen);
+#endif
+}
+int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return send(pSocket->fd, buf, len, 0);
+  ;
+#else
+  return write(pSocket->fd, buf, len);
+#endif
+}
+int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return recv(pSocket->fd, buf, len, 0);
+  ;
+#else
+  return read(pSocket->fd, buf, len);
+#endif
+}
+
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+  return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen);
+}
+int32_t taosCloseSocketNoCheck1(SocketFd fd) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return closesocket(fd);
+#else
+  return close(fd);
+#endif
+}
+int32_t taosCloseSocket(TdSocketPtr *ppSocket) {
+  int32_t code;
+  if (ppSocket == NULL || *ppSocket == NULL || (*ppSocket)->fd < 0) {
+    return -1;
+  }
+  code = taosCloseSocketNoCheck1((*ppSocket)->fd);
+  (*ppSocket)->fd = -1;
+  free(*ppSocket);
+  return code;
+}
+int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) {
+  int32_t code;
+  if (ppSocketServer == NULL || *ppSocketServer == NULL || (*ppSocketServer)->fd < 0) {
+    return -1;
+  }
+  code = taosCloseSocketNoCheck1((*ppSocketServer)->fd);
+  (*ppSocketServer)->fd = -1;
+  free(*ppSocketServer);
+  return code;
+}
+
+int32_t taosShutDownSocketRD(TdSocketPtr pSocket) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocket->fd);
+#elif __APPLE__
+  return close(pSocket->fd);
+#else
+  return shutdown(pSocket->fd, SHUT_RD);
+#endif
+}
+int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer) {
+  if (pSocketServer == NULL || pSocketServer->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocketServer->fd);
+#elif __APPLE__
+  return close(pSocketServer->fd);
+#else
+  return shutdown(pSocketServer->fd, SHUT_RD);
+#endif
+}
+
+int32_t taosShutDownSocketWR(TdSocketPtr pSocket) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocket->fd);
+#elif __APPLE__
+  return close(pSocket->fd);
+#else
+  return shutdown(pSocket->fd, SHUT_WR);
+#endif
+}
+int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer) {
+  if (pSocketServer == NULL || pSocketServer->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocketServer->fd);
+#elif __APPLE__
+  return close(pSocketServer->fd);
+#else
+  return shutdown(pSocketServer->fd, SHUT_WR);
+#endif
+}
+int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocket->fd);
+#elif __APPLE__
+  return close(pSocket->fd);
+#else
+  return shutdown(pSocket->fd, SHUT_RDWR);
+#endif
+}
+int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) {
+  if (pSocketServer == NULL || pSocketServer->fd < 0) {
+    return -1;
+  }
+#ifdef WINDOWS
+  return closesocket(pSocketServer->fd);
+#elif __APPLE__
+  return close(pSocketServer->fd);
+#else
+  return shutdown(pSocketServer->fd, SHUT_RDWR);
+#endif
+}
+
+#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 
+  #if defined(_TD_GO_DLL_)
+    uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); }
+  #endif
+#endif
+
+void taosWinSocketInit1() {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   static char flag = 0;
   if (flag == 0) {
     WORD    wVersionRequested;
@@ -197,21 +227,46 @@ void taosWinSocketInit() {
       flag = 1;
     }
   }
+#else
+#endif
 }
-
-int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
+int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   u_long mode;
   if (on) {
     mode = 1;
-    ioctlsocket(sock, FIONBIO, &mode);
+    ioctlsocket(pSocket->fd, FIONBIO, &mode);
   } else {
     mode = 0;
-    ioctlsocket(sock, FIONBIO, &mode);
+    ioctlsocket(pSocket->fd, FIONBIO, &mode);
   }
+#else
+  int32_t flags = 0;
+  if ((flags = fcntl(pSocket->fd, F_GETFL, 0)) < 0) {
+    // printf("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  if (on)
+    flags |= O_NONBLOCK;
+  else
+    flags &= ~O_NONBLOCK;
+
+  if ((flags = fcntl(pSocket->fd, F_SETFL, flags)) < 0) {
+    // printf("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+#endif
   return 0;
 }
-
-int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
+int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {
     return 0;
   }
@@ -228,13 +283,23 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op
     return 0;
   }
 
-  return setsockopt(socketfd, level, optname, optval, optlen);
+  return setsockopt(pSocket->fd, level, optname, optval, optlen);
+#else
+  return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen);
+#endif
+}
+int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  return 0;
+#else
+  return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen);
+#endif
 }
-
-#ifdef _MSC_VER
-//#if _MSC_VER >= 1900
-
 uint32_t taosInetAddr(const char *ipAddr) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   uint32_t value;
   int32_t  ret = inet_pton(AF_INET, ipAddr, &value);
   if (ret <= 0) {
@@ -242,39 +307,37 @@ uint32_t taosInetAddr(const char *ipAddr) {
   } else {
     return value;
   }
+#else
+  return inet_addr(ipAddr);
+#endif
 }
-
 const char *taosInetNtoa(struct in_addr ipInt) {
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
   // not thread safe, only for debug usage while print log
   static char tmpDstStr[16];
   return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN);
+#else
+  return inet_ntoa(ipInt);
+#endif
 }
 
-//#endif
-#endif
-
-#if defined(_TD_GO_DLL_)
-
-uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); }
-
-#endif
-
-#endif
-
 #ifndef SIGPIPE
-  #define SIGPIPE EPIPE
+#define SIGPIPE EPIPE
 #endif
 
 #define TCP_CONN_TIMEOUT 3000  // conn timeout
 
-int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
+int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
   int32_t nleft, nwritten;
-  char *  ptr = (char *)buf;
+  char   *ptr = (char *)buf;
 
   nleft = nbytes;
 
   while (nleft > 0) {
-    nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
+    nwritten = taosWriteSocket(pSocket, (char *)ptr, (size_t)nleft);
     if (nwritten <= 0) {
       if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */)
         continue;
@@ -293,20 +356,21 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
   return (nbytes - nleft);
 }
 
-int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
+int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
   int32_t nleft, nread;
-  char *  ptr = (char *)buf;
+  char   *ptr = (char *)buf;
 
   nleft = nbytes;
 
-  if (fd < 0) return -1;
-
   while (nleft > 0) {
-    nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft);
+    nread = taosReadSocket(pSocket, ptr, (size_t)nleft);
     if (nread == 0) {
       break;
     } else if (nread < 0) {
-      if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) {
+      if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK*/) {
         continue;
       } else {
         return -1;
@@ -324,11 +388,14 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
   return (nbytes - nleft);
 }
 
-int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) {
-  taosSetNonblocking(fd, 1);
+int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+  taosSetNonblocking(pSocket, 1);
 
-  int32_t nleft, nwritten, nready;
-  fd_set  fset;
+  int32_t        nleft, nwritten, nready;
+  fd_set         fset;
   struct timeval tv;
 
   nleft = nbytes;
@@ -336,24 +403,24 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) {
     tv.tv_sec = 30;
     tv.tv_usec = 0;
     FD_ZERO(&fset);
-    FD_SET(fd, &fset);
-    if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
+    FD_SET(pSocket->fd, &fset);
+    if ((nready = select((SocketFd)(pSocket->fd + 1), NULL, &fset, NULL, &tv)) == 0) {
       errno = ETIMEDOUT;
-      //printf("fd %d timeout, no enough space to write", fd);
+      // printf("fd %d timeout, no enough space to write", fd);
       break;
 
     } else if (nready < 0) {
       if (errno == EINTR) continue;
 
-      //printf("select error, %d (%s)", errno, strerror(errno));
+      // printf("select error, %d (%s)", errno, strerror(errno));
       return -1;
     }
 
-    nwritten = (int32_t)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+    nwritten = (int32_t)send(pSocket->fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
     if (nwritten <= 0) {
       if (errno == EAGAIN || errno == EINTR) continue;
 
-      //printf("write error, %d (%s)", errno, strerror(errno));
+      // printf("write error, %d (%s)", errno, strerror(errno));
       return -1;
     }
 
@@ -361,121 +428,99 @@ int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes) {
     ptr += nwritten;
   }
 
-  taosSetNonblocking(fd, 0);
+  taosSetNonblocking(pSocket, 0);
 
   return (nbytes - nleft);
 }
 
-int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) {
-  int32_t nread, nready, nleft = nbytes;
-
-  fd_set fset;
-  struct timeval tv;
-
-  while (nleft > 0) {
-    tv.tv_sec = 30;
-    tv.tv_usec = 0;
-    FD_ZERO(&fset);
-    FD_SET(fd, &fset);
-    if ((nready = select((int32_t)(fd + 1), NULL, &fset, NULL, &tv)) == 0) {
-      errno = ETIMEDOUT;
-      //printf("fd %d timeout\n", fd);
-      break;
-    } else if (nready < 0) {
-      if (errno == EINTR) continue;
-      //printf("select error, %d (%s)", errno, strerror(errno));
-      return -1;
-    }
-
-    if ((nread = (int32_t)taosReadSocket(fd, ptr, (size_t)nleft)) < 0) {
-      if (errno == EINTR) continue;
-      //printf("read error, %d (%s)", errno, strerror(errno));
-      return -1;
-
-    } else if (nread == 0) {
-      //printf("fd %d EOF", fd);
-      break;  // EOF
-    }
-
-    nleft -= nread;
-    ptr += nread;
-  }
-
-  return (nbytes - nleft);
-}
-
-SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) {
+TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) {
   struct sockaddr_in localAddr;
-  SOCKET sockFd;
-  int32_t bufSize = 1024000;
+  SocketFd           fd;
+  int32_t            bufSize = 1024000;
 
-  //printf("open udp socket:0x%x:%hu", ip, port);
+  // printf("open udp socket:0x%x:%hu", ip, port);
 
   memset((char *)&localAddr, 0, sizeof(localAddr));
   localAddr.sin_family = AF_INET;
   localAddr.sin_addr.s_addr = ip;
   localAddr.sin_port = (uint16_t)htons(port);
 
-  if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
-    //printf("failed to open udp socket: %d (%s)", errno, strerror(errno));
-    taosCloseSocketNoCheck(sockFd);
-    return -1;
+  if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) {
+    // printf("failed to open udp socket: %d (%s)", errno, strerror(errno));
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
   }
 
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
-    //printf("failed to set the send buffer size for UDP socket\n");
-    taosCloseSocket(sockFd);
-    return -1;
+  TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket));
+  if (pSocket == NULL) {
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
+  }
+  pSocket->fd = fd;
+  pSocket->refId = 0;
+
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    // printf("failed to set the send buffer size for UDP socket\n");
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
-    //printf("failed to set the receive buffer size for UDP socket\n");
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    // printf("failed to set the receive buffer size for UDP socket\n");
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
   /* bind socket to local address */
-  if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
-    //printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port);
-    taosCloseSocket(sockFd);
-    return -1;
+  if (bind(pSocket->fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
+    // printf("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port);
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  return sockFd;
+  return pSocket;
 }
 
-SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
-  SOCKET sockFd = 0;
-  int32_t ret;
+TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) {
+  SocketFd           fd = -1;
+  int32_t            ret;
   struct sockaddr_in serverAddr, clientAddr;
-  int32_t bufSize = 1024 * 1024;
+  int32_t            bufSize = 1024 * 1024;
 
-  sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+  fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 
-  if (sockFd <= 2) {
-    //printf("failed to open the socket: %d (%s)", errno, strerror(errno));
-    taosCloseSocketNoCheck(sockFd);
-    return -1;
+  if (fd <= 2) {
+    // printf("failed to open the socket: %d (%s)", errno, strerror(errno));
+    if (fd >= 0) taosCloseSocketNoCheck1(fd);
+    return NULL;
   }
 
+  TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket));
+  if (pSocket == NULL) {
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
+  }
+  pSocket->fd = fd;
+  pSocket->refId = 0;
+
   /* set REUSEADDR option, so the portnumber can be re-used */
   int32_t reuse = 1;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
-    //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
+    // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
-    //printf("failed to set the send buffer size for TCP socket\n");
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    // printf("failed to set the send buffer size for TCP socket\n");
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
-    //printf("failed to set the receive buffer size for TCP socket\n");
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) {
+    // printf("failed to set the receive buffer size for TCP socket\n");
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
   if (clientIp != 0) {
@@ -485,11 +530,11 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
     clientAddr.sin_port = 0;
 
     /* bind socket to client address */
-    if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
-      //printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort,
-      //       strerror(errno));
-      taosCloseSocket(sockFd);
-      return -1;
+    if (bind(pSocket->fd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) {
+      // printf("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", clientIp, destIp, destPort,
+      //        strerror(errno));
+      taosCloseSocket(&pSocket);
+      return NULL;
     }
   }
 
@@ -498,159 +543,193 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie
   serverAddr.sin_addr.s_addr = destIp;
   serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort);
 
-#ifdef _TD_LINUX 
-  taosSetNonblocking(sockFd, 1);   
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); 
+#ifdef _TD_LINUX
+  taosSetNonblocking(pSocket, 1);
+  ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
   if (ret == -1) {
     if (errno == EHOSTUNREACH) {
-      //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
-      taosCloseSocket(sockFd);
-      return -1; 
+      // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
+      taosCloseSocket(&pSocket);
+      return -1;
     } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) {
-      struct pollfd wfd[1]; 
+      struct pollfd wfd[1];
 
-      wfd[0].fd = sockFd;
+      wfd[0].fd = pSocket->fd;
       wfd[0].events = POLLOUT;
-    
+
       int res = poll(wfd, 1, TCP_CONN_TIMEOUT);
       if (res == -1 || res == 0) {
-        //printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort);
-        taosCloseSocket(sockFd); //  
+        // printf("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort);
+        taosCloseSocket(&pSocket);  //
         return -1;
       }
-      int optVal = -1, optLen = sizeof(int);  
-      if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) {
-        //printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort);
-        taosCloseSocket(sockFd); //  
+      int optVal = -1, optLen = sizeof(int);
+      if ((0 != taosGetSockOpt(pSocket, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) {
+        // printf("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort);
+        taosCloseSocket(&pSocket);  //
         return -1;
       }
       ret = 0;
-    } else { // Other error
-      //printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort);
-      taosCloseSocket(sockFd); //  
-      return -1; 
-    } 
+    } else {  // Other error
+      // printf("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort);
+      taosCloseSocket(&pSocket);  //
+      return -1;
+    }
   }
-  taosSetNonblocking(sockFd, 0);   
+  taosSetNonblocking(pSocket, 0);
 
 #else
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); 
+  ret = connect(pSocket->fd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
 #endif
 
   if (ret != 0) {
-    //printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
-    taosCloseSocket(sockFd);
-    sockFd = -1;
+    // printf("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   } else {
-    taosKeepTcpAlive(sockFd);
+    taosKeepTcpAlive(pSocket);
   }
 
-  return sockFd;
+  return pSocket;
 }
 
-int32_t taosKeepTcpAlive(SOCKET sockFd) {
+int32_t taosKeepTcpAlive(TdSocketPtr pSocket) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
   int32_t alive = 1;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) {
-    //printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_KEEPALIVE, (void *)&alive, sizeof(alive)) < 0) {
+    // printf("fd:%d setsockopt SO_KEEPALIVE failed: %d (%s)", sockFd, errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
 
 #ifndef __APPLE__
   // all fails on macosx
   int32_t probes = 3;
-  if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
-    //printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) {
+    // printf("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
 
   int32_t alivetime = 10;
-  if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
-    //printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPIDLE, (void *)&alivetime, sizeof(alivetime)) < 0) {
+    // printf("fd:%d setsockopt SO_KEEPIDLE failed: %d (%s)", sockFd, errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
 
   int32_t interval = 3;
-  if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
-    //printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, SOL_TCP, TCP_KEEPINTVL, (void *)&interval, sizeof(interval)) < 0) {
+    // printf("fd:%d setsockopt SO_KEEPINTVL failed: %d (%s)", sockFd, errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
-#endif // __APPLE__
+#endif  // __APPLE__
 
   int32_t nodelay = 1;
-  if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) {
-    //printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) {
+    // printf("fd:%d setsockopt TCP_NODELAY failed %d (%s)", sockFd, errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
 
   struct linger linger = {0};
   linger.l_onoff = 1;
   linger.l_linger = 3;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
-    //printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
-    taosCloseSocket(sockFd);
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) {
+    // printf("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno));
+    taosCloseSocket(&pSocket);
     return -1;
   }
 
   return 0;
 }
 
-SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
+TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
   struct sockaddr_in serverAdd;
-  SOCKET             sockFd;
+  SocketFd           fd;
   int32_t            reuse;
 
-  //printf("open tcp server socket:0x%x:%hu", ip, port);
+  // printf("open tcp server socket:0x%x:%hu", ip, port);
 
   bzero((char *)&serverAdd, sizeof(serverAdd));
   serverAdd.sin_family = AF_INET;
   serverAdd.sin_addr.s_addr = ip;
   serverAdd.sin_port = (uint16_t)htons(port);
 
-  if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
-    //printf("failed to open TCP socket: %d (%s)", errno, strerror(errno));
-    taosCloseSocketNoCheck(sockFd);
-    return -1;
+  if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
+    // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno));
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
   }
 
+  TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket));
+  if (pSocket == NULL) {
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
+  }
+  pSocket->refId = 0;
+  pSocket->fd = fd;
+
   /* set REUSEADDR option, so the portnumber can be re-used */
   reuse = 1;
-  if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
-    //printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) {
+    // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
   /* bind socket to server address */
-  if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
-    //printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
-    taosCloseSocket(sockFd);
-    return -1;
+  if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
+    // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  if (taosKeepTcpAlive(sockFd) < 0) {
-    //printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno));
-    taosCloseSocket(sockFd);
-    return -1;
+  if (taosKeepTcpAlive(pSocket) < 0) {
+    // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  if (listen(sockFd, 1024) < 0) {
-    //printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
-    taosCloseSocket(sockFd);
-    return -1;
+  if (listen(pSocket->fd, 1024) < 0) {
+    // printf("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
+    taosCloseSocket(&pSocket);
+    return NULL;
   }
 
-  return sockFd;
+  return (TdSocketServerPtr)pSocket;
 }
 
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr,
+                                       socklen_t *addrLen) {
+  if (pServerSocket == NULL || pServerSocket->fd < 0) {
+    return NULL;
+  }
+  SocketFd fd = accept(pServerSocket->fd, destAddr, addrLen);
+  if (fd == -1) {
+    // tError("TCP accept failure(%s)", strerror(errno));
+    return NULL;
+  }
+
+  TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket));
+  if (pSocket == NULL) {
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
+  }
+  pSocket->fd = fd;
+  pSocket->refId = 0;
+  return pSocket;
+}
 #define COPY_SIZE 32768
 // sendfile shall be used
 
-int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) {
+int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len) {
+  if (pSrcSocket == NULL || pSrcSocket->fd < 0 || pDestSocket == NULL || pDestSocket->fd < 0) {
+    return -1;
+  }
   int64_t leftLen;
   int64_t readLen, writeLen;
   char    temp[COPY_SIZE];
@@ -663,18 +742,18 @@ int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) {
     else
       readLen = COPY_SIZE;  // 4K
 
-    int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen);
+    int64_t retLen = taosReadMsg(pSrcSocket, temp, (int32_t)readLen);
     if (readLen != retLen) {
-      //printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s",
-      //       readLen, retLen, len, leftLen, strerror(errno));
+      // printf("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s",
+      //        readLen, retLen, len, leftLen, strerror(errno));
       return -1;
     }
 
-    writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen);
+    writeLen = taosWriteMsg(pDestSocket, temp, (int32_t)readLen);
 
     if (readLen != writeLen) {
-      //printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s",
-      //       readLen, writeLen, len, leftLen, strerror(errno));
+      // printf("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s",
+      //        readLen, writeLen, len, leftLen, strerror(errno));
       return -1;
     }
 
@@ -692,7 +771,7 @@ void taosBlockSIGPIPE() {
   sigaddset(&signal_mask, SIGPIPE);
   int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
   if (rc != 0) {
-    //printf("failed to block SIGPIPE");
+    // printf("failed to block SIGPIPE");
   }
 #endif
 }
@@ -706,7 +785,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
 
   int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result);
   if (result) {
-    struct sockaddr *   sa = result->ai_addr;
+    struct sockaddr    *sa = result->ai_addr;
     struct sockaddr_in *si = (struct sockaddr_in *)sa;
     struct in_addr      ia = si->sin_addr;
     uint32_t            ip = ia.s_addr;
@@ -715,12 +794,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
   } else {
 #ifdef EAI_SYSTEM
     if (ret == EAI_SYSTEM) {
-      //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno));
+      // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno));
     } else {
-      //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
+      // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
     }
 #else
-    //printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
+    // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
 #endif
     return 0xFFFFFFFF;
   }
@@ -730,7 +809,7 @@ int32_t taosGetFqdn(char *fqdn) {
   char hostname[1024];
   hostname[1023] = '\0';
   if (gethostname(hostname, 1023) == -1) {
-    //printf("failed to get hostname, reason:%s", strerror(errno));
+    // printf("failed to get hostname, reason:%s", strerror(errno));
     return -1;
   }
 
@@ -742,21 +821,21 @@ int32_t taosGetFqdn(char *fqdn) {
   // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return
   // immediately
   hints.ai_family = AF_INET;
-#else // __APPLE__
+#else   // __APPLE__
   hints.ai_flags = AI_CANONNAME;
-#endif // __APPLE__
+#endif  // __APPLE__
   int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
   if (!result) {
-    //printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+    // printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
     return -1;
   }
 
 #ifdef __APPLE__
   // refer to comments above
   strcpy(fqdn, hostname);
-#else // __APPLE__
+#else   // __APPLE__
   strcpy(fqdn, result->ai_canonname);
-#endif // __APPLE__
+#endif  // __APPLE__
   freeaddrinfo(result);
   return 0;
 }
@@ -793,7 +872,6 @@ void tinet_ntoa(char *ipstr, uint32_t ip) {
   sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
 }
 
-
 void taosIgnSIGPIPE() {
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 #else
@@ -809,7 +887,67 @@ void taosSetMaskSIGPIPE() {
   sigaddset(&signal_mask, SIGPIPE);
   int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL);
   if (rc != 0) {
-    //printf("failed to setmask SIGPIPE");
+    // printf("failed to setmask SIGPIPE");
   }
 #endif
-}
\ No newline at end of file
+}
+
+int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) {
+  if (pSocket == NULL || pSocket->fd < 0) {
+    return -1;
+  }
+  return getsockname(pSocket->fd, destAddr, addrLen);
+}
+
+
+TdEpollPtr taosCreateEpoll(int32_t size) {
+  EpollFd fd = -1;
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#else
+  fd = epoll_create(size);
+#endif
+  if (fd < 0) {
+    return NULL;
+  }
+
+  TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll));
+  if (pEpoll == NULL) {
+    taosCloseSocketNoCheck1(fd);
+    return NULL;
+  }
+  pEpoll->fd = fd;
+  pEpoll->refId = 0;
+  return pEpoll;
+}
+int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event) {
+  int32_t code = -1;
+  if (pEpoll == NULL || pEpoll->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#else
+  code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event);
+#endif
+  return code;
+}
+int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout) {
+  int32_t code = -1;
+  if (pEpoll == NULL || pEpoll->fd < 0) {
+    return -1;
+  }
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#else
+  code = epoll_wait(pEpoll->fd, event, maxEvents, timeout);
+#endif
+  return code;
+}
+int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
+  int32_t code;
+  if (ppEpoll == NULL || *ppEpoll == NULL || (*ppEpoll)->fd < 0) {
+    return -1;
+  }
+  code = taosCloseSocketNoCheck1((*ppEpoll)->fd);
+  (*ppEpoll)->fd = -1;
+  free(*ppEpoll);
+  return code;
+}

From 3b4149ba3268a9460f3c8fcfd8f52cacf929bd6a Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 9 Mar 2022 10:15:40 +0800
Subject: [PATCH 17/29] sync refactor

---
 source/libs/sync/inc/syncVoteMgr.h            |   6 +-
 source/libs/sync/src/syncVoteMgr.c            |  51 +++++-
 source/libs/sync/test/CMakeLists.txt          |  14 ++
 .../libs/sync/test/syncVotesRespondTest.cpp   | 156 ++++++++++++++++++
 4 files changed, 221 insertions(+), 6 deletions(-)
 create mode 100644 source/libs/sync/test/syncVotesRespondTest.cpp

diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h
index a769bfbccd..ae9cfe8d01 100644
--- a/source/libs/sync/inc/syncVoteMgr.h
+++ b/source/libs/sync/inc/syncVoteMgr.h
@@ -31,8 +31,8 @@ extern "C" {
 // SVotesGranted -----------------------------
 typedef struct SVotesGranted {
   SRaftId (*replicas)[TSDB_MAX_REPLICA];
-  int32_t    replicaNum;
   bool       isGranted[TSDB_MAX_REPLICA];
+  int32_t    replicaNum;
   int32_t    votes;
   SyncTerm   term;
   int32_t    quorum;
@@ -61,7 +61,9 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode);
 void           votesRespondDestory(SVotesRespond *pVotesRespond);
 bool           votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId);
 void           votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg);
-void           Reset(SVotesRespond *pVotesRespond, SyncTerm term);
+void           votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term);
+cJSON *        votesRespond2Json(SVotesRespond *pVotesRespond);
+char *         votesRespond2Str(SVotesRespond *pVotesRespond);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c
index 893a4e3a55..a2f10ce339 100644
--- a/source/libs/sync/src/syncVoteMgr.c
+++ b/source/libs/sync/src/syncVoteMgr.c
@@ -153,8 +153,8 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) {
 void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) {
   assert(pVotesRespond->term == pMsg->term);
   for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
-    if (syncUtilSameId(&(*pVotesRespond->replicas)[i], &pMsg->srcId)) {
-      assert(pVotesRespond->isRespond[i] == false);
+    if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) {
+      // assert(pVotesRespond->isRespond[i] == false);
       pVotesRespond->isRespond[i] = true;
       return;
     }
@@ -162,9 +162,52 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p
   assert(0);
 }
 
-void Reset(SVotesRespond *pVotesRespond, SyncTerm term) {
+void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) {
   pVotesRespond->term = term;
+  memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond));
+  /*
+    for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
+      pVotesRespond->isRespond[i] = false;
+    }
+  */
+}
+
+cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum);
+  cJSON *pReplicas = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
   for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
-    pVotesRespond->isRespond[i] = false;
+    cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i]));
   }
+  int  respondNum = 0;
+  int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum);
+  for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
+    arr[i] = pVotesRespond->isRespond[i];
+    if (pVotesRespond->isRespond[i]) {
+      respondNum++;
+    }
+  }
+  cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum);
+  free(arr);
+  cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond);
+  cJSON_AddNumberToObject(pRoot, "respondNum", respondNum);
+
+  snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term);
+  cJSON_AddStringToObject(pRoot, "term", u64buf);
+  snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode);
+  cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot);
+  return pJson;
+}
+
+char *votesRespond2Str(SVotesRespond *pVotesRespond) {
+  cJSON *pJson = votesRespond2Json(pVotesRespond);
+  char * serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
 }
\ No newline at end of file
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 18b7748105..8172a42a80 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -13,6 +13,7 @@ add_executable(syncIndexTest "")
 add_executable(syncInitTest "")
 add_executable(syncUtilTest "")
 add_executable(syncVotesGrantedTest "")
+add_executable(syncVotesRespondTest "")
 
 
 target_sources(syncTest
@@ -75,6 +76,10 @@ target_sources(syncVotesGrantedTest
     PRIVATE
     "syncVotesGrantedTest.cpp"
 )
+target_sources(syncVotesRespondTest
+    PRIVATE
+    "syncVotesRespondTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -152,6 +157,11 @@ target_include_directories(syncVotesGrantedTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncVotesRespondTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -214,6 +224,10 @@ target_link_libraries(syncVotesGrantedTest
     sync
     gtest_main
 )
+target_link_libraries(syncVotesRespondTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp
new file mode 100644
index 0000000000..74d42cd531
--- /dev/null
+++ b/source/libs/sync/test/syncVotesRespondTest.cpp
@@ -0,0 +1,156 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 3;
+int32_t  myIndex = 0;
+
+SRaftId    ids[TSDB_MAX_REPLICA];
+SSyncInfo  syncInfo;
+SSyncFSM*  pFsm;
+SSyncNode* pSyncNode;
+
+SSyncNode* syncNodeInit() {
+  syncInfo.vgId = 1234;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = pFsm;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
+  snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+  pCfg->myIndex = myIndex;
+  pCfg->replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    pCfg->nodeInfo[i].nodePort = ports[i];
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  }
+
+  pSyncNode = syncNodeOpen(&syncInfo);
+  assert(pSyncNode != NULL);
+
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->pSyncNode = pSyncNode;
+
+  return pSyncNode;
+}
+
+SSyncNode* syncInitTest() { return syncNodeInit(); }
+
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
+}
+
+int main(int argc, char** argv) {
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
+  tsAsyncLog = 0;
+  sDebugFlag = 143 + 64;
+
+  myIndex = 0;
+  if (argc >= 2) {
+    myIndex = atoi(argv[1]);
+  }
+
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  assert(ret == 0);
+
+  ret = syncEnvStart();
+  assert(ret == 0);
+
+  SSyncNode* pSyncNode = syncInitTest();
+  assert(pSyncNode != NULL);
+
+  char* serialized = syncNode2Str(pSyncNode);
+  printf("%s\n", serialized);
+  free(serialized);
+
+  initRaftId(pSyncNode);
+
+  SVotesRespond* pVotesRespond = votesRespondCreate(pSyncNode);
+  assert(pVotesRespond != NULL);
+
+  printf("---------------------------------------\n");
+  {
+    char* serialized = votesRespond2Str(pVotesRespond);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  SyncTerm term = 1234;
+  printf("---------------------------------------\n");
+  votesRespondReset(pVotesRespond, term);
+  {
+    char* serialized = votesRespond2Str(pVotesRespond);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  for (int i = 0; i < replicaNum; ++i) {
+    SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
+    reply->destId = pSyncNode->myRaftId;
+    reply->srcId = ids[i];
+    reply->term = term;
+    reply->voteGranted = true;
+
+    votesRespondAdd(pVotesRespond, reply);
+    {
+      char* serialized = votesRespond2Str(pVotesRespond);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+
+    votesRespondAdd(pVotesRespond, reply);
+    {
+      char* serialized = votesRespond2Str(pVotesRespond);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+  }
+
+  printf("---------------------------------------\n");
+  votesRespondReset(pVotesRespond, 123456789);
+  {
+    char* serialized = votesRespond2Str(pVotesRespond);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  votesRespondDestory(pVotesRespond);
+  return 0;
+}

From 6667446db3979a5c7930f0deaf40936a6b992f93 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 9 Mar 2022 10:46:31 +0800
Subject: [PATCH 18/29] [TD-13876]<fix>: forbid big-end machine.

---
 include/os/osSysinfo.h                 | 1 +
 source/dnode/mgmt/daemon/src/dmnMain.c | 5 +++++
 source/os/src/osSysinfo.c              | 9 +++++++++
 3 files changed, 15 insertions(+)

diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h
index 1ebad370b5..54a3cfef7b 100644
--- a/include/os/osSysinfo.h
+++ b/include/os/osSysinfo.h
@@ -33,6 +33,7 @@ typedef struct {
   SDiskSize size;
 } SDiskSpace;
 
+bool    taosCheckSystemIsSmallEnd();
 void    taosGetSystemInfo();
 int32_t taosGetEmail(char *email, int32_t maxLen);
 int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c
index df705898ca..7ba272453c 100644
--- a/source/dnode/mgmt/daemon/src/dmnMain.c
+++ b/source/dnode/mgmt/daemon/src/dmnMain.c
@@ -97,6 +97,11 @@ int32_t dmnRunDnode() {
 }
 
 int main(int argc, char const *argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    uError("TDengine does not run on non-small-end machines.");
+    return -1;
+  }
+
   if (dmnParseOption(argc, argv) != 0) {
     return -1;
   }
diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c
index ff9d5fb71b..02d7e6c0e9 100644
--- a/source/os/src/osSysinfo.c
+++ b/source/os/src/osSysinfo.c
@@ -16,6 +16,15 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
+bool taosCheckSystemIsSmallEnd() {
+    union check{
+        int16_t i;
+        char ch[2];
+    }c;
+    c.i=1;
+    return c.ch[0]==1;
+}
+
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 
 /*

From d3b1d5d8f1063928d958cff873bd639748eca668 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 9 Mar 2022 10:58:22 +0800
Subject: [PATCH 19/29] sync refactor

---
 source/libs/sync/inc/syncIndexMgr.h        |  49 +++++++
 source/libs/sync/src/syncIndexMgr.c        |  99 +++++++++++++
 source/libs/sync/test/CMakeLists.txt       |  14 ++
 source/libs/sync/test/syncIndexMgrTest.cpp | 156 +++++++++++++++++++++
 4 files changed, 318 insertions(+)
 create mode 100644 source/libs/sync/inc/syncIndexMgr.h
 create mode 100644 source/libs/sync/src/syncIndexMgr.c
 create mode 100644 source/libs/sync/test/syncIndexMgrTest.cpp

diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h
new file mode 100644
index 0000000000..7116ae9d46
--- /dev/null
+++ b/source/libs/sync/inc/syncIndexMgr.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_INDEX_MGR_H
+#define _TD_LIBS_SYNC_INDEX_MGR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "syncInt.h"
+#include "taosdef.h"
+
+// SIndexMgr -----------------------------
+typedef struct SSyncIndexMgr {
+  SRaftId (*replicas)[TSDB_MAX_REPLICA];
+  SyncIndex  index[TSDB_MAX_REPLICA];
+  int32_t    replicaNum;
+  SSyncNode *pSyncNode;
+} SSyncIndexMgr;
+
+SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode);
+void           syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr);
+void           syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr);
+void           syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index);
+SyncIndex      syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
+cJSON *        syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr);
+char *         syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_INDEX_MGR_H*/
diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c
new file mode 100644
index 0000000000..86b719ba1e
--- /dev/null
+++ b/source/libs/sync/src/syncIndexMgr.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncIndexMgr.h"
+#include "syncUtil.h"
+
+// SMatchIndex -----------------------------
+
+SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
+  SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr));
+  assert(pSyncIndexMgr != NULL);
+  memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr));
+
+  pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
+  pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
+  pSyncIndexMgr->pSyncNode = pSyncNode;
+  syncIndexMgrClear(pSyncIndexMgr);
+
+  return pSyncIndexMgr;
+}
+
+void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
+  if (pSyncIndexMgr != NULL) {
+    free(pSyncIndexMgr);
+  }
+}
+
+void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) {
+  memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index));
+  /*
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    pSyncIndexMgr->index[i] = 0;
+  }
+  */
+}
+
+void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) {
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
+      (pSyncIndexMgr->index)[i] = index;
+      return;
+    }
+  }
+  assert(0);
+}
+
+SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
+      SyncIndex idx = (pSyncIndexMgr->index)[i];
+      return idx;
+    }
+  }
+  assert(0);
+}
+
+cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum);
+  cJSON *pReplicas = cJSON_CreateArray();
+  cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i]));
+  }
+  int  respondNum = 0;
+  int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum);
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    arr[i] = pSyncIndexMgr->index[i];
+  }
+  cJSON *pIsRespond = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
+  free(arr);
+  snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
+  cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "pSyncIndexMgr", pRoot);
+  return pJson;
+}
+
+char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
+  cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
+  char  *serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
\ No newline at end of file
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 8172a42a80..5a5186c7e2 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -14,6 +14,7 @@ add_executable(syncInitTest "")
 add_executable(syncUtilTest "")
 add_executable(syncVotesGrantedTest "")
 add_executable(syncVotesRespondTest "")
+add_executable(syncIndexMgrTest "")
 
 
 target_sources(syncTest
@@ -80,6 +81,10 @@ target_sources(syncVotesRespondTest
     PRIVATE
     "syncVotesRespondTest.cpp"
 )
+target_sources(syncIndexMgrTest
+    PRIVATE
+    "syncIndexMgrTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -162,6 +167,11 @@ target_include_directories(syncVotesRespondTest
     "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncIndexMgrTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -228,6 +238,10 @@ target_link_libraries(syncVotesRespondTest
     sync
     gtest_main
 )
+target_link_libraries(syncIndexMgrTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp
new file mode 100644
index 0000000000..3edde509f8
--- /dev/null
+++ b/source/libs/sync/test/syncIndexMgrTest.cpp
@@ -0,0 +1,156 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+#include "syncVoteMgr.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
+int32_t  replicaNum = 3;
+int32_t  myIndex = 0;
+
+SRaftId    ids[TSDB_MAX_REPLICA];
+SSyncInfo  syncInfo;
+SSyncFSM*  pFsm;
+SSyncNode* pSyncNode;
+
+SSyncNode* syncNodeInit() {
+  syncInfo.vgId = 1234;
+  syncInfo.rpcClient = gSyncIO->clientRpc;
+  syncInfo.FpSendMsg = syncIOSendMsg;
+  syncInfo.queue = gSyncIO->pMsgQ;
+  syncInfo.FpEqMsg = syncIOEqMsg;
+  syncInfo.pFsm = pFsm;
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
+  snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
+
+  SSyncCfg* pCfg = &syncInfo.syncCfg;
+  pCfg->myIndex = myIndex;
+  pCfg->replicaNum = replicaNum;
+
+  for (int i = 0; i < replicaNum; ++i) {
+    pCfg->nodeInfo[i].nodePort = ports[i];
+    snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
+    // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+  }
+
+  pSyncNode = syncNodeOpen(&syncInfo);
+  assert(pSyncNode != NULL);
+
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
+  gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
+  gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
+  gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
+  gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
+  gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
+  gSyncIO->pSyncNode = pSyncNode;
+
+  return pSyncNode;
+}
+
+SSyncNode* syncInitTest() { return syncNodeInit(); }
+
+void initRaftId(SSyncNode* pSyncNode) {
+  for (int i = 0; i < replicaNum; ++i) {
+    ids[i] = pSyncNode->replicasId[i];
+    char* s = syncUtilRaftId2Str(&ids[i]);
+    printf("raftId[%d] : %s\n", i, s);
+    free(s);
+  }
+}
+
+int main(int argc, char** argv) {
+  // taosInitLog((char *)"syncTest.log", 100000, 10);
+  tsAsyncLog = 0;
+  sDebugFlag = 143 + 64;
+
+  myIndex = 0;
+  if (argc >= 2) {
+    myIndex = atoi(argv[1]);
+  }
+
+  int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
+  assert(ret == 0);
+
+  ret = syncEnvStart();
+  assert(ret == 0);
+
+  SSyncNode* pSyncNode = syncInitTest();
+  assert(pSyncNode != NULL);
+
+  char* serialized = syncNode2Str(pSyncNode);
+  printf("%s\n", serialized);
+  free(serialized);
+
+  initRaftId(pSyncNode);
+
+  SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
+  assert(pVotesGranted != NULL);
+
+  printf("---------------------------------------\n");
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  SyncTerm term = 1234;
+  printf("---------------------------------------\n");
+  voteGrantedReset(pVotesGranted, term);
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  for (int i = 0; i < replicaNum; ++i) {
+    SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
+    reply->destId = pSyncNode->myRaftId;
+    reply->srcId = ids[i];
+    reply->term = term;
+    reply->voteGranted = true;
+
+    voteGrantedVote(pVotesGranted, reply);
+    {
+      char* serialized = voteGranted2Str(pVotesGranted);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+
+    voteGrantedVote(pVotesGranted, reply);
+    {
+      char* serialized = voteGranted2Str(pVotesGranted);
+      assert(serialized != NULL);
+      printf("%s\n", serialized);
+      free(serialized);
+    }
+  }
+
+  printf("---------------------------------------\n");
+  voteGrantedReset(pVotesGranted, 123456789);
+  {
+    char* serialized = voteGranted2Str(pVotesGranted);
+    assert(serialized != NULL);
+    printf("%s\n", serialized);
+    free(serialized);
+  }
+
+  voteGrantedDestroy(pVotesGranted);
+  return 0;
+}

From 2fcc970d79727ba9ac3d06e4ff1fc67e771d960e Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 9 Mar 2022 11:27:22 +0800
Subject: [PATCH 20/29] sync refactor

---
 source/libs/sync/inc/syncInt.h             |  7 +++-
 source/libs/sync/src/syncIndexMgr.c        |  5 ++-
 source/libs/sync/test/syncIndexMgrTest.cpp | 49 ++++++++--------------
 3 files changed, 25 insertions(+), 36 deletions(-)

diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 447b75a5e8..8b77e292c4 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -103,6 +103,9 @@ typedef struct SVotesGranted SVotesGranted;
 struct SVotesRespond;
 typedef struct SVotesRespond SVotesRespond;
 
+struct SSyncIndexMgr;
+typedef struct SSyncIndexMgr SSyncIndexMgr;
+
 typedef struct SRaftId {
   SyncNodeId  addr;  // typedef uint64_t SyncNodeId;
   SyncGroupId vgId;  // typedef int32_t  SyncGroupId;
@@ -148,8 +151,8 @@ typedef struct SSyncNode {
   SVotesRespond* pVotesRespond;
 
   // tla+ leader vars
-  SHashObj* pNextIndex;
-  SHashObj* pMatchIndex;
+  SSyncIndexMgr* pNextIndex;
+  SSyncIndexMgr* pMatchIndex;
 
   // tla+ log vars
   SSyncLogStore* pLogStore;
diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c
index 86b719ba1e..fff54638e2 100644
--- a/source/libs/sync/src/syncIndexMgr.c
+++ b/source/libs/sync/src/syncIndexMgr.c
@@ -81,8 +81,9 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
   for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
     arr[i] = pSyncIndexMgr->index[i];
   }
-  cJSON *pIsRespond = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
+  cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
   free(arr);
+  cJSON_AddItemToObject(pRoot, "index", pIndex);
   snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
   cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
 
@@ -93,7 +94,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
 
 char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
   cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
-  char  *serialized = cJSON_Print(pJson);
+  char * serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
\ No newline at end of file
diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp
index 3edde509f8..4e4cd9222b 100644
--- a/source/libs/sync/test/syncIndexMgrTest.cpp
+++ b/source/libs/sync/test/syncIndexMgrTest.cpp
@@ -1,4 +1,5 @@
-#include <gtest/gtest.h>
+#include "syncIndexMgr.h"
+//#include <gtest/gtest.h>
 #include <stdio.h>
 #include "syncEnv.h"
 #include "syncIO.h"
@@ -97,60 +98,44 @@ int main(int argc, char** argv) {
 
   initRaftId(pSyncNode);
 
-  SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
-  assert(pVotesGranted != NULL);
+  SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode);
+  assert(pSyncIndexMgr != NULL);
 
   printf("---------------------------------------\n");
   {
-    char* serialized = voteGranted2Str(pVotesGranted);
+    char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     free(serialized);
   }
 
-  SyncTerm term = 1234;
+  syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100);
+  syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200);
+  syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300);
+
   printf("---------------------------------------\n");
-  voteGrantedReset(pVotesGranted, term);
   {
-    char* serialized = voteGranted2Str(pVotesGranted);
+    char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     free(serialized);
   }
 
-  for (int i = 0; i < replicaNum; ++i) {
-    SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
-    reply->destId = pSyncNode->myRaftId;
-    reply->srcId = ids[i];
-    reply->term = term;
-    reply->voteGranted = true;
-
-    voteGrantedVote(pVotesGranted, reply);
-    {
-      char* serialized = voteGranted2Str(pVotesGranted);
-      assert(serialized != NULL);
-      printf("%s\n", serialized);
-      free(serialized);
-    }
-
-    voteGrantedVote(pVotesGranted, reply);
-    {
-      char* serialized = voteGranted2Str(pVotesGranted);
-      assert(serialized != NULL);
-      printf("%s\n", serialized);
-      free(serialized);
-    }
+  printf("---------------------------------------\n");
+  for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
+    SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]);
+    printf("index %d : %lu \n", i, idx);
   }
 
+  syncIndexMgrClear(pSyncIndexMgr);
   printf("---------------------------------------\n");
-  voteGrantedReset(pVotesGranted, 123456789);
   {
-    char* serialized = voteGranted2Str(pVotesGranted);
+    char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
     assert(serialized != NULL);
     printf("%s\n", serialized);
     free(serialized);
   }
 
-  voteGrantedDestroy(pVotesGranted);
+  syncIndexMgrDestroy(pSyncIndexMgr);
   return 0;
 }

From 5afb3e2c00804cbdbdb7fb352ba384f8e2b0c3f5 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 9 Mar 2022 15:47:29 +0800
Subject: [PATCH 21/29] [TD-13761]<fix>: redefine dir api.

---
 include/os/osDir.h                     | 22 +++++++++
 source/dnode/vnode/src/tsdb/tsdbFile.c |  2 +-
 source/libs/index/src/index_tfile.c    | 12 ++---
 source/libs/tfs/inc/tfsInt.h           |  2 +-
 source/libs/tfs/src/tfs.c              | 67 +++++++++++++-------------
 source/libs/wal/src/walMeta.c          | 24 ++++-----
 source/os/src/osDir.c                  | 48 ++++++++++++++++++
 source/os/src/osSemaphore.c            | 56 ++++++++++-----------
 source/os/src/osTimer.c                |  8 +--
 tests/test/c/tmqDemo.c                 | 17 ++++---
 10 files changed, 165 insertions(+), 93 deletions(-)

diff --git a/include/os/osDir.h b/include/os/osDir.h
index 223c603352..6cf28fb878 100644
--- a/include/os/osDir.h
+++ b/include/os/osDir.h
@@ -16,10 +16,24 @@
 #ifndef _TD_OS_DIR_H_
 #define _TD_OS_DIR_H_
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define opendir OPENDIR_FUNC_TAOS_FORBID
+    #define readdir READDIR_FUNC_TAOS_FORBID
+    #define closedir CLOSEDIR_FUNC_TAOS_FORBID
+    #define dirname DIRNAME_FUNC_TAOS_FORBID
+    #undef basename
+    #define basename BASENAME_FUNC_TAOS_FORBID
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct TdDir *TdDirPtr;
+typedef struct TdDirEntry *TdDirEntryPtr;
+
+
 void    taosRemoveDir(const char *dirname);
 bool    taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
@@ -27,6 +41,14 @@ void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
+char*   taosDirName(char *dirname);
+char*   taosDirEntryBaseName(char *dirname);
+
+TdDirPtr      taosOpenDir(const char *dirname);
+TdDirEntryPtr taosReadDir(TdDirPtr pDir);
+bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
+char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
+int32_t       taosCloseDir(TdDirPtr pDir);
 
 #ifdef __cplusplus
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c
index 74fb8c1c1f..00e97c7b61 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile.c
@@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T
     if (errno == ENOENT) {
       // Try to create directory recursively
       char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
-      if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) {
+      if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) {
         tfree(s);
         return -1;
       }
diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c
index fd267fbf03..89f3f8ba8a 100644
--- a/source/libs/index/src/index_tfile.c
+++ b/source/libs/index/src/index_tfile.c
@@ -722,13 +722,13 @@ static SArray* tfileGetFileList(const char* path) {
   uint32_t version;
   SArray*  files = taosArrayInit(4, sizeof(void*));
 
-  DIR* dir = opendir(path);
-  if (NULL == dir) {
+  TdDirPtr pDir = taosOpenDir(path);
+  if (NULL == pDir) {
     return NULL;
   }
-  struct dirent* entry;
-  while ((entry = readdir(dir)) != NULL) {
-    char* file = entry->d_name;
+  TdDirEntryPtr pDirEntry;
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char* file = taosGetDirEntryName(pDirEntry);
     if (0 != tfileParseFileName(file, &suid, buf, &version)) {
       continue;
     }
@@ -738,7 +738,7 @@ static SArray* tfileGetFileList(const char* path) {
     sprintf(buf, "%s/%s", path, file);
     taosArrayPush(files, &buf);
   }
-  closedir(dir);
+  taosCloseDir(pDir);
 
   taosArraySort(files, tfileCompare);
   tfileRmExpireFile(files);
diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h
index 913f34d6c2..f16d0445c6 100644
--- a/source/libs/tfs/inc/tfsInt.h
+++ b/source/libs/tfs/inc/tfsInt.h
@@ -59,7 +59,7 @@ typedef struct STfsDir {
   SDiskID   did;
   char      dirname[TSDB_FILENAME_LEN];
   STfsFile  tfile;
-  DIR      *dir;
+  TdDirPtr  pDir;
   STfs     *pTfs;
 } STfsDir;
 
diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c
index 2579490791..c46989dc5d 100644
--- a/source/libs/tfs/src/tfs.c
+++ b/source/libs/tfs/src/tfs.c
@@ -192,14 +192,14 @@ void tfsBasename(const STfsFile *pFile, char *dest) {
   char tname[TSDB_FILENAME_LEN] = "\0";
 
   tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
-  tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN);
+  tstrncpy(dest, taosDirEntryBaseName(tname), TSDB_FILENAME_LEN);
 }
 
 void tfsDirname(const STfsFile *pFile, char *dest) {
   char tname[TSDB_FILENAME_LEN] = "\0";
 
   tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
-  tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
+  tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN);
 }
 
 int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); }
@@ -233,7 +233,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) {
       // the pointer directly in this recursion.
       // See
       // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html
-      char *dir = strdup(dirname(s));
+      char *dir = strdup(taosDirName(s));
 
       if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) {
         free(s);
@@ -324,45 +324,46 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) {
   return pDir;
 }
 
-const STfsFile *tfsReaddir(STfsDir *pDir) {
-  if (pDir == NULL || pDir->dir == NULL) return NULL;
+const STfsFile *tfsReaddir(STfsDir *pTfsDir) {
+  if (pTfsDir == NULL || pTfsDir->pDir == NULL) return NULL;
   char bname[TMPNAME_LEN * 2] = "\0";
 
   while (true) {
-    struct dirent *dp = NULL;
-    dp = readdir(pDir->dir);
-    if (dp != NULL) {
+    TdDirEntryPtr pDirEntry = NULL;
+    pDirEntry = taosReadDir(pTfsDir->pDir);
+    if (pDirEntry != NULL) {
       // Skip . and ..
-      if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
+      char *name = taosGetDirEntryName(pDirEntry);
+      if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
 
-      if (pDir->dirname == NULL || pDir->dirname[0] == 0) {
-        snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name);
+      if (pTfsDir->dirname == NULL || pTfsDir->dirname[0] == 0) {
+        snprintf(bname, TMPNAME_LEN * 2, "%s", name);
       } else {
-        snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name);
+        snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pTfsDir->dirname, TD_DIRSEP, name);
       }
 
-      tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname);
-      return &pDir->tfile;
+      tfsInitFile(pTfsDir->pTfs, &pTfsDir->tfile, pTfsDir->did, bname);
+      return &pTfsDir->tfile;
     }
 
-    if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) {
+    if (tfsOpendirImpl(pTfsDir->pTfs, pTfsDir) < 0) {
       return NULL;
     }
 
-    if (pDir->dir == NULL) {
+    if (pTfsDir->pDir == NULL) {
       terrno = TSDB_CODE_SUCCESS;
       return NULL;
     }
   }
 }
 
-void tfsClosedir(STfsDir *pDir) {
-  if (pDir) {
-    if (pDir->dir != NULL) {
-      closedir(pDir->dir);
-      pDir->dir = NULL;
+void tfsClosedir(STfsDir *pTfsDir) {
+  if (pTfsDir) {
+    if (pTfsDir->pDir != NULL) {
+      taosCloseDir(pTfsDir->pDir);
+      pTfsDir->pDir = NULL;
     }
-    free(pDir);
+    free(pTfsDir);
   }
 }
 
@@ -487,29 +488,29 @@ static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) {
   return pDisk;
 }
 
-static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) {
+static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) {
   STfsDisk *pDisk = NULL;
   char      adir[TMPNAME_LEN * 2] = "\0";
 
-  if (pDir->dir != NULL) {
-    closedir(pDir->dir);
-    pDir->dir = NULL;
+  if (pTfsDir->pDir != NULL) {
+    taosCloseDir(pTfsDir->pDir);
+    pTfsDir->pDir = NULL;
   }
 
   while (true) {
-    pDisk = tfsNextDisk(pTfs, &pDir->iter);
+    pDisk = tfsNextDisk(pTfs, &pTfsDir->iter);
     if (pDisk == NULL) return 0;
 
-    pDir->did.level = pDisk->level;
-    pDir->did.id = pDisk->id;
+    pTfsDir->did.level = pDisk->level;
+    pTfsDir->did.id = pDisk->id;
 
     if (pDisk->path == NULL || pDisk->path[0] == 0) {
-      snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname);
+      snprintf(adir, TMPNAME_LEN * 2, "%s", pTfsDir->dirname);
     } else {
-      snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname);
+      snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pTfsDir->dirname);
     }
-    pDir->dir = opendir(adir);
-    if (pDir->dir != NULL) break;
+    pTfsDir->pDir = taosOpenDir(adir);
+    if (pTfsDir->pDir != NULL) break;
   }
 
   return 0;
diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c
index 2a4f3497e4..248f758787 100644
--- a/source/libs/wal/src/walMeta.c
+++ b/source/libs/wal/src/walMeta.c
@@ -130,16 +130,16 @@ int walCheckAndRepairMeta(SWal* pWal) {
   regcomp(&logRegPattern, logPattern, REG_EXTENDED);
   regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
 
-  DIR* dir = opendir(pWal->path);
-  if (dir == NULL) {
+  TdDirPtr pDir = taosOpenDir(pWal->path);
+  if (pDir == NULL) {
     wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     return -1;
   }
 
   // scan log files and build new meta
-  struct dirent* ent;
-  while ((ent = readdir(dir)) != NULL) {
-    char* name = basename(ent->d_name);
+  TdDirEntryPtr pDirEntry;
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry));
     int   code = regexec(&logRegPattern, name, 0, NULL, 0);
     if (code == 0) {
       SWalFileInfo fileInfo;
@@ -149,7 +149,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
     }
   }
 
-  closedir(dir);
+  taosCloseDir(pDir);
   regfree(&logRegPattern);
   regfree(&idxRegPattern);
 
@@ -337,25 +337,25 @@ static int walFindCurMetaVer(SWal* pWal) {
   regex_t     walMetaRegexPattern;
   regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED);
 
-  DIR* dir = opendir(pWal->path);
-  if (dir == NULL) {
+  TdDirPtr pDir = taosOpenDir(pWal->path);
+  if (pDir == NULL) {
     wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     return -1;
   }
 
-  struct dirent* ent;
+  TdDirEntryPtr pDirEntry;
 
   // find existing meta-ver[x].json
   int metaVer = -1;
-  while ((ent = readdir(dir)) != NULL) {
-    char* name = basename(ent->d_name);
+  while ((pDirEntry = taosReadDir(pDir)) != NULL) {
+    char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry));
     int   code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
     if (code == 0) {
       sscanf(name, "meta-ver%d", &metaVer);
       break;
     }
   }
-  closedir(dir);
+  taosCloseDir(pDir);
   regfree(&walMetaRegexPattern);
   return metaVer;
 }
diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c
index 7d7382d83f..b4058b3c0e 100644
--- a/source/os/src/osDir.c
+++ b/source/os/src/osDir.c
@@ -14,6 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
+#define ALLOW_FORBID_FUNC
 
 #include "os.h"
 #include "osString.h"
@@ -36,6 +37,10 @@
 #include <unistd.h>
 #include <wordexp.h>
 
+typedef struct dirent dirent;
+typedef struct DIR TdDir;
+typedef struct dirent TdDirent;
+
 void taosRemoveDir(const char *dirname) {
   DIR *dir = opendir(dirname);
   if (dir == NULL) return;
@@ -149,4 +154,47 @@ bool taosIsDir(const char *dirname) {
   return false;
 }
 
+char* taosDirName(char *name) {
+  return dirname(name);
+}
+
+char* taosDirEntryBaseName(char *name) {
+  return basename(name);
+}
+
+TdDirPtr taosOpenDir(const char *dirname) {
+  if (dirname == NULL) {
+    return NULL;
+  }
+  return (TdDirPtr)opendir(dirname);
+}
+
+TdDirEntryPtr taosReadDir(TdDirPtr pDir) {
+  if (pDir == NULL) {
+    return NULL;
+  }
+  return (TdDirEntryPtr)readdir((DIR*)pDir);
+}
+
+bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) {
+  if (pDirEntry == NULL) {
+    return false;
+  }
+  return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0;
+}
+
+char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) {
+  if (pDirEntry == NULL) {
+    return NULL;
+  }
+  return ((dirent*)pDirEntry)->d_name;
+}
+
+int32_t taosCloseDir(TdDirPtr pDir) {
+  if (pDir == NULL) {
+    return -1;
+  }
+  return closedir((DIR*)pDir);
+}
+
 #endif
diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c
index 0d7066b5c8..e5b506e8d8 100644
--- a/source/os/src/osSemaphore.c
+++ b/source/os/src/osSemaphore.c
@@ -99,7 +99,7 @@ static void *sem_thread_routine(void *arg) {
   sem_port = mach_task_self();
   kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0);
   if (ret != KERN_SUCCESS) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__);
+    fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
     sem_inited = -1;
     return NULL;
   }
@@ -112,7 +112,7 @@ static void once_init(void) {
   int r = 0;
   r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL);
   if (r) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__);
+    fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
     return;
   }
   while (sem_inited == 0) {
@@ -139,14 +139,14 @@ struct tsem_s {
 };
 
 int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
-  // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem);
+  // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
   if (*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p));
   if (!p) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 
@@ -162,7 +162,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
     p->val = value;
   } while (0);
   if (r) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 #elif defined(SEM_USE_POSIX)
@@ -181,27 +181,27 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
     int e = errno;
     if (e == EEXIST) continue;
     if (e == EINTR) continue;
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e,
+    fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e,
             strerror(e));
     abort();
   } while (p->sem == SEM_FAILED);
 #elif defined(SEM_USE_SEM)
   pthread_once(&sem_once, once_init);
   if (sem_inited != 1) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     errno = ENOMEM;
     return -1;
   }
   kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value);
   if (ret != KERN_SUCCESS) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     // we fail-fast here, because we have less-doc about semaphore_create for the moment
     abort();
   }
 #else   // SEM_USE_PTHREAD
   p->sem = dispatch_semaphore_create(value);
   if (p->sem == NULL) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 #endif  // SEM_USE_PTHREAD
@@ -215,28 +215,28 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
 
 int tsem_wait(tsem_t *sem) {
   if (!*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   struct tsem_s *p = *sem;
   if (!p->valid) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 #ifdef SEM_USE_PTHREAD
   if (pthread_mutex_lock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   p->val -= 1;
   if (p->val < 0) {
     if (pthread_cond_wait(&p->cond, &p->lock)) {
-      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
       abort();
     }
   }
   if (pthread_mutex_unlock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   return 0;
@@ -251,28 +251,28 @@ int tsem_wait(tsem_t *sem) {
 
 int tsem_post(tsem_t *sem) {
   if (!*sem) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   struct tsem_s *p = *sem;
   if (!p->valid) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 #ifdef SEM_USE_PTHREAD
   if (pthread_mutex_lock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   p->val += 1;
   if (p->val <= 0) {
     if (pthread_cond_signal(&p->cond)) {
-      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+      fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
       abort();
     }
   }
   if (pthread_mutex_unlock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   return 0;
@@ -286,34 +286,34 @@ int tsem_post(tsem_t *sem) {
 }
 
 int tsem_destroy(tsem_t *sem) {
-  // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem);
+  // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
   if (!*sem) {
-    // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
+    // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     // abort();
     return 0;
   }
   struct tsem_s *p = *sem;
   if (!p->valid) {
-    // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
+    // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     // abort();
     return 0;
   }
 #ifdef SEM_USE_PTHREAD
   if (pthread_mutex_lock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   p->valid = 0;
   if (pthread_cond_destroy(&p->cond)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   if (pthread_mutex_unlock(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
   if (pthread_mutex_destroy(&p->lock)) {
-    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
+    fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
     abort();
   }
 #elif defined(SEM_USE_POSIX)
@@ -322,7 +322,7 @@ int tsem_destroy(tsem_t *sem) {
   int r = sem_unlink(name);
   if (r) {
     int e = errno;
-    fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e,
+    fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e,
             strerror(e));
     abort();
   }
diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c
index bb526e0ba0..6b60923189 100644
--- a/source/os/src/osTimer.c
+++ b/source/os/src/osTimer.c
@@ -79,7 +79,7 @@ static void* timer_routine(void* arg) {
     struct kevent64_s kev[10] = {0};
     r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to);
     if (r != 0) {
-      fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__);
+      fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
       abort();
     }
     timer_callback(SIGALRM);  // just mock
@@ -97,14 +97,14 @@ int taosInitTimer(void (*callback)(int), int ms) {
 
   timer_kq = kqueue();
   if (timer_kq == -1) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__);
+    fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
     // since no caller of this func checks the return value for the moment
     abort();
   }
 
   r = pthread_create(&timer_thread, NULL, timer_routine, NULL);
   if (r) {
-    fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__);
+    fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
     // since no caller of this func checks the return value for the moment
     abort();
   }
@@ -116,7 +116,7 @@ void taosUninitTimer() {
   timer_stop = 1;
   r = pthread_join(timer_thread, NULL);
   if (r) {
-    fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__);
+    fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
     // since no caller of this func checks the return value for the moment
     abort();
   }
diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c
index 3eb8e60d56..cb7d7c67ce 100644
--- a/tests/test/c/tmqDemo.c
+++ b/tests/test/c/tmqDemo.c
@@ -192,11 +192,11 @@ static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }
 // calc dir size (not include itself 4096Byte)
 int64_t getDirectorySize(char *dir)
 {
-    DIR *dp;
-    struct dirent *entry;
+    TdDirPtr pDir;
+    TdDirEntryPtr pDirEntry;
     int64_t totalSize=0;
 
-    if ((dp = opendir(dir)) == NULL) {
+    if ((pDir = taosOpenDir(dir)) == NULL) {
         fprintf(stderr, "Cannot open dir: %s\n", dir);
         return -1; 
     }
@@ -204,26 +204,27 @@ int64_t getDirectorySize(char *dir)
     //lstat(dir, &statbuf);
     //totalSize+=statbuf.st_size;
 
-    while ((entry = readdir(dp)) != NULL) {
+    while ((pDirEntry = taosReadDir(pDir)) != NULL) {
         char subdir[1024];
-        sprintf(subdir, "%s/%s", dir, entry->d_name);
+        char* fileName = taosGetDirEntryName(pDirEntry);
+        sprintf(subdir, "%s/%s", dir, fileName);
 
         //printf("===d_name: %s\n", entry->d_name);
         if (taosIsDir(subdir)) {
-            if (strcmp(".", entry->d_name) == 0 || strcmp("..", entry->d_name) == 0) {
+            if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) {
                 continue;
             }
 
             int64_t subDirSize = getDirectorySize(subdir);
             totalSize+=subDirSize;
-        } else if (0 == strcmp(strchr(entry->d_name, '.'), ".log")) { // only calc .log file size, and not include .idx file	
+        } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file	
             int64_t file_size = 0;
             taosStatFile(subdir, &file_size, NULL);
             totalSize+=file_size;
         }
     }
 
-    closedir(dp);
+    taosCloseDir(pDir);
     return totalSize;
 }
 

From aad01e0155a02ceb461a575f7ef628359ff443bc Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Wed, 9 Mar 2022 16:38:26 +0800
Subject: [PATCH 22/29] add SSmaStat in tsdb to demonstrate window valid status

---
 include/common/taosdef.h                  |   7 +-
 source/dnode/vnode/src/inc/tsdbDef.h      |   1 +
 source/dnode/vnode/src/inc/tsdbFS.h       |   5 +-
 source/dnode/vnode/src/inc/tsdbSma.h      |  11 +-
 source/dnode/vnode/src/meta/metaBDBImpl.c |   8 +-
 source/dnode/vnode/src/tsdb/tsdbBDBImpl.c |  14 +++
 source/dnode/vnode/src/tsdb/tsdbMain.c    |   1 +
 source/dnode/vnode/src/tsdb/tsdbSma.c     | 128 ++++++++++++++++++++++
 8 files changed, 163 insertions(+), 12 deletions(-)
 create mode 100644 source/dnode/vnode/src/tsdb/tsdbBDBImpl.c

diff --git a/include/common/taosdef.h b/include/common/taosdef.h
index 69c2618ac8..7788dfd871 100644
--- a/include/common/taosdef.h
+++ b/include/common/taosdef.h
@@ -50,12 +50,17 @@ typedef enum {
   TSDB_CHECK_ITEM_MAX
 } ECheckItemType;
 
-typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig;
+typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2} TDUpdateConfig;
 typedef enum {
   TSDB_STATIS_OK = 0,    // statis part exist and load successfully
   TSDB_STATIS_NONE = 1,  // statis part not exist
 } ETsdbStatisStatus;
 
+typedef enum {
+  TSDB_SMA_STAT_OK = 0,       // ready to provide service
+  TSDB_SMA_STAT_EXPIRED = 1,  // not ready or expired
+} ETsdbSmaStat;
+
 extern char *qtypeStr[];
 
 #ifdef __cplusplus
diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h
index 96a76ea7d4..1451ac9685 100644
--- a/source/dnode/vnode/src/inc/tsdbDef.h
+++ b/source/dnode/vnode/src/inc/tsdbDef.h
@@ -52,6 +52,7 @@ struct STsdb {
   STsdbFS *             fs;
   SMeta *               pMeta;
   STfs *                pTfs;
+  SSmaStat *            pSmaStat;
 };
 
 #define REPO_ID(r) ((r)->vgId)
diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h
index 641255a294..173e991631 100644
--- a/source/dnode/vnode/src/inc/tsdbFS.h
+++ b/source/dnode/vnode/src/inc/tsdbFS.h
@@ -42,7 +42,10 @@ typedef struct {
 typedef struct {
   STsdbFSMeta meta;  // FS meta
   SArray *    df;    // data file array
-  SArray *    smaf;  // sma data file array
+
+  // SArray *    v2f100.tsma.index_name
+  
+  SArray *    smaf;  // sma data file array    v2f1900.tsma.index_name
 } SFSStatus;
 
 typedef struct {
diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h
index e4de7a6685..6e4ad909ae 100644
--- a/source/dnode/vnode/src/inc/tsdbSma.h
+++ b/source/dnode/vnode/src/inc/tsdbSma.h
@@ -16,6 +16,8 @@
 #ifndef _TD_TSDB_SMA_H_
 #define _TD_TSDB_SMA_H_
 
+typedef struct SSmaStat SSmaStat;
+
 // insert/update interface
 int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData);
 int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData);
@@ -26,13 +28,14 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData);
 int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult);
 
 // management interface
-int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result);
+int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg);
+int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
 int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-
-
-
+int32_t tsdbFreeSmaState(SSmaStat *pSmaStat);
 
 // internal func
+
+
 static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) {
   int32_t len = 0;
   len += taosEncodeFixedU64(pData, tableUid);
diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c
index f49515412b..efdb3e0fe4 100644
--- a/source/dnode/vnode/src/meta/metaBDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaBDBImpl.c
@@ -923,6 +923,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
   SMetaDB *pDB = pMeta->pDB;
   DBC *    pCur = NULL;
   DBT      pkey = {0}, pval = {0};
+  uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP;
   int      ret;
 
   pUids = taosArrayInit(16, sizeof(tb_uid_t));
@@ -941,13 +942,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
   void *pBuf = NULL;
 
   // TODO: lock?
-  while (true) {
-    ret = pCur->get(pCur, &pkey, &pval, isDup ? DB_NEXT_DUP : DB_NEXT_NODUP);
-    if(ret == 0) {
+  while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) {
       taosArrayPush(pUids, pkey.data);
-      continue;
-    }
-    break;
   }
 
   if (pCur) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c
new file mode 100644
index 0000000000..f2f48bbc8a
--- /dev/null
+++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c
@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c
index 2d8c470113..1b3e00f090 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMain.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMain.c
@@ -89,6 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
 static void tsdbFree(STsdb *pTsdb) {
   if (pTsdb) {
     tsdbFreeFS(pTsdb->fs);
+    tsdbFreeSmaState(pTsdb->pSmaStat);
     tfree(pTsdb->path);
     free(pTsdb);
   }
diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c
index b465dc3a88..5cfa597eb2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSma.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSma.c
@@ -21,6 +21,10 @@
 
 #define SMA_STORE_SINGLE_BLOCKS  // store SMA data by single block or multiple blocks
 
+#define SMA_STATE_HASH_SLOT      4
+#define SMA_STATE_ITEM_HASH_SLOT 32
+
+#define SMA_TEST_INDEX_NAME "smaTestIndexName"  // TODO: just for test
 typedef enum {
   SMA_STORAGE_LEVEL_TSDB = 0,     // store TSma in dir  e.g. vnode${N}/tsdb/.tsma
   SMA_STORAGE_LEVEL_DFILESET = 1  // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name}
@@ -48,6 +52,22 @@ typedef struct {
   // TODO
 } STSmaReadH;
 
+typedef struct {
+  /**
+   * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
+   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
+   * without information about its previous state.
+   *    - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
+   * Streaming Module or TSDB local persistence.
+   */
+  int8_t    state;           // ETsdbSmaStat
+  SHashObj *expiredWindows;  // key: skey of time window, value: N/A
+} SSmaStatItem;
+
+struct SSmaStat {
+  SHashObj *smaStatItems;  // key: indexName, value: SSmaStatItem
+};
+
 // declaration of static functions
 static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
 static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
@@ -64,6 +84,114 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param,
 static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
 static bool    tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
 
+static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
+  ASSERT(pSmaStat != NULL);
+  // TODO: lock and create when each put, or create during tsdbNew.
+  if (*pSmaStat == NULL) {
+    *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat));
+    if (*pSmaStat == NULL) {
+      // TODO: unlock
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+
+    (*pSmaStat)->smaStatItems =
+        taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
+    if ((*pSmaStat)->smaStatItems == NULL) {
+      tfree(*pSmaStat);
+      // TODO: unlock
+      return TSDB_CODE_FAILED;
+    }
+  }
+  // TODO: unlock
+  return TSDB_CODE_SUCCESS;
+}
+
+static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
+  SSmaStatItem *pItem = NULL;
+
+  pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem));
+  if (pItem) {
+    pItem->state = state;
+    pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP),
+                                         true, HASH_ENTRY_LOCK);
+    if (!pItem->expiredWindows) {
+      tfree(pItem);
+    }
+  }
+  return pItem;
+}
+
+int32_t tsdbFreeSmaState(SSmaStat *pSmaStat) {
+  if (pSmaStat) {
+    // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
+    SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
+    while (item != NULL) {
+      taosHashCleanup(item->expiredWindows);
+      item = taosHashIterate(pSmaStat->smaStatItems, item);
+    }
+
+    taosHashCleanup(pSmaStat->smaStatItems);
+    free(pSmaStat);
+  }
+}
+
+/**
+ * @brief Update expired window according to msg from stream computing module.
+ *
+ * @param pTsdb
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
+  if (msg == NULL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  // TODO: decode the msg => start
+  const char *  indexName = SMA_TEST_INDEX_NAME;
+  const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
+  TSKEY         expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE];
+  int64_t       now = taosGetTimestampMs();
+  for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
+    expiredWindows[i] = now + i;
+  }
+  // TODO: decode the msg <= end
+
+  SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems;
+
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName));
+  if (!pItem) {
+    pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED);  // TODO use the real state
+    if (!pItem) {
+      // Response to stream computing: OOM
+      // For query, if the indexName not found, the TSDB should tell query module to query raw TS data.
+      return TSDB_CODE_FAILED;
+    }
+
+    if (taosHashPut(pItemsHash, indexName, strnlen(indexName, TSDB_INDEX_NAME_LEN), &pItem, sizeof(pItem)) != 0) {
+      // If error occurs during put smaStatItem, free the resources of pItem
+      taosHashCleanup(pItem->expiredWindows);
+      free(pItem);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  int8_t state = TSDB_SMA_STAT_EXPIRED;
+  for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
+    if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) {
+      // If error occurs during put expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would tell
+      // query module to query raw TS data.
+      taosHashCleanup(pItem->expiredWindows);
+      taosHashRemove(pItemsHash, indexName, sizeof(indexName));
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 /**
  * @brief Judge the tSma storage level
  *

From 031b50ff764ca2233b9a8dadbd5d12e6f4e9ea99 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 9 Mar 2022 17:52:59 +0800
Subject: [PATCH 23/29] [TD-13761]<fix>: redefine dir api.

---
 source/os/src/osSocket.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 9330cb7b32..24db2d7c5f 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -809,7 +809,7 @@ int32_t taosGetFqdn(char *fqdn) {
   char hostname[1024];
   hostname[1023] = '\0';
   if (gethostname(hostname, 1023) == -1) {
-    // printf("failed to get hostname, reason:%s", strerror(errno));
+    printf("failed to get hostname, reason:%s", strerror(errno));
     return -1;
   }
 
@@ -826,7 +826,7 @@ int32_t taosGetFqdn(char *fqdn) {
 #endif  // __APPLE__
   int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
   if (!result) {
-    // printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+    printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
     return -1;
   }
 

From a5fece22c6819ac14f1f561fc9792cd398755a97 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 9 Mar 2022 18:08:02 +0800
Subject: [PATCH 24/29] [TD-13761]<fix>: redefine dir api.

---
 source/os/src/osSocket.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 24db2d7c5f..38f933dcbf 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -827,6 +827,7 @@ int32_t taosGetFqdn(char *fqdn) {
   int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
   if (!result) {
     printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+    assert(0);
     return -1;
   }
 

From bd3451a8bd068a8ecdd41522be5f57fe45b210d9 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 9 Mar 2022 18:08:26 +0800
Subject: [PATCH 25/29] [TD-13761]<fix>: redefine dir api.

---
 source/os/src/osSocket.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c
index 38f933dcbf..698ceded16 100644
--- a/source/os/src/osSocket.c
+++ b/source/os/src/osSocket.c
@@ -810,6 +810,7 @@ int32_t taosGetFqdn(char *fqdn) {
   hostname[1023] = '\0';
   if (gethostname(hostname, 1023) == -1) {
     printf("failed to get hostname, reason:%s", strerror(errno));
+    assert(0);
     return -1;
   }
 

From a455105de39c1e38feb0aceb06a1bc6d297a1171 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Wed, 9 Mar 2022 18:33:23 +0800
Subject: [PATCH 26/29] mark expired query window by SSmaStat

---
 source/dnode/vnode/src/tsdb/tsdbSma.c | 35 ++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c
index 5cfa597eb2..c5f1261282 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSma.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSma.c
@@ -86,7 +86,12 @@ static bool    tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWin
 
 static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) {
   ASSERT(pSmaStat != NULL);
-  // TODO: lock and create when each put, or create during tsdbNew.
+
+  if (*pSmaStat != NULL) {  // no lock
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // TODO: lock. lazy mode when update expired window, or hungry mode during tsdbNew.
   if (*pSmaStat == NULL) {
     *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat));
     if (*pSmaStat == NULL) {
@@ -149,6 +154,8 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
     return TSDB_CODE_FAILED;
   }
 
+  tsdbInitSmaStat(&pTsdb->pSmaStat);  // lazy mode
+
   // TODO: decode the msg => start
   const char *  indexName = SMA_TEST_INDEX_NAME;
   const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10;
@@ -157,8 +164,8 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
   for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
     expiredWindows[i] = now + i;
   }
-  // TODO: decode the msg <= end
 
+  // TODO: decode the msg <= end
   SHashObj *pItemsHash = pTsdb->pSmaStat->smaStatItems;
 
   SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pItemsHash, indexName, strlen(indexName));
@@ -181,8 +188,12 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
   int8_t state = TSDB_SMA_STAT_EXPIRED;
   for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) {
     if (taosHashPut(pItem->expiredWindows, &expiredWindows[i], sizeof(TSKEY), &state, sizeof(state)) != 0) {
-      // If error occurs during put expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would tell
-      // query module to query raw TS data.
+      // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would
+      // tell query module to query raw TS data.
+      // N.B.
+      //  1) It is assumed to be extemely little probability event of fail to taosHashPut.
+      //  2) This would solve the inconsistency to some extent, but not completely, unless we record all expired
+      // windows failed to put into hash table.
       taosHashCleanup(pItem->expiredWindows);
       taosHashRemove(pItemsHash, indexName, sizeof(indexName));
       return TSDB_CODE_FAILED;
@@ -612,6 +623,22 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow
  * @return int32_t
  */
 int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) {
+  const char *indexName = param->indexName;
+
+  SSmaStatItem *pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName));
+  if (pItem == NULL) {
+    // mark all window as expired and notify query module to query raw TS data.
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t nQueryWin = 0;
+  for (int32_t n = 0; n < nQueryWin; ++n) {
+    TSKEY thisWindow = n;
+    if (taosHashGet(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)) != NULL) {
+      // TODO: mark this window as expired.
+    }
+  }
+
   STSmaReadH tReadH = {0};
   tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData);
 

From 221a9bf97a36544d45de5bca2aa251b9448fba52 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Wed, 9 Mar 2022 18:36:13 +0800
Subject: [PATCH 27/29] revert

---
 include/common/taosdef.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/common/taosdef.h b/include/common/taosdef.h
index 7788dfd871..99360b8d3f 100644
--- a/include/common/taosdef.h
+++ b/include/common/taosdef.h
@@ -50,7 +50,7 @@ typedef enum {
   TSDB_CHECK_ITEM_MAX
 } ECheckItemType;
 
-typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2} TDUpdateConfig;
+typedef enum { TD_ROW_DISCARD_UPDATE = 0, TD_ROW_OVERWRITE_UPDATE = 1, TD_ROW_PARTIAL_UPDATE = 2 } TDUpdateConfig;
 typedef enum {
   TSDB_STATIS_OK = 0,    // statis part exist and load successfully
   TSDB_STATIS_NONE = 1,  // statis part not exist

From 913e8d70d8738047bb3b826c1c5b12d91f24a130 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Wed, 9 Mar 2022 10:49:42 +0800
Subject: [PATCH 28/29] define stream token

---
 source/dnode/vnode/inc/vnode.h       | 18 +++++++++++++++++-
 source/dnode/vnode/src/inc/tqPush.h  | 13 ++++++++-----
 source/dnode/vnode/src/tq/tq.c       | 20 ++++++++++++++++++++
 source/dnode/vnode/src/tq/tqPush.c   |  2 +-
 source/dnode/vnode/src/tq/tqRead.c   |  2 +-
 source/libs/wal/src/walWrite.c       |  3 ---
 source/libs/wal/test/walMetaTest.cpp | 22 +++++++++++-----------
 7 files changed, 58 insertions(+), 22 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 319bd9fde6..b6dd90a4e2 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -59,7 +59,7 @@ typedef struct {
   SWalCfg  walCfg;
   uint32_t hashBegin;
   uint32_t hashEnd;
-  int8_t   hashMethod;  
+  int8_t   hashMethod;
 } SVnodeCfg;
 
 typedef struct {
@@ -202,6 +202,22 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 
 /* ------------------------- TQ READ --------------------------- */
 
+enum {
+  TQ_STREAM_TOKEN__DATA = 1,
+  TQ_STREAM_TOKEN__WATERMARK,
+  TQ_STREAM_TOKEN__CHECKPOINT,
+};
+
+typedef struct {
+  int8_t type;
+  int8_t reserved[7];
+  union {
+    void   *data;
+    int64_t wmTs;
+    int64_t checkpointId;
+  };
+} STqStreamToken;
+
 STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta);
 
 static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) {
diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h
index 32fd7c3ddf..a6121c5dc1 100644
--- a/source/dnode/vnode/src/inc/tqPush.h
+++ b/source/dnode/vnode/src/inc/tqPush.h
@@ -16,9 +16,11 @@
 #ifndef _TQ_PUSH_H_
 #define _TQ_PUSH_H_
 
+#include "executor.h"
 #include "thash.h"
 #include "trpc.h"
 #include "ttimer.h"
+#include "vnode.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -39,11 +41,12 @@ typedef struct {
 } STqClientPusher;
 
 typedef struct {
-  int8_t  type;
-  int8_t  nodeType;
-  int8_t  reserved[6];
-  int64_t streamId;
-  SEpSet  epSet;
+  int8_t      type;
+  int8_t      nodeType;
+  int8_t      reserved[6];
+  int64_t     streamId;
+  qTaskInfo_t task;
+  // TODO sync function
 } STqStreamPusher;
 
 typedef struct {
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 16809f1527..d15481b4aa 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -67,6 +67,26 @@ void tqClose(STQ* pTq) {
 }
 
 int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
+  if (msgType != TDMT_VND_SUBMIT) return 0;
+  void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL);
+  while (pIter != NULL) {
+    STqPusher* pusher = *(STqPusher**)pIter;
+    if (pusher->type == TQ_PUSHER_TYPE__STREAM) {
+      STqStreamPusher* streamPusher = (STqStreamPusher*)pusher;
+      // repack
+      STqStreamToken* token = malloc(sizeof(STqStreamToken));
+      if (token == NULL) {
+        taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      token->type = TQ_STREAM_TOKEN__DATA;
+      token->data = msg;
+      // set input
+      // exec
+    }
+    // send msg to ep
+  }
   // iterate hash
   // process all msg
   // if waiting
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index fea65846be..4186f29e2a 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -73,7 +73,7 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet
   streamPusher->type = TQ_PUSHER_TYPE__STREAM;
   streamPusher->nodeType = 0;
   streamPusher->streamId = streamId;
-  memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));
+  /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/
 
   if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 2f24df0309..92a111298f 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -12,7 +12,6 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
 
 #include "vnode.h"
 
@@ -37,6 +36,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
   pMsg->length = htonl(pMsg->length);
   pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
 
+  // iterate and convert
   if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
   while (true) {
     if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index 699da75790..0f6ac0b214 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -34,9 +34,6 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
 int32_t walRollback(SWal *pWal, int64_t ver) {
   int  code;
   char fnameStr[WAL_FILE_LEN];
-  if (ver == pWal->vers.lastVer) {
-    return 0;
-  }
   if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
     terrno = TSDB_CODE_WAL_INVALID_VER;
     return -1;
diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp
index 230555e016..5bfea9ab5e 100644
--- a/source/libs/wal/test/walMetaTest.cpp
+++ b/source/libs/wal/test/walMetaTest.cpp
@@ -124,13 +124,8 @@ class WalRetentionEnv : public ::testing::Test {
 
   void SetUp() override {
     SWalCfg cfg;
-    cfg.rollPeriod = -1,
-    cfg.segSize = -1,
-    cfg.retentionPeriod = -1,
-    cfg.retentionSize = 0,
-    cfg.rollPeriod = 0,
-    cfg.vgId = 0,
-    cfg.level = TAOS_WAL_FSYNC;
+    cfg.rollPeriod = -1, cfg.segSize = -1, cfg.retentionPeriod = -1, cfg.retentionSize = 0, cfg.rollPeriod = 0,
+    cfg.vgId = 0, cfg.level = TAOS_WAL_FSYNC;
     pWal = walOpen(pathName, &cfg);
     ASSERT(pWal != NULL);
   }
@@ -241,6 +236,12 @@ TEST_F(WalCleanEnv, rollback) {
     ASSERT_EQ(code, 0);
     ASSERT_EQ(pWal->vers.lastVer, i);
   }
+  code = walRollback(pWal, 12);
+  ASSERT_NE(code, 0);
+  ASSERT_EQ(pWal->vers.lastVer, 9);
+  code = walRollback(pWal, 9);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(pWal->vers.lastVer, 8);
   code = walRollback(pWal, 5);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(pWal->vers.lastVer, 4);
@@ -324,7 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
 TEST_F(WalRetentionEnv, repairMeta1) {
   walResetEnv();
   int code;
-  
+
   int i;
   for (i = 0; i < 100; i++) {
     char newStr[100];
@@ -336,14 +337,14 @@ TEST_F(WalRetentionEnv, repairMeta1) {
 
   TearDown();
 
-  //getchar();
+  // getchar();
   char buf[100];
   sprintf(buf, "%s/meta-ver%d", pathName, 0);
   taosRemoveFile(buf);
   sprintf(buf, "%s/meta-ver%d", pathName, 1);
   taosRemoveFile(buf);
   SetUp();
-  //getchar();
+  // getchar();
 
   ASSERT_EQ(pWal->vers.lastVer, 99);
 
@@ -401,5 +402,4 @@ TEST_F(WalRetentionEnv, repairMeta1) {
       EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
     }
   }
-
 }

From c6de1967698180061767a59bd7a6d656c8c5ae11 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Thu, 10 Mar 2022 00:36:30 +0800
Subject: [PATCH 29/29] [TD-13765]<fix>: redefine rand api.

---
 contrib/test/craft/raftMain.c                 |  2 +-
 contrib/test/craft/simulate_vnode.c           |  4 +-
 examples/c/schemaless.c                       |  2 +-
 include/os/osRand.h                           |  9 ++++
 source/client/src/clientEnv.c                 |  2 +-
 source/dnode/vnode/test/tqMetaTest.cpp        | 18 +++----
 source/libs/catalog/test/catalogTests.cpp     | 10 ++--
 source/libs/executor/src/executorimpl.c       |  6 +--
 source/libs/executor/test/executorTests.cpp   |  8 +--
 source/libs/executor/test/lhashTests.cpp      |  2 +-
 source/libs/index/test/indexTests.cc          |  2 +-
 source/libs/qworker/test/qworkerTests.cpp     | 50 +++++++++----------
 .../libs/scalar/test/filter/filterTests.cpp   |  2 +-
 .../libs/scalar/test/scalar/scalarTests.cpp   |  2 +-
 source/libs/scheduler/test/schedulerTests.cpp |  4 +-
 source/libs/sync/src/syncEnv.c                |  2 +-
 source/libs/sync/src/syncIO.c                 |  2 +-
 source/libs/sync/src/syncUtil.c               |  2 +-
 source/libs/tdb/src/db/tdbUtil.c              |  2 +-
 source/libs/transport/src/rpcMain.c           |  2 +-
 source/libs/wal/test/walMetaTest.cpp          |  6 +--
 source/os/src/osEnv.c                         |  2 +-
 source/os/src/osRand.c                        |  6 ++-
 source/util/src/tdes.c                        |  2 +-
 source/util/src/tskiplist.c                   |  8 +--
 source/util/test/codingTests.cpp              |  4 +-
 source/util/test/pageBufferTest.cpp           |  2 +-
 source/util/test/skiplistTest.cpp             |  8 +--
 28 files changed, 92 insertions(+), 79 deletions(-)

diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c
index b28adfaaca..12be3deb2e 100644
--- a/contrib/test/craft/raftMain.c
+++ b/contrib/test/craft/raftMain.c
@@ -377,7 +377,7 @@ void printConf(SRaftServerConfig *pConf) {
 
 
 int main(int argc, char **argv) { 
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 	int32_t ret;
 
 	exe_name = argv[0];
diff --git a/contrib/test/craft/simulate_vnode.c b/contrib/test/craft/simulate_vnode.c
index 668fe638b7..7ee9b9f8f0 100644
--- a/contrib/test/craft/simulate_vnode.c
+++ b/contrib/test/craft/simulate_vnode.c
@@ -132,7 +132,7 @@ static void proposeValue(struct raft *r) {
     buf.base = raft_malloc(buf.len);
 
     // mock ts value
-    int vid = rand() % VNODE_COUNT;
+    int vid = taosRand() % VNODE_COUNT;
     snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL));
 
 	printf("propose value: %s \n", (char*)buf.base);
@@ -174,7 +174,7 @@ void usage() {
 }
 
 int main(int argc, char **argv) {
-	srand(time(NULL));
+	taosSeedRand(time(NULL));
 
 	exe_name = argv[0];
 	if (argc < 2) {
diff --git a/examples/c/schemaless.c b/examples/c/schemaless.c
index 21f39213cd..99aa361b0a 100644
--- a/examples/c/schemaless.c
+++ b/examples/c/schemaless.c
@@ -19,7 +19,7 @@ void shuffle(char**lines, size_t n)
     size_t i;
     for (i = 0; i < n - 1; i++)
     {
-      size_t j = i + rand() / (RAND_MAX / (n - i) + 1);
+      size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1);
       char* t = lines[j];
       lines[j] = lines[i];
       lines[i] = t;
diff --git a/include/os/osRand.h b/include/os/osRand.h
index 422ea92a71..09e1f1b41d 100644
--- a/include/os/osRand.h
+++ b/include/os/osRand.h
@@ -20,7 +20,16 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define rand RAND_FUNC_TAOS_FORBID
+    #define srand SRAND_FUNC_TAOS_FORBID
+    #define rand_r RANDR_FUNC_TAOS_FORBID
+#endif
+
+void taosSeedRand(uint32_t seed);
 uint32_t taosRand(void);
+uint32_t taosRandR(uint32_t *pSeed);
 void     taosRandStr(char* str, int32_t size);
 uint32_t taosSafeRand(void);
 
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 08285c9d26..f5cc034e09 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -208,7 +208,7 @@ void taos_init_imp(void) {
   atexit(taos_cleanup);
 
   errno = TSDB_CODE_SUCCESS;
-  srand(taosGetTimestampSec());
+  taosSeedRand(taosGetTimestampSec());
 
   deltaToUtcInitOnce();
 
diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp
index d3c9b50e4a..4f15185254 100644
--- a/source/dnode/vnode/test/tqMetaTest.cpp
+++ b/source/dnode/vnode/test/tqMetaTest.cpp
@@ -168,10 +168,10 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
 }
 
 TEST_F(TqMetaUpdateAppendTest, multiplePage) {
-  srand(0);
+  taosSeedRand(0);
   std::vector<int> v;
   for (int i = 0; i < 1000; i++) {
-    v.push_back(rand());
+    v.push_back(taosRand());
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
@@ -202,10 +202,10 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) {
 }
 
 TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
-  srand(0);
+  taosSeedRand(0);
   std::vector<int> v;
   for (int i = 0; i < 1000; i++) {
-    v.push_back(rand());
+    v.push_back(taosRand());
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
@@ -213,14 +213,14 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
 
   for (int i = 0; i < 500; i++) {
     tqHandleCommit(pMeta, i);
-    v[i] = rand();
+    v[i] = taosRand();
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
   }
 
   for (int i = 500; i < 1000; i++) {
-    v[i] = rand();
+    v[i] = taosRand();
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
@@ -235,7 +235,7 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
   ASSERT(pMeta);
 
   for (int i = 500; i < 1000; i++) {
-    v[i] = rand();
+    v[i] = taosRand();
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
@@ -250,10 +250,10 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
 }
 
 TEST_F(TqMetaUpdateAppendTest, dupCommit) {
-  srand(0);
+  taosSeedRand(0);
   std::vector<int> v;
   for (int i = 0; i < 1000; i++) {
-    v.push_back(rand());
+    v.push_back(taosRand());
     Foo foo;
     foo.a = v[i];
     tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp
index c7867c4da5..00f6a508b7 100644
--- a/source/libs/catalog/test/catalogTests.cpp
+++ b/source/libs/catalog/test/catalogTests.cpp
@@ -723,7 +723,7 @@ void *ctgTestGetDbVgroupThread(void *param) {
     }
 
     if (ctgTestEnableSleep) {
-      usleep(rand() % 5);
+      usleep(taosRand() % 5);
     }
     if (++n % ctgTestPrintNum == 0) {
       printf("Get:%d\n", n);
@@ -747,7 +747,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) {
     }
 
     if (ctgTestEnableSleep) {
-      usleep(rand() % 5);
+      usleep(taosRand() % 5);
     }
     if (++n % ctgTestPrintNum == 0) {
       printf("Set:%d\n", n);
@@ -771,7 +771,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) {
     }
 
     if (ctgTestEnableSleep) {
-      usleep(rand() % 5);
+      usleep(taosRand() % 5);
     }
     if (++n % ctgTestPrintNum == 0) {
       printf("Set:%d\n", n);
@@ -801,7 +801,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
     tfree(tbMeta);
 
     if (ctgTestEnableSleep) {
-      usleep(rand() % 5);
+      usleep(taosRand() % 5);
     }
 
     if (++n % ctgTestPrintNum == 0) {
@@ -838,7 +838,7 @@ void *ctgTestSetCtableMetaThread(void *param) {
     }
 
     if (ctgTestEnableSleep) {
-      usleep(rand() % 5);
+      usleep(taosRand() % 5);
     }
     if (++n % ctgTestPrintNum == 0) {
       printf("Set:%d\n", n);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index eeb48a1f3d..e30b51cbdf 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -61,7 +61,7 @@ typedef enum SResultTsInterpType {
 
 #if 0
 static UNUSED_FUNC void *u_malloc (size_t __size) {
-  uint32_t v = rand();
+  uint32_t v = taosRand();
 
   if (v % 1000 <= 0) {
     return NULL;
@@ -71,7 +71,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) {
 }
 
 static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) {
-  uint32_t v = rand();
+  uint32_t v = taosRand();
   if (v % 1000 <= 0) {
     return NULL;
   } else {
@@ -80,7 +80,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) {
 }
 
 static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
-  uint32_t v = rand();
+  uint32_t v = taosRand();
   if (v % 5 <= 1) {
     return NULL;
   } else {
diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp
index ff29d5f355..72ef13124a 100644
--- a/source/libs/executor/test/executorTests.cpp
+++ b/source/libs/executor/test/executorTests.cpp
@@ -869,7 +869,7 @@ TEST(testCase, external_sort_Test) {
 #if 0
   su* v = static_cast<su*>(calloc(1000000, sizeof(su)));
   for(int32_t i = 0; i < 1000000; ++i) {
-    v[i].v = rand();
+    v[i].v = taosRand();
     v[i].c = static_cast<char*>(malloc(4));
     *(int32_t*) v[i].c = i;
   }
@@ -882,7 +882,7 @@ TEST(testCase, external_sort_Test) {
   return;
 #endif
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
   SOrder o = {0};
@@ -943,7 +943,7 @@ TEST(testCase, external_sort_Test) {
 }
 
 TEST(testCase, sorted_merge_Test) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
   SOrder o = {0};
@@ -1015,7 +1015,7 @@ TEST(testCase, sorted_merge_Test) {
 }
 
 TEST(testCase, time_interval_Operator_Test) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
   SOrder o = {0};
diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp
index 66ef3b0877..88cf713727 100644
--- a/source/libs/executor/test/lhashTests.cpp
+++ b/source/libs/executor/test/lhashTests.cpp
@@ -25,7 +25,7 @@
 #pragma GCC diagnostic ignored "-Wsign-compare"
 
 TEST(testCase, linear_hash_Tests) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
 #if 0
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index ce3f7fe25e..699c785be5 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -699,7 +699,7 @@ class IndexObj {
     for (int i = 0; i < numOfTable; i++) {
       for (int k = 0; k < 10 && k < colVal.size(); k++) {
         // opt
-        tColVal[rand() % colValSize] = 'a' + k % 26;
+        tColVal[taosRand() % colValSize] = 'a' + k % 26;
       }
       SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          tColVal.c_str(), tColVal.size());
diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp
index 231f0c7fff..8658c4cead 100644
--- a/source/libs/qworker/test/qworkerTests.cpp
+++ b/source/libs/qworker/test/qworkerTests.cpp
@@ -266,7 +266,7 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas
   int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum);
 
   qwtTestSinkBlockNum = 0;
-  qwtTestSinkMaxBlockNum = rand() % 100 + 1;
+  qwtTestSinkMaxBlockNum = taosRand() % 100 + 1;
   qwtTestSinkQueryEnd = false;
   
   if (0 == idx) {
@@ -295,15 +295,15 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
   } else {
     if (qwtTestSinkQueryEnd) {
       *pRes = NULL;
-      *useconds = rand() % 10;
+      *useconds = taosRand() % 10;
       return 0;
     }
     
-    endExec = rand() % 5;
+    endExec = taosRand() % 5;
     
     int32_t runTime = 0;
     if (qwtTestEnableSleep && qwtTestMaxExecTaskUsec > 0) {
-      runTime = rand() % qwtTestMaxExecTaskUsec;
+      runTime = taosRand() % qwtTestMaxExecTaskUsec;
     }
 
     if (qwtTestEnableSleep) {
@@ -314,10 +314,10 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
       
     if (endExec) {
       *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock));
-      (*pRes)->info.rows = rand() % 1000;
+      (*pRes)->info.rows = taosRand() % 1000;
     } else {
       *pRes = NULL;
-      *useconds = rand() % 10;
+      *useconds = taosRand() % 10;
     }
   }
   
@@ -376,7 +376,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) {
 
   taosWLockLatch(&qwtTestSinkLock);
   if (qwtTestSinkBlockNum > 0) {
-    *pLen = rand() % 100 + 1;
+    *pLen = taosRand() % 100 + 1;
     qwtTestSinkBlockNum--;
   } else {
     *pLen = 0;
@@ -392,7 +392,7 @@ void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) {
 int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) {
   taosWLockLatch(&qwtTestSinkLock);
   if (qwtTestSinkLastLen > 0) {
-    pOutput->numOfRows = rand() % 10 + 1;
+    pOutput->numOfRows = taosRand() % 10 + 1;
     pOutput->compressed = 1;
     pOutput->queryEnd = qwtTestSinkQueryEnd;
     if (qwtTestSinkBlockNum == 0) {
@@ -402,7 +402,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) {
     } else {
       pOutput->bufStatus = DS_BUF_FULL;
     }
-    pOutput->useconds = rand() % 10 + 1;
+    pOutput->useconds = taosRand() % 10 + 1;
     pOutput->precision = 1;
   } else if (qwtTestSinkLastLen == 0) {
     pOutput->numOfRows = 0;
@@ -416,7 +416,7 @@ int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) {
     } else {
       pOutput->bufStatus = DS_BUF_FULL;
     }
-    pOutput->useconds = rand() % 10 + 1;
+    pOutput->useconds = taosRand() % 10 + 1;
     pOutput->precision = 1;
   } else {
     assert(0);
@@ -590,7 +590,7 @@ void *queryThread(void *param) {
     qwtBuildQueryReqMsg(&queryRpc);
     qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);    
     if (qwtTestEnableSleep) {
-      usleep(rand()%5);
+      usleep(taosRand()%5);
     }
     if (++n % qwtTestPrintNum == 0) {
       printf("query:%d\n", n);
@@ -612,7 +612,7 @@ void *readyThread(void *param) {
     qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
     code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
     if (qwtTestEnableSleep) {
-      usleep(rand()%5);
+      usleep(taosRand()%5);
     }
     if (++n % qwtTestPrintNum == 0) {
       printf("ready:%d\n", n);
@@ -634,7 +634,7 @@ void *fetchThread(void *param) {
     qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
     code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
     if (qwtTestEnableSleep) {
-      usleep(rand()%5);
+      usleep(taosRand()%5);
     }
     if (++n % qwtTestPrintNum == 0) {
       printf("fetch:%d\n", n);
@@ -656,7 +656,7 @@ void *dropThread(void *param) {
     qwtBuildDropReqMsg(&dropMsg, &dropRpc);
     code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
     if (qwtTestEnableSleep) {
-      usleep(rand()%5);
+      usleep(taosRand()%5);
     }
     if (++n % qwtTestPrintNum == 0) {
       printf("drop:%d\n", n);
@@ -678,7 +678,7 @@ void *statusThread(void *param) {
     qwtBuildStatusReqMsg(&statusMsg, &statusRpc);
     code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
     if (qwtTestEnableSleep) {
-      usleep(rand()%5);
+      usleep(taosRand()%5);
     }
     if (++n % qwtTestPrintNum == 0) {
       printf("status:%d\n", n);
@@ -748,7 +748,7 @@ void *queryQueueThread(void *param) {
 
 
     if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) {
-      int32_t delay = rand() % qwtTestReqMaxDelayUsec;
+      int32_t delay = taosRand() % qwtTestReqMaxDelayUsec;
 
       if (delay) {
         usleep(delay);
@@ -804,7 +804,7 @@ void *fetchQueueThread(void *param) {
     taosWUnLockLatch(&qwtTestFetchQueueLock);
 
     if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) {
-      int32_t delay = rand() % qwtTestReqMaxDelayUsec;
+      int32_t delay = taosRand() % qwtTestReqMaxDelayUsec;
 
       if (delay) {
         usleep(delay);
@@ -963,7 +963,7 @@ TEST(seqTest, randCase) {
   stubSetRpcSendResponse();
   stubSetCreateExecTask();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   
   code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
   ASSERT_EQ(code, 0);
@@ -971,7 +971,7 @@ TEST(seqTest, randCase) {
   int32_t t = 0;
   int32_t maxr = 10001;
   while (true) {
-    int32_t r = rand() % maxr;
+    int32_t r = taosRand() % maxr;
     
     if (r >= 0 && r < maxr/5) {
       printf("Query,%d\n", t++);      
@@ -1025,7 +1025,7 @@ TEST(seqTest, multithreadRand) {
   stubSetStringToPlan();
   stubSetRpcSendResponse();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   
   code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
   ASSERT_EQ(code, 0);
@@ -1076,7 +1076,7 @@ TEST(rcTest, shortExecshortDelay) {
   stubSetPutDataBlock();
   stubSetGetDataBlock();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   qwtTestStop = false;
   qwtTestQuitThreadNum = 0;
 
@@ -1157,7 +1157,7 @@ TEST(rcTest, longExecshortDelay) {
   stubSetPutDataBlock();
   stubSetGetDataBlock();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   qwtTestStop = false;
   qwtTestQuitThreadNum = 0;
 
@@ -1240,7 +1240,7 @@ TEST(rcTest, shortExeclongDelay) {
   stubSetPutDataBlock();
   stubSetGetDataBlock();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   qwtTestStop = false;
   qwtTestQuitThreadNum = 0;
 
@@ -1324,7 +1324,7 @@ TEST(rcTest, dropTest) {
   stubSetPutDataBlock();
   stubSetGetDataBlock();
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   
   code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue);
   ASSERT_EQ(code, 0);
@@ -1358,7 +1358,7 @@ TEST(rcTest, dropTest) {
 
 
 int main(int argc, char** argv) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp
index 08210aa2f0..c69a1eb247 100644
--- a/source/libs/scalar/test/filter/filterTests.cpp
+++ b/source/libs/scalar/test/filter/filterTests.cpp
@@ -1286,7 +1286,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
 
 
 int main(int argc, char** argv) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp
index faf13f0a82..6518dbec87 100644
--- a/source/libs/scalar/test/scalar/scalarTests.cpp
+++ b/source/libs/scalar/test/scalar/scalarTests.cpp
@@ -1427,7 +1427,7 @@ TEST(columnTest, greater_and_lower) {
 
 
 int main(int argc, char** argv) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp
index 8ed963d875..f17dcbd1f5 100644
--- a/source/libs/scheduler/test/schedulerTests.cpp
+++ b/source/libs/scheduler/test/schedulerTests.cpp
@@ -532,7 +532,7 @@ void* schtRunJobThread(void *aa) {
 
 void* schtFreeJobThread(void *aa) {
   while (!schtTestStop) {
-    usleep(rand() % 100);
+    usleep(taosRand() % 100);
     schtFreeQueryJob(1);
   }
 }
@@ -713,7 +713,7 @@ TEST(multiThread, forceFree) {
 }
 
 int main(int argc, char** argv) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c
index 6917df1597..fa58bda76f 100644
--- a/source/libs/sync/src/syncEnv.c
+++ b/source/libs/sync/src/syncEnv.c
@@ -28,7 +28,7 @@ static void    doSyncEnvStopTimer(SSyncEnv *pSyncEnv, tmr_h *pTimer);
 
 int32_t syncEnvStart() {
   int32_t ret;
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv));
   assert(gSyncEnv != NULL);
   ret = doSyncEnvStart(gSyncEnv);
diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c
index d37c821a24..7cb42c9f87 100644
--- a/source/libs/sync/src/syncIO.c
+++ b/source/libs/sync/src/syncIO.c
@@ -44,7 +44,7 @@ int32_t syncIOStart(char *host, uint16_t port) {
   gSyncIO = syncIOCreate(host, port);
   assert(gSyncIO != NULL);
 
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   int32_t ret = syncIOStartInternal(gSyncIO);
   assert(ret == 0);
 
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index 3fb430a714..216dccd62c 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -95,7 +95,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) {
 
 // ---- misc ----
 
-int32_t syncUtilRand(int32_t max) { return rand() % max; }
+int32_t syncUtilRand(int32_t max) { return taosRand() % max; }
 
 int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
 
diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c
index fe0f3befd6..237a39e47d 100644
--- a/source/libs/tdb/src/db/tdbUtil.c
+++ b/source/libs/tdb/src/db/tdbUtil.c
@@ -27,7 +27,7 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
   ((uint64_t *)fileid)[0] = stDev;
   ((uint64_t *)fileid)[1] = stIno;
   if (unique) {
-    ((uint64_t *)fileid)[2] = rand();
+    ((uint64_t *)fileid)[2] = taosRand();
   }
 
   return 0;
diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c
index e1319da162..615c576a9b 100644
--- a/source/libs/transport/src/rpcMain.c
+++ b/source/libs/transport/src/rpcMain.c
@@ -749,7 +749,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
     memcpy(pConn->user, pHead->user, tListLen(pConn->user));
     pConn->pRpc = pRpc;
     pConn->sid = sid;
-    pConn->tranId = (uint16_t)(rand() & 0xFFFF);
+    pConn->tranId = (uint16_t)(taosRand() & 0xFFFF);
     pConn->ownId = htonl(pConn->sid);
     pConn->linkUid = pHead->linkUid;
     if (pRpc->afp) {
diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp
index 5bfea9ab5e..f44fc71964 100644
--- a/source/libs/wal/test/walMetaTest.cpp
+++ b/source/libs/wal/test/walMetaTest.cpp
@@ -300,7 +300,7 @@ TEST_F(WalKeepEnv, readHandleRead) {
     ASSERT_EQ(code, 0);
   }
   for (int i = 0; i < 1000; i++) {
-    int ver = rand() % 100;
+    int ver = taosRand() % 100;
     code = walReadWithHandle(pRead, ver);
     ASSERT_EQ(code, 0);
 
@@ -352,7 +352,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
   ASSERT(pRead != NULL);
 
   for (int i = 0; i < 1000; i++) {
-    int ver = rand() % 100;
+    int ver = taosRand() % 100;
     code = walReadWithHandle(pRead, ver);
     ASSERT_EQ(code, 0);
 
@@ -382,7 +382,7 @@ TEST_F(WalRetentionEnv, repairMeta1) {
   }
 
   for (int i = 0; i < 1000; i++) {
-    int ver = rand() % 200;
+    int ver = taosRand() % 200;
     code = walReadWithHandle(pRead, ver);
     ASSERT_EQ(code, 0);
 
diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c
index 0b2fe904b3..63fa600217 100644
--- a/source/os/src/osEnv.c
+++ b/source/os/src/osEnv.c
@@ -38,7 +38,7 @@ float      tsNumOfCores = 0;
 int64_t    tsTotalMemoryKB = 0;
 
 void osInit() {
-  srand(taosSafeRand());
+  taosSeedRand(taosSafeRand());
   taosGetSystemLocale(tsLocale, tsCharset);
   taosGetSystemTimezone(tsTimezone);
   taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight);
diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c
index b81e41b3cf..f3dd9b74c5 100644
--- a/source/os/src/osRand.c
+++ b/source/os/src/osRand.c
@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
+#define ALLOW_FORBID_FUNC
 #define _DEFAULT_SOURCE
 #include "os.h"
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@@ -21,8 +21,12 @@
 #include <unistd.h>
 #endif
 
+void taosSeedRand(uint32_t seed) { return srand(seed); }
+
 uint32_t taosRand(void) { return rand(); }
 
+uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); }
+
 uint32_t taosSafeRand(void) {
   TdFilePtr pFile;
   int seed;
diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c
index 105dd7f95f..d12b47efe8 100644
--- a/source/util/src/tdes.c
+++ b/source/util/src/tdes.c
@@ -32,7 +32,7 @@ void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set*
 #if 0
 int64_t taosDesGenKey() {
   uint32_t iseed = (uint32_t)time(NULL);
-  srand(iseed);
+  taosSeedRand(iseed);
 
   uint8_t key[8] = {0};
   generate_key(key);
diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c
index 6b89ed2c43..d9d6e4e3da 100644
--- a/source/util/src/tskiplist.c
+++ b/source/util/src/tskiplist.c
@@ -51,7 +51,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
   pSkipList->len = keyLen;
   pSkipList->flags = flags;
   pSkipList->keyFn = fn;
-  pSkipList->seed = rand();
+  pSkipList->seed = taosRand();
 
 #if 0 
   // the function getkeycomparfunc is defined in common
@@ -82,7 +82,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
     }
   }
 
-  srand((uint32_t)time(NULL));
+  taosSeedRand((uint32_t)time(NULL));
 
 #if SKIP_LIST_RECORD_PERFORMANCE
   pSkipList->state.nTotalMemSize += sizeof(SSkipList);
@@ -560,9 +560,9 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
 
   int32_t n = 1;
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-  while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
+  while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) {
 #else
-  while ((rand_r(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) {
+  while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) {
 #endif
     n++;
   }
diff --git a/source/util/test/codingTests.cpp b/source/util/test/codingTests.cpp
index 0cd9524646..b991411047 100644
--- a/source/util/test/codingTests.cpp
+++ b/source/util/test/codingTests.cpp
@@ -150,7 +150,7 @@ static bool test_variant_int64(int64_t value) {
 }
 
 TEST(codingTest, fixed_encode_decode) {
-  srand(time(0));
+  taosSeedRand(time(0));
 
   // uint16_t
   for (uint16_t value = 0; value <= UINT16_MAX; value++) {
@@ -204,7 +204,7 @@ TEST(codingTest, fixed_encode_decode) {
 }
 
 TEST(codingTest, variant_encode_decode) {
-  srand(time(0));
+  taosSeedRand(time(0));
 
   // uint16_t
   for (uint16_t value = 0; value <= UINT16_MAX; value++) {
diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp
index f392aac7d1..e63e6f04a1 100644
--- a/source/util/test/pageBufferTest.cpp
+++ b/source/util/test/pageBufferTest.cpp
@@ -161,7 +161,7 @@ void recyclePageTest() {
 
 
 TEST(testCase, resultBufferTest) {
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
   simpleTest();
   writeDownTest();
   recyclePageTest();
diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp
index f2e696b0e5..f61ebfd890 100644
--- a/source/util/test/skiplistTest.cpp
+++ b/source/util/test/skiplistTest.cpp
@@ -47,7 +47,7 @@ void doubleSkipListTest() {
   SSkipListKey    sk;
   for (int32_t i = 0; i < 100; ++i) {
     sk.nType = TSDB_DATA_TYPE_DOUBLE;
-    int32_t idx = abs((i * rand()) % 1000);
+    int32_t idx = abs((i * taosRand()) % 1000);
 
     sk.dKey = doubleVal[idx];
 
@@ -74,7 +74,7 @@ void randKeyTest() {
       false, getkey);
 
   int32_t size = 200000;
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   printf("generated %d keys is: \n", size);
 
@@ -87,7 +87,7 @@ void randKeyTest() {
     d->level = level;
 
     int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d);
-    key[0] = rand() % 1000000000;
+    key[0] = taosRand() % 1000000000;
 
     key[1] = key[0];
 
@@ -337,7 +337,7 @@ void duplicatedKeyTest() {
 
 TEST(testCase, skiplist_test) {
   assert(sizeof(SSkipListKey) == 8);
-  srand(time(NULL));
+  taosSeedRand(time(NULL));
 
   stringKeySkiplistTest();
   doubleSkipListTest();