diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h
index cf5a3c8e05..e70a8539c3 100644
--- a/include/common/taosmsg.h
+++ b/include/common/taosmsg.h
@@ -672,16 +672,16 @@ typedef struct {
 } SDnodeCfg;
 
 typedef struct {
-  int32_t  dnodeId;
+  int32_t  id;
   int8_t   isMnode;
   int8_t   reserved;
-  uint16_t dnodePort;
-  char     dnodeFqdn[TSDB_FQDN_LEN];
+  uint16_t port;
+  char     fqdn[TSDB_FQDN_LEN];
 } SDnodeEp;
 
 typedef struct {
-  int32_t  dnodeNum;
-  SDnodeEp dnodeEps[];
+  int32_t  num;
+  SDnodeEp eps[];
 } SDnodeEps;
 
 typedef struct {
@@ -820,9 +820,9 @@ typedef struct {
 } SCreateDnodeMsg, SDropDnodeMsg;
 
 typedef struct {
-  int32_t    dnodeId;
-  int8_t     replica;
-  int8_t     reserved[3];
+  int32_t  dnodeId;
+  int8_t   replica;
+  int8_t   reserved[3];
   SReplica replicas[TSDB_MAX_REPLICA];
 } SCreateMnodeMsg, SAlterMnodeMsg, SDropMnodeMsg;
 
diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h
new file mode 100644
index 0000000000..7dd7730443
--- /dev/null
+++ b/include/dnode/mgmt/dnode.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DNODE_H_
+#define _TD_DNODE_H_
+
+#include "tdef.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ------------------------ TYPES EXPOSED ------------------------ */
+typedef struct SDnode SDnode;
+
+typedef struct {
+  /**
+   * @brief software version of the program.
+   *
+   */
+  int32_t sver;
+
+  /**
+   * @brief num of CPU cores.
+   *
+   */
+  int32_t numOfCores;
+
+  /**
+   * @brief number of threads per CPU core.
+   *
+   */
+  float numOfThreadsPerCore;
+
+  /**
+   * @brief the proportion of total CPU cores available for query processing.
+   *
+   */
+  float ratioOfQueryCores;
+
+  /**
+   * @brief max number of connections allowed in dnode.
+   *
+   */
+  int32_t maxShellConns;
+
+  /**
+   * @brief time interval of heart beat from shell to dnode, seconds.
+   *
+   */
+  int32_t shellActivityTimer;
+
+  /**
+   * @brief time interval of dnode status reporting to mnode, seconds, for cluster only.
+   *
+   */
+  int32_t statusInterval;
+
+  /**
+   * @brief first port number for the connection (12 continuous UDP/TCP port number are used).
+   *
+   */
+  uint16_t serverPort;
+
+  /**
+   * @brief data file's directory.
+   *
+   */
+  char dataDir[PATH_MAX];
+
+  /**
+   * @brief local endpoint.
+   *
+   */
+  char localEp[TSDB_EP_LEN];
+
+  /**
+   * @brieflocal fully qualified domain name (FQDN).
+   *
+   */
+  char localFqdn[TSDB_FQDN_LEN];
+
+  /**
+   * @brief first fully qualified domain name (FQDN) for TDengine system.
+   *
+   */
+  char firstEp[TSDB_EP_LEN];
+
+  /**
+   * @brief system time zone.
+   *
+   */
+  char timezone[TSDB_TIMEZONE_LEN];
+
+  /**
+   * @brief system locale.
+   *
+   */
+  char locale[TSDB_LOCALE_LEN];
+
+  /**
+   * @briefdefault system charset.
+   *
+   */
+  char charset[TSDB_LOCALE_LEN];
+} SDnodeOpt;
+
+/* ------------------------ SDnode ------------------------ */
+/**
+ * @brief Initialize and start the dnode.
+ *
+ * @param pOptions Options of the dnode.
+ * @return SDnode* The dnode object.
+ */
+SDnode *dndInit(SDnodeOpt *pOptions);
+
+/**
+ * @brief Stop and cleanup the dnode.
+ *
+ * @param pDnode The dnode object to close.
+ */
+void dndCleanup(SDnode *pDnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DNODE_H_*/
diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h
index e03d3ffd18..98aefc6db3 100644
--- a/include/dnode/mnode/mnode.h
+++ b/include/dnode/mnode/mnode.h
@@ -20,17 +20,16 @@
 extern "C" {
 #endif
 
-typedef enum { MN_MSG_TYPE_WRITE = 1, MN_MSG_TYPE_APPLY, MN_MSG_TYPE_SYNC, MN_MSG_TYPE_READ } EMnMsgType;
-
+/* ------------------------ TYPES EXPOSED ------------------------ */
+typedef struct SDnode    SDnode;
+typedef struct SMnode    SMnode;
 typedef struct SMnodeMsg SMnodeMsg;
+typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
+typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg);
+typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell);
+typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnd, SMnodeMsg *pMsg);
 
-typedef struct {
-  int8_t   replica;
-  int8_t   selfIndex;
-  SReplica replicas[TSDB_MAX_REPLICA];
-} SMnodeCfg;
-
-typedef struct {
+typedef struct SMnodeLoad {
   int64_t numOfDnode;
   int64_t numOfMnode;
   int64_t numOfVgroup;
@@ -44,29 +43,132 @@ typedef struct {
 } SMnodeLoad;
 
 typedef struct {
-  int32_t dnodeId;
-  int64_t clusterId;
-  void (*SendMsgToDnode)(struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-  void (*SendMsgToMnode)(struct SRpcMsg *rpcMsg);
-  void (*SendRedirectMsg)(struct SRpcMsg *rpcMsg, bool forShell);
-  int32_t (*PutMsgIntoApplyQueue)(SMnodeMsg *pMsg);
-} SMnodePara;
+  int32_t           dnodeId;
+  int64_t           clusterId;
+  int8_t            replica;
+  int8_t            selfIndex;
+  SReplica          replicas[TSDB_MAX_REPLICA];
+  struct SDnode    *pDnode;
+  PutMsgToMnodeQFp  putMsgToApplyMsgFp;
+  SendMsgToDnodeFp  sendMsgToDnodeFp;
+  SendMsgToMnodeFp  sendMsgToMnodeFp;
+  SendRedirectMsgFp sendRedirectMsgFp;
+} SMnodeOptions;
 
-int32_t mnodeInit(SMnodePara para);
-void    mnodeCleanup();
+/* ------------------------ SMnode ------------------------ */
+/**
+ * @brief Open a mnode.
+ *
+ * @param path Path of the mnode
+ * @param pOptions Options of the mnode
+ * @return SMnode* The mnode object
+ */
+SMnode *mnodeOpen(const char *path, const SMnodeOptions *pOptions);
 
-int32_t mnodeDeploy(SMnodeCfg *pCfg);
-void    mnodeUnDeploy();
-int32_t mnodeStart(SMnodeCfg *pCfg);
-int32_t mnodeAlter(SMnodeCfg *pCfg);
-void    mnodeStop();
+/**
+ * @brief Close a mnode
+ *
+ * @param pMnode The mnode object to close
+ */
+void mnodeClose(SMnode *pMnode);
 
-int32_t mnodeGetLoad(SMnodeLoad *pLoad);
-int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
+/**
+ * @brief Close a mnode
+ *
+ * @param pMnode The mnode object to close
+ * @param pOptions Options of the mnode
+ * @return int32_t 0 for success, -1 for failure
+ */
+int32_t mnodeAlter(SMnode *pMnode, const SMnodeOptions *pOptions);
 
-SMnodeMsg *mnodeInitMsg(SRpcMsg *pRpcMsg);
-void       mnodeCleanupMsg(SMnodeMsg *pMsg);
-void       mnodeProcessMsg(SMnodeMsg *pMsg, EMnMsgType msgType);
+/**
+ * @brief Drop a mnode.
+ *
+ * @param path Path of the mnode.
+ */
+void mnodeDestroy(const char *path);
+
+/**
+ * @brief Get mnode statistics info
+ *
+ * @param pMnode The mnode object
+ * @param pLoad Statistics of the mnode.
+ * @return int32_t 0 for success, -1 for failure
+ */
+int32_t mnodeGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
+
+/**
+ * @brief Get user authentication info
+ *
+ * @param pMnode The mnode object
+ * @param user
+ * @param spi
+ * @param encrypt
+ * @param secret
+ * @param ckey
+ * @return int32_t 0 for success, -1 for failure
+ */
+int32_t mnodeRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
+
+/**
+ * @brief Initialize mnode msg
+ *
+ * @param pMnode The mnode object
+ * @param pMsg The request rpc msg
+ * @return int32_t The created mnode msg
+ */
+SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg);
+
+/**
+ * @brief Cleanup mnode msg
+ *
+ * @param pMsg The request msg
+ */
+void mnodeCleanupMsg(SMnodeMsg *pMsg);
+
+/**
+ * @brief Cleanup mnode msg
+ *
+ * @param pMsg The request msg
+ * @param code The error code
+ */
+void mnodeSendRsp(SMnodeMsg *pMsg, int32_t code);
+
+/**
+ * @brief Process the read request
+ *
+ * @param pMnode The mnode object
+ * @param pMsg The request msg
+ * @return int32_t 0 for success, -1 for failure
+ */
+void mnodeProcessReadMsg(SMnode *pMnode, SMnodeMsg *pMsg);
+
+/**
+ * @brief Process the write request
+ *
+ * @param pMnode The mnode object
+ * @param pMsg The request msg
+ * @return int32_t 0 for success, -1 for failure
+ */
+void mnodeProcessWriteMsg(SMnode *pMnode, SMnodeMsg *pMsg);
+
+/**
+ * @brief Process the sync request
+ *
+ * @param pMnode The mnode object
+ * @param pMsg The request msg
+ * @return int32_t 0 for success, -1 for failure
+ */
+void mnodeProcessSyncMsg(SMnode *pMnode, SMnodeMsg *pMsg);
+
+/**
+ * @brief Process the apply request
+ *
+ * @param pMnode The mnode object
+ * @param pMsg The request msg
+ * @return int32_t 0 for success, -1 for failure
+ */
+void mnodeProcessApplyMsg(SMnode *pMnode, SMnodeMsg *pMsg);
 
 #ifdef __cplusplus
 }
diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h
index 90c5ef0c4a..784672e0ec 100644
--- a/include/dnode/mnode/sdb/sdb.h
+++ b/include/dnode/mnode/sdb/sdb.h
@@ -56,39 +56,39 @@ extern "C" {
     dataPos += valLen;                                      \
   }
 
-#define SDB_SET_INT64(pData, dataPos, val)         \
+#define SDB_SET_INT64(pRaw, dataPos, val)          \
   {                                                \
     if (sdbSetRawInt64(pRaw, dataPos, val) != 0) { \
-      sdbFreeRaw(pRaw);                               \
+      sdbFreeRaw(pRaw);                            \
       return NULL;                                 \
-    };                                             \
+    }                                              \
     dataPos += sizeof(int64_t);                    \
   }
 
-#define SDB_SET_INT32(pData, dataPos, val)         \
+#define SDB_SET_INT32(pRaw, dataPos, val)          \
   {                                                \
     if (sdbSetRawInt32(pRaw, dataPos, val) != 0) { \
-      sdbFreeRaw(pRaw);                               \
+      sdbFreeRaw(pRaw);                            \
       return NULL;                                 \
-    };                                             \
+    }                                              \
     dataPos += sizeof(int32_t);                    \
   }
 
-#define SDB_SET_INT8(pData, dataPos, val)         \
+#define SDB_SET_INT8(pRaw, dataPos, val)          \
   {                                               \
     if (sdbSetRawInt8(pRaw, dataPos, val) != 0) { \
-      sdbFreeRaw(pRaw);                              \
+      sdbFreeRaw(pRaw);                           \
       return NULL;                                \
-    };                                            \
+    }                                             \
     dataPos += sizeof(int8_t);                    \
   }
 
 #define SDB_SET_BINARY(pRaw, dataPos, val, valLen)          \
   {                                                         \
     if (sdbSetRawBinary(pRaw, dataPos, val, valLen) != 0) { \
-      sdbFreeRaw(pRaw);                                        \
+      sdbFreeRaw(pRaw);                                     \
       return NULL;                                          \
-    };                                                      \
+    }                                                       \
     dataPos += valLen;                                      \
   }
 
@@ -97,7 +97,7 @@ extern "C" {
     if (sdbSetRawDataLen(pRaw, dataLen) != 0) { \
       sdbFreeRaw(pRaw);                         \
       return NULL;                              \
-    };                                          \
+    }                                           \
   }
 
 typedef struct SSdbRaw SSdbRaw;
@@ -144,6 +144,8 @@ typedef struct {
   SdbDeleteFp deleteFp;
 } SSdbTable;
 
+typedef struct SSdb SSdb;
+
 int32_t sdbInit();
 void    sdbCleanup();
 void    sdbSetTable(SSdbTable table);
diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h
index 2c534fd3e3..febcdafacd 100644
--- a/include/dnode/vnode/vnode.h
+++ b/include/dnode/vnode/vnode.h
@@ -184,10 +184,16 @@ typedef struct {
   SRpcMsg rpcMsg[];
 } SVnodeMsg;
 
+typedef struct SDnode SDnode;
+typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
+typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg);
+typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell);
+typedef int32_t (*PutMsgToVnodeQFp)(SDnode *pDnd, int32_t vgId, SVnodeMsg *pMsg);
+
 typedef struct {
-  void (*SendMsgToDnode)(SEpSet *pEpSet, SRpcMsg *pMsg);
-  void (*SendMsgToMnode)(SRpcMsg *pMsg);
-  int32_t (*PutMsgIntoApplyQueue)(int32_t vgId, SVnodeMsg *pMsg);
+  PutMsgToVnodeQFp putMsgToApplyQueueFp;
+  SendMsgToDnodeFp sendMsgToDnodeFp;
+  SendMsgToMnodeFp sendMsgToMnodeFp;
 } SVnodePara;
 
 int32_t vnodeInit(SVnodePara);
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 5460ae5401..2a8a7aad4b 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -51,7 +51,7 @@ typedef struct SRpcMsg {
 } SRpcMsg;
 
 typedef struct SRpcInit {
-  uint16_t localPort; // local port
+ uint16_t localPort; // local port
   char  *label;        // for debug purpose
   int    numOfThreads; // number of threads to handle connections
   int    sessions;     // number of sessions allowed
@@ -66,10 +66,12 @@ typedef struct SRpcInit {
   char *ckey;         // ciphering key
 
   // call back to process incoming msg, code shall be ignored by server app
-  void (*cfp)(SRpcMsg *, SEpSet *);  
+  void (*cfp)(void *parent, SRpcMsg *, SEpSet *);
 
-  // call back to retrieve the client auth info, for server app only 
-  int  (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+  // call back to retrieve the client auth info, for server app only
+  int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+
+  void *parent;
 } SRpcInit;
 
 int32_t rpcInit();
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index ba37e6880b..94346d705e 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -16,11 +16,21 @@
 #define _TD_WAL_H_
 
 #include "os.h"
-
+#include "tdef.h"
+#include "tlog.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+extern int32_t wDebugFlag;
+
+#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
+#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
+#define wWarn(...)  { if (wDebugFlag & DEBUG_WARN)  { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
+#define wInfo(...)  { if (wDebugFlag & DEBUG_INFO)  { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
+#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+
 typedef enum {
   TAOS_WAL_NOLOG = 0,
   TAOS_WAL_WRITE = 1,
@@ -28,9 +38,8 @@ typedef enum {
 } EWalType;
 
 typedef struct {
-  int8_t   msgType;
-  int8_t   sver;  // sver 2 for WAL SDataRow/SMemRow compatibility
-  int8_t   reserved[2];
+  int8_t   sver;
+  int8_t   reserved[3];
   int32_t  len;
   int64_t  version;
   uint32_t signature;
@@ -44,11 +53,33 @@ typedef struct {
   EWalType walLevel;     // wal level
 } SWalCfg;
 
+#define WAL_PREFIX     "wal"
+#define WAL_PREFIX_LEN 3
+#define WAL_REFRESH_MS 1000
+#define WAL_MAX_SIZE   (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
+#define WAL_SIGNATURE  ((uint32_t)(0xFAFBFDFE))
+#define WAL_PATH_LEN   (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN   (WAL_PATH_LEN + 32)
+#define WAL_FILE_NUM   1 // 3
+
 typedef struct SWal {
-  int8_t unused;
+  int64_t version;
+  int64_t fileId;
+  int64_t rId;
+  int64_t tfd;
+  int32_t vgId;
+  int32_t keep;
+  int32_t level;
+  int32_t fsyncPeriod;
+  int32_t fsyncSeq;
+  int8_t stop;
+  int8_t reseved[3];
+  char path[WAL_PATH_LEN];
+  char name[WAL_FILE_LEN];
+  pthread_mutex_t mutex;
 } SWal;  // WAL HANDLE
 
-typedef int32_t (*FWalWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
+typedef int32_t (*FWalWrite)(void *ahandle, void *pHead, void *pMsg);
 
 // module initialization
 int32_t walInit();
@@ -82,6 +113,11 @@ int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);
 // int32_t  walDataCorrupted(SWal*);
 
+//internal
+int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId);
+int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId);
+int32_t walGetNewFile(SWal *pWal, int64_t *newFileId);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/os/os.h b/include/os/os.h
index 44ce6f81ec..53a6cef96a 100644
--- a/include/os/os.h
+++ b/include/os/os.h
@@ -46,6 +46,7 @@ extern "C" {
 #include <math.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <dirent.h>
 #include <unistd.h>
 
 #include "osAtomic.h"
diff --git a/include/os/osDir.h b/include/os/osDir.h
index 8aefaa171a..35c294af39 100644
--- a/include/os/osDir.h
+++ b/include/os/osDir.h
@@ -20,12 +20,12 @@
 extern "C" {
 #endif
 
-void taosRemoveDir(const char *dirname);
-bool taosDirExist(char *dirname);
-bool taosMkDir(const char *dirname);
-void taosRemoveOldFiles(char *dirname, int32_t keepDays);
-bool taosExpandDir(char *dirname, char *outname, int32_t maxlen);
-bool taosRealPath(char *dirname, int32_t maxlen);
+void    taosRemoveDir(const char *dirname);
+int32_t taosDirExist(char *dirname);
+int32_t taosMkDir(const char *dirname);
+void    taosRemoveOldFiles(char *dirname, int32_t keepDays);
+int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen);
+int32_t taosRealPath(char *dirname, int32_t maxlen);
 
 #ifdef __cplusplus
 }
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 022c3e9096..304fb56a6a 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -68,12 +68,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FILE_CORRUPTED                TAOS_DEF_ERROR_CODE(0, 0x0106)
 #define TSDB_CODE_CHECKSUM_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0107)
 #define TSDB_CODE_INVALID_MSG                   TAOS_DEF_ERROR_CODE(0, 0x0108)
-#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0109)
-#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x010A)
-#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x010B)
-#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x010C)
-#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x010D)
-#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x010E)
+#define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0109)
+#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
+#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
+#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0112)
+#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x0113)
+#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0114)
+#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x0115)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)  //"Invalid Operation")
@@ -223,20 +224,20 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0395)  //"Topic already exists)
 
 // dnode
-#define TSDB_CODE_DND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0400)  //"Message not processed")
-#define TSDB_CODE_DND_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x0401)  //"Dnode out of memory")
-#define TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE  TAOS_DEF_ERROR_CODE(0, 0x0402)  //"Mnode Id not match Dnode")
-#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0403)  //"Mnode already deployed")
-#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0404)  //"Mnode not deployed")
-#define TSDB_CODE_DND_READ_MNODE_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0405)  //"Read mnode.json error")
-#define TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0406)  //"Write mnode.json error")
-#define TSDB_CODE_DND_NO_WRITE_ACCESS           TAOS_DEF_ERROR_CODE(0, 0x0407)  //"No permission for disk files in dnode")
-#define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0408)  //"Invalid message length")
-#define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0409)  //"Action in progress")
-#define TSDB_CODE_DND_TOO_MANY_VNODES           TAOS_DEF_ERROR_CODE(0, 0x040A)  //"Too many vnode directories")
-#define TSDB_CODE_DND_EXITING                   TAOS_DEF_ERROR_CODE(0, 0x040B)  //"Dnode is exiting"
-#define TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x040C)  //"Parse vnodes.json error")
-#define TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x040D)  //"Parse dnodes.json error")
+#define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)
+#define TSDB_CODE_DND_EXITING                   TAOS_DEF_ERROR_CODE(0, 0x0401)
+#define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0402)
+#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0410)
+#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0411)
+#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0420)
+#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0421)
+#define TSDB_CODE_DND_MNODE_ID_INVALID          TAOS_DEF_ERROR_CODE(0, 0x0422)
+#define TSDB_CODE_DND_MNODE_ID_NOT_FOUND        TAOS_DEF_ERROR_CODE(0, 0x0423)
+#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0424)
+#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0425)
+#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0430)
+#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0431)
+#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0432)
 
 // vnode
 #define TSDB_CODE_VND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0500)  //"Action in progress")
diff --git a/include/util/tconfig.h b/include/util/tconfig.h
index bc56d21b3a..15100423d5 100644
--- a/include/util/tconfig.h
+++ b/include/util/tconfig.h
@@ -82,10 +82,10 @@ extern SGlobalCfg tsGlobalConfig[];
 extern int32_t    tsGlobalConfigNum;
 extern char *     tsCfgStatusStr[];
 
-void taosReadGlobalLogCfg();
-bool taosReadGlobalCfg();
-void taosPrintGlobalCfg();
-void taosDumpGlobalCfg();
+void    taosReadGlobalLogCfg();
+int32_t taosReadGlobalCfg();
+void    taosPrintGlobalCfg();
+void    taosDumpGlobalCfg();
 
 void        taosInitConfigOption(SGlobalCfg cfg);
 SGlobalCfg *taosGetConfigOption(const char *option);
diff --git a/include/util/tlog.h b/include/util/tlog.h
index 5c6e59b103..2ee60e4324 100644
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@@ -16,6 +16,8 @@
 #ifndef _TD_UTIL_LOG_H
 #define _TD_UTIL_LOG_H
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/include/util/tnote.h b/include/util/tnote.h
index e4f28d8cff..e613ec7e41 100644
--- a/include/util/tnote.h
+++ b/include/util/tnote.h
@@ -38,9 +38,9 @@ extern SNoteObj tsHttpNote;
 extern SNoteObj tsTscNote;
 extern SNoteObj tsInfoNote;
 
-void taosInitNotes();
-void taosNotePrint(SNoteObj* pNote, const char* const format, ...);
-void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len);
+int32_t taosInitNotes();
+void    taosNotePrint(SNoteObj* pNote, const char* const format, ...);
+void    taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len);
 
 #define nPrintHttp(...)                      \
   if (tsHttpEnableRecordSql) {               \
@@ -53,7 +53,7 @@ void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len);
   }
 
 #define nInfo(buffer, len)                         \
-  if (tscEmbedded == 1) {                          \
+  if (tscEmbeddedInUtil == 1) {                          \
     taosNotePrintBuffer(&tsInfoNote, buffer, len); \
   }
 
diff --git a/include/util/ulog.h b/include/util/ulog.h
index ba59f4eb79..89d9f89476 100644
--- a/include/util/ulog.h
+++ b/include/util/ulog.h
@@ -20,20 +20,21 @@
 extern "C" {
 #endif
 
+#include "os.h"
 #include "tlog.h"
 
 extern int32_t uDebugFlag;
-extern int8_t  tscEmbedded;
+extern int8_t tscEmbeddedInUtil;
 
-#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
-#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
-#define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
-#define uInfo(...)  { if (uDebugFlag & DEBUG_INFO)  { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
+#define uInfo(...)  { if (uDebugFlag & DEBUG_INFO)  { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
 #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
 
-#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }  
-#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }  
+#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }
+#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }
 
 #ifdef __cplusplus
 }
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index a19f6ec04b..501936d354 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -1679,7 +1679,7 @@ int32_t taosCheckGlobalCfg() {
 
   taosCheckDataDirCfg();
 
-  if (!taosDirExist(tsTempDir)) {
+  if (taosDirExist(tsTempDir) != 0) {
     return -1;
   }
   
diff --git a/source/dnode/CMakeLists.txt b/source/dnode/CMakeLists.txt
index d719a2d106..af132dea80 100644
--- a/source/dnode/CMakeLists.txt
+++ b/source/dnode/CMakeLists.txt
@@ -1,4 +1,4 @@
 add_subdirectory(mnode)
 add_subdirectory(vnode)
 add_subdirectory(qnode)
-add_subdirectory(mgmt)
+add_subdirectory(mgmt)
\ No newline at end of file
diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt
index 194c317991..64e8980219 100644
--- a/source/dnode/mgmt/CMakeLists.txt
+++ b/source/dnode/mgmt/CMakeLists.txt
@@ -1,16 +1,2 @@
-aux_source_directory(src DNODE_SRC)
-add_executable(taosd ${DNODE_SRC})
-target_link_libraries(
-    taosd
-    PUBLIC cjson
-    PUBLIC mnode
-    PUBLIC vnode
-    PUBLIC wal
-    PUBLIC sync
-    PUBLIC taos
-)
-target_include_directories(
-    taosd
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode"
-    private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
+add_subdirectory(daemon)
+add_subdirectory(impl)
\ No newline at end of file
diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt
new file mode 100644
index 0000000000..f1ce726d85
--- /dev/null
+++ b/source/dnode/mgmt/daemon/CMakeLists.txt
@@ -0,0 +1,8 @@
+aux_source_directory(src DAEMON_SRC)
+add_executable(taosd ${DAEMON_SRC})
+target_link_libraries(
+    taosd
+    PUBLIC dnode
+    PUBLIC util
+    PUBLIC os
+)
diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c
new file mode 100644
index 0000000000..4d1116466c
--- /dev/null
+++ b/source/dnode/mgmt/daemon/src/daemon.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dnode.h"
+#include "os.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tnote.h"
+#include "ulog.h"
+
+static struct {
+  bool stop;
+  bool dumpConfig;
+  bool generateGrant;
+  bool printAuth;
+  bool printVersion;
+  char configDir[PATH_MAX];
+} global = {0};
+
+void dmnSigintHandle(int signum, void *info, void *ctx) { global.stop = true; }
+
+void dmnSetSignalHandle() {
+  taosSetSignal(SIGTERM, dmnSigintHandle);
+  taosSetSignal(SIGHUP, dmnSigintHandle);
+  taosSetSignal(SIGINT, dmnSigintHandle);
+  taosSetSignal(SIGABRT, dmnSigintHandle);
+  taosSetSignal(SIGBREAK, dmnSigintHandle);
+}
+
+int dmnParseOption(int argc, char const *argv[]) {
+  tstrncpy(global.configDir, "/etc/taos", PATH_MAX);
+
+  for (int i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(global.configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-C") == 0) {
+      global.dumpConfig = true;
+    } else if (strcmp(argv[i], "-k") == 0) {
+      global.generateGrant = true;
+    } else if (strcmp(argv[i], "-A") == 0) {
+      global.printAuth = true;
+    } else if (strcmp(argv[i], "-V") == 0) {
+      global.printVersion = true;
+    } else {
+    }
+  }
+
+  return 0;
+}
+
+void dmnGenerateGrant() {
+#if 0
+  grantParseParameter();
+#endif
+}
+
+void dmnPrintVersion() {
+#ifdef TD_ENTERPRISE
+  char *releaseName = "enterprise";
+#else
+  char *releaseName = "community";
+#endif
+  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
+  printf("gitinfo: %s\n", gitinfo);
+  printf("gitinfoI: %s\n", gitinfoOfInternal);
+  printf("builuInfo: %s\n", buildinfo);
+}
+
+int dmnReadConfig(const char *path) {
+  taosInitGlobalCfg();
+  taosReadGlobalLogCfg();
+
+  if (taosMkDir(tsLogDir) != 0) {
+    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
+    return -1;
+  }
+
+  char temp[PATH_MAX];
+  snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir);
+  if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
+    printf("failed to init log file\n");
+    return -1;
+  }
+
+  if (taosInitNotes() != 0) {
+    printf("failed to init log file\n");
+    return -1;
+  }
+
+  if (taosReadGlobalCfg() != 0) {
+    uError("failed to read global config");
+    return -1;
+  }
+
+  if (taosCheckGlobalCfg() != 0) {
+    uError("failed to check global config");
+    return -1;
+  }
+
+  taosSetCoreDump(tsEnableCoreFile);
+  return 0;
+}
+
+void dmnDumpConfig() { taosDumpGlobalCfg(); }
+
+void dmnWaitSignal() {
+  dmnSetSignalHandle();
+  while (!global.stop) {
+    taosMsleep(100);
+  }
+}
+
+void dmnInitOption(SDnodeOpt *pOption) {
+  pOption->sver = tsVersion;
+  pOption->numOfCores = tsNumOfCores;
+  pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore;
+  pOption->ratioOfQueryCores = tsRatioOfQueryCores;
+  pOption->maxShellConns = tsMaxShellConns;
+  pOption->shellActivityTimer = tsShellActivityTimer;
+  pOption->statusInterval = tsStatusInterval;
+  pOption->serverPort = tsServerPort;
+  tstrncpy(pOption->dataDir, tsDataDir, TSDB_EP_LEN);
+  tstrncpy(pOption->localEp, tsLocalEp, TSDB_EP_LEN);
+  tstrncpy(pOption->localFqdn, tsLocalEp, TSDB_FQDN_LEN);
+  tstrncpy(pOption->firstEp, tsFirst, TSDB_FQDN_LEN);
+  tstrncpy(pOption->timezone, tsLocalEp, TSDB_TIMEZONE_LEN);
+  tstrncpy(pOption->locale, tsLocalEp, TSDB_LOCALE_LEN);
+  tstrncpy(pOption->charset, tsLocalEp, TSDB_LOCALE_LEN);
+}
+
+int dmnRunDnode() {
+  SDnodeOpt option = {0};
+  dmnInitOption(&option);
+
+  SDnode *pDnode = dndInit(&option);
+  if (pDnode == NULL) {
+    uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
+    return -1;
+  }
+
+  uInfo("Started TDengine service successfully.");
+  dmnWaitSignal();
+  uInfo("TDengine is shut down!");
+
+  dndCleanup(pDnode);
+  taosCloseLog();
+  return 0;
+}
+
+int main(int argc, char const *argv[]) {
+  if (dmnParseOption(argc, argv) != 0) {
+    return -1;
+  }
+
+  if (global.generateGrant) {
+    dmnGenerateGrant();
+    return 0;
+  }
+
+  if (global.printVersion) {
+    dmnPrintVersion();
+    return 0;
+  }
+
+  if (dmnReadConfig(global.configDir) != 0) {
+    return -1;
+  }
+
+  if (global.dumpConfig) {
+    dmnDumpConfig();
+    return 0;
+  }
+
+  return dmnRunDnode();
+}
diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt
new file mode 100644
index 0000000000..d0830484ed
--- /dev/null
+++ b/source/dnode/mgmt/impl/CMakeLists.txt
@@ -0,0 +1,16 @@
+aux_source_directory(src DNODE_SRC)
+add_library(dnode STATIC ${DNODE_SRC})
+target_link_libraries(
+    dnode
+    PUBLIC cjson
+    PUBLIC mnode
+    PUBLIC vnode
+    PUBLIC wal
+    PUBLIC sync
+    PUBLIC taos
+)
+target_include_directories(
+    dnode
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
+    private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
diff --git a/source/dnode/mgmt/inc/dnodeDnode.h b/source/dnode/mgmt/impl/inc/dndDnode.h
similarity index 53%
rename from source/dnode/mgmt/inc/dnodeDnode.h
rename to source/dnode/mgmt/impl/inc/dndDnode.h
index 2ca1368e63..ef16b1c8f0 100644
--- a/source/dnode/mgmt/inc/dnodeDnode.h
+++ b/source/dnode/mgmt/impl/inc/dndDnode.h
@@ -13,27 +13,27 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DNODE_DNODE_H_
-#define _TD_DNODE_DNODE_H_
+#ifndef _TD_DND_DNODE_H_
+#define _TD_DND_DNODE_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dnodeInt.h"
+#include "dndInt.h"
 
-int32_t dnodeInitDnode();
-void    dnodeCleanupDnode();
-void    dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
+int32_t dndInitDnode(SDnode *pDnd);
+void    dndCleanupDnode(SDnode *pDnd);
+void    dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet);
 
-int32_t dnodeGetDnodeId();
-int64_t dnodeGetClusterId();
-void    dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
-void    dnodeGetMnodeEpSetForPeer(SEpSet *epSet);
-void    dnodeGetMnodeEpSetForShell(SEpSet *epSet);
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
+int32_t dndGetDnodeId(SDnode *pDnd);
+int64_t dndGetClusterId(SDnode *pDnd);
+void    dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
+void    dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet);
+void    dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DNODE_DNODE_H_*/
\ No newline at end of file
+#endif /*_TD_DND_DNODE_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h
new file mode 100644
index 0000000000..966781426b
--- /dev/null
+++ b/source/dnode/mgmt/impl/inc/dndInt.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_INT_H_
+#define _TD_DND_INT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "cJSON.h"
+#include "os.h"
+#include "taosmsg.h"
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tqueue.h"
+#include "trpc.h"
+#include "tthread.h"
+#include "ttime.h"
+#include "tworker.h"
+#include "mnode.h"
+#include "vnode.h"
+#include "dnode.h"
+
+extern int32_t dDebugFlag;
+
+#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("SRV FATAL ", 255, __VA_ARGS__); }}
+#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("SRV ERROR ", 255, __VA_ARGS__); }}
+#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("SRV WARN ", 255, __VA_ARGS__); }}
+#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("SRV ", 255, __VA_ARGS__); }}
+#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }}
+#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }}
+
+typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
+typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
+
+typedef struct {
+  char *dnode;
+  char *mnode;
+  char *vnodes;
+} SDnodeDir;
+
+typedef struct {
+  int32_t         dnodeId;
+  uint32_t        rebootTime;
+  int32_t         dropped;
+  int64_t         clusterId;
+  SEpSet          shellEpSet;
+  SEpSet          peerEpSet;
+  char           *file;
+  SHashObj       *dnodeHash;
+  SDnodeEps      *dnodeEps;
+  pthread_t      *threadId;
+  pthread_mutex_t mutex;
+} SDnodeMgmt;
+
+typedef struct {
+  int32_t     refCount;
+  int8_t      deployed;
+  int8_t      dropped;
+  SWorkerPool mgmtPool;
+  SWorkerPool readPool;
+  SWorkerPool writePool;
+  SWorkerPool syncPool;
+  taos_queue  pReadQ;
+  taos_queue  pWriteQ;
+  taos_queue  pApplyQ;
+  taos_queue  pSyncQ;
+  taos_queue  pMgmtQ;
+  char       *file;
+  SMnode     *pMnode;
+  SRWLatch    latch;
+} SMnodeMgmt;
+
+typedef struct {
+  SHashObj    *hash;
+  SWorkerPool  mgmtPool;
+  SWorkerPool  queryPool;
+  SWorkerPool  fetchPool;
+  SMWorkerPool syncPool;
+  SMWorkerPool writePool;
+  taos_queue   pMgmtQ;
+  int32_t      openVnodes;
+  int32_t      totalVnodes;
+  SRWLatch     latch;
+} SVnodesMgmt;
+
+typedef struct {
+  void    *serverRpc;
+  void    *clientRpc;
+  DndMsgFp msgFp[TSDB_MSG_TYPE_MAX];
+} STransMgmt;
+
+typedef struct SDnode {
+  EStat       stat;
+  SDnodeOpt   opt;
+  SDnodeDir   dir;
+  SDnodeMgmt  d;
+  SMnodeMgmt  m;
+  SVnodesMgmt vmgmt;
+  STransMgmt  t;
+  SStartupMsg startup;
+} SDnode;
+
+EStat dndGetStat(SDnode *pDnode);
+void  dndSetStat(SDnode *pDnode, EStat stat);
+char *dndStatStr(EStat stat);
+
+void dndReportStartup(SDnode *pDnode, char *name, char *desc);
+void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_INT_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/inc/dnodeVnodes.h b/source/dnode/mgmt/impl/inc/dndMnode.h
similarity index 53%
rename from source/dnode/mgmt/inc/dnodeVnodes.h
rename to source/dnode/mgmt/impl/inc/dndMnode.h
index 31eae049ab..67c51e51a8 100644
--- a/source/dnode/mgmt/inc/dnodeVnodes.h
+++ b/source/dnode/mgmt/impl/inc/dndMnode.h
@@ -13,26 +13,24 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DNODE_VNODES_H_
-#define _TD_DNODE_VNODES_H_
+#ifndef _TD_DND_MNODE_H_
+#define _TD_DND_MNODE_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dnodeInt.h"
+#include "dndInt.h"
 
-int32_t dnodeInitVnodes();
-void    dnodeCleanupVnodes();
-void    dnodeGetVnodeLoads(SVnodeLoads *pVloads);
-
-void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
+int32_t dndInitMnode(SDnode *pDnode);
+void    dndCleanupMnode(SDnode *pDnode);
+int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
+void    dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DNODE_VNODES_H_*/
\ No newline at end of file
+#endif /*_TD_DND_MNODE_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/inc/dnodeTransport.h b/source/dnode/mgmt/impl/inc/dndTransport.h
similarity index 68%
rename from source/dnode/mgmt/inc/dnodeTransport.h
rename to source/dnode/mgmt/impl/inc/dndTransport.h
index 95ca1b81e5..312da69fa2 100644
--- a/source/dnode/mgmt/inc/dnodeTransport.h
+++ b/source/dnode/mgmt/impl/inc/dndTransport.h
@@ -13,21 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DNODE_TRANSPORT_H_
-#define _TD_DNODE_TRANSPORT_H_
+#ifndef _TD_DND_TRANSPORT_H_
+#define _TD_DND_TRANSPORT_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dnodeInt.h"
+#include "dndInt.h"
 
-int32_t dnodeInitTrans();
-void    dnodeCleanupTrans();
-void    dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
-void    dnodeSendMsgToDnode(SEpSet *epSet, SRpcMsg *rpcMsg);
+int32_t dndInitTrans(SDnode *pDnode);
+void    dndCleanupTrans(SDnode *pDnode);
+void    dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg);
+void    dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DNODE_TRANSPORT_H_*/
+#endif /*_TD_DND_TRANSPORT_H_*/
diff --git a/source/dnode/mgmt/inc/dnodeMnode.h b/source/dnode/mgmt/impl/inc/dndVnodes.h
similarity index 52%
rename from source/dnode/mgmt/inc/dnodeMnode.h
rename to source/dnode/mgmt/impl/inc/dndVnodes.h
index 9f52e586fe..35f99ee73b 100644
--- a/source/dnode/mgmt/inc/dnodeMnode.h
+++ b/source/dnode/mgmt/impl/inc/dndVnodes.h
@@ -13,25 +13,25 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DNODE_MNODE_H_
-#define _TD_DNODE_MNODE_H_
+#ifndef _TD_DND_VNODES_H_
+#define _TD_DND_VNODES_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dnodeInt.h"
+#include "dndInt.h"
 
-int32_t dnodeInitMnode();
-void    dnodeCleanupMnode();
-int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-
-void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
+int32_t dndInitVnodes(SDnode *pDnode);
+void    dndCleanupVnodes(SDnode *pDnode);
+void    dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pVloads);
+void    dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void    dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DNODE_MNODE_H_*/
\ No newline at end of file
+#endif /*_TD_DND_VNODES_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c
new file mode 100644
index 0000000000..378d76e046
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndDnode.c
@@ -0,0 +1,582 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndDnode.h"
+#include "dndTransport.h"
+#include "dndVnodes.h"
+
+static inline void dndLockDnode(SDnode *pDnd) { pthread_mutex_lock(&pDnd->d.mutex); }
+
+static inline void dndUnLockDnode(SDnode *pDnd) { pthread_mutex_unlock(&pDnd->d.mutex); }
+
+int32_t dndGetDnodeId(SDnode *pDnd) {
+  dndLockDnode(pDnd);
+  int32_t dnodeId = pDnd->d.dnodeId;
+  dndUnLockDnode(pDnd);
+  return dnodeId;
+}
+
+int64_t dndGetClusterId(SDnode *pDnd) {
+  dndLockDnode(pDnd);
+  int64_t clusterId = pDnd->d.clusterId;
+  dndUnLockDnode(pDnd);
+  return clusterId;
+}
+
+void dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
+  dndLockDnode(pDnd);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    if (pPort != NULL) {
+      *pPort = pDnodeEp->port;
+    }
+    if (pFqdn != NULL) {
+      tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN);
+    }
+    if (pEp != NULL) {
+      snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
+    }
+  }
+
+  dndUnLockDnode(pDnd);
+}
+
+void dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+  dndLockDnode(pDnd);
+  *pEpSet = pDnd->d.peerEpSet;
+  dndUnLockDnode(pDnd);
+}
+
+void dndGetShellEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+  dndLockDnode(pDnd);
+  *pEpSet = pDnd->d.shellEpSet;
+  dndUnLockDnode(pDnd);
+}
+
+void dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell) {
+  int32_t msgType = pMsg->msgType;
+
+  SEpSet epSet = {0};
+  if (forShell) {
+    dndGetShellEpSet(pDnd, &epSet);
+  } else {
+    dndGetMnodeEpSet(pDnd, &epSet);
+  }
+
+  dDebug("RPC %p, msg:%s is redirected, num:%d use:%d", pMsg->handle, taosMsg[msgType], epSet.numOfEps, epSet.inUse);
+
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]);
+    if (strcmp(epSet.fqdn[i], pDnd->opt.localFqdn) == 0) {
+      if ((epSet.port[i] == pDnd->opt.serverPort + TSDB_PORT_DNODEDNODE && !forShell) ||
+          (epSet.port[i] == pDnd->opt.serverPort && forShell)) {
+        epSet.inUse = (i + 1) % epSet.numOfEps;
+        dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse);
+      }
+    }
+
+    epSet.port[i] = htons(epSet.port[i]);
+  }
+
+  rpcSendRedirectRsp(pMsg->handle, &epSet);
+}
+
+static void dndUpdateMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+  dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
+
+  dndLockDnode(pDnd);
+
+  pDnd->d.peerEpSet = *pEpSet;
+  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
+    pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
+    dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
+  }
+  pDnd->d.shellEpSet = *pEpSet;
+
+  dndUnLockDnode(pDnd);
+}
+
+static void dndPrintDnodes(SDnode *pDnd) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  dDebug("print dnode endpoint list, num:%d", pDnode->dnodeEps->num);
+  for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) {
+    SDnodeEp *pEp = &pDnode->dnodeEps->eps[i];
+    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode);
+  }
+}
+
+static void dndResetDnodes(SDnode *pDnd, SDnodeEps *pDnodeEps) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  int32_t size = sizeof(SDnodeEps) + pDnodeEps->num * sizeof(SDnodeEp);
+
+  if (pDnodeEps->num > pDnode->dnodeEps->num) {
+    SDnodeEps *tmp = calloc(1, size);
+    if (tmp == NULL) return;
+
+    tfree(pDnode->dnodeEps);
+    pDnode->dnodeEps = tmp;
+  }
+
+  if (pDnode->dnodeEps != pDnodeEps) {
+    memcpy(pDnode->dnodeEps, pDnodeEps, size);
+  }
+
+  pDnode->peerEpSet.inUse = 0;
+  pDnode->shellEpSet.inUse = 0;
+
+  int32_t mIndex = 0;
+  for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) {
+    SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+    if (!pDnodeEp->isMnode) continue;
+    if (mIndex >= TSDB_MAX_REPLICA) continue;
+    strcpy(pDnode->shellEpSet.fqdn[mIndex], pDnodeEp->fqdn);
+    strcpy(pDnode->peerEpSet.fqdn[mIndex], pDnodeEp->fqdn);
+    pDnode->shellEpSet.port[mIndex] = pDnodeEp->port;
+    pDnode->shellEpSet.port[mIndex] = pDnodeEp->port + TSDB_PORT_DNODEDNODE;
+    mIndex++;
+  }
+
+  for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) {
+    SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+    taosHashPut(pDnode->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
+  }
+
+  dndPrintDnodes(pDnd);
+}
+
+static bool dndIsEpChanged(SDnode *pDnd, int32_t dnodeId) {
+  bool changed = false;
+  dndLockDnode(pDnd);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    char epstr[TSDB_EP_LEN + 1];
+    snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
+    changed = strcmp(pDnd->opt.localEp, epstr) != 0;
+  }
+
+  dndUnLockDnode(pDnd);
+  return changed;
+}
+
+static int32_t dndReadDnodes(SDnode *pDnd) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  int32_t len = 0;
+  int32_t maxLen = 30000;
+  char   *content = calloc(1, maxLen + 1);
+  cJSON  *root = NULL;
+  FILE   *fp = NULL;
+
+  fp = fopen(pDnode->file, "r");
+  if (!fp) {
+    dDebug("file %s not exist", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  len = (int32_t)fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_String) {
+    dError("failed to read %s since dnodeId not found", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dnodeId = atoi(dnodeId->valuestring);
+
+  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_String) {
+    dError("failed to read %s since clusterId not found", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->clusterId = atoll(clusterId->valuestring);
+
+  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
+  if (!dropped || dropped->type != cJSON_String) {
+    dError("failed to read %s since dropped not found", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dropped = atoi(dropped->valuestring);
+
+  cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
+  if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
+    dError("failed to read %s since dnodeInfos not found", pDnode->file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
+  if (dnodeInfosSize <= 0) {
+    dError("failed to read %s since dnodeInfos size:%d invalid", pDnode->file, dnodeInfosSize);
+    goto PRASE_DNODE_OVER;
+  }
+
+  pDnode->dnodeEps = calloc(1, dnodeInfosSize * sizeof(SDnodeEp) + sizeof(SDnodeEps));
+  if (pDnode->dnodeEps == NULL) {
+    dError("failed to calloc dnodeEpList since %s", strerror(errno));
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dnodeEps->num = dnodeInfosSize;
+
+  for (int32_t i = 0; i < dnodeInfosSize; ++i) {
+    cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
+    if (dnodeInfo == NULL) break;
+
+    SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+
+    cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
+    if (!dnodeId || dnodeId->type != cJSON_String) {
+      dError("failed to read %s, dnodeId not found", pDnode->file);
+      goto PRASE_DNODE_OVER;
+    }
+    pDnodeEp->id = atoi(dnodeId->valuestring);
+
+    cJSON *isMnode = cJSON_GetObjectItem(dnodeInfo, "isMnode");
+    if (!isMnode || isMnode->type != cJSON_String) {
+      dError("failed to read %s, isMnode not found", pDnode->file);
+      goto PRASE_DNODE_OVER;
+    }
+    pDnodeEp->isMnode = atoi(isMnode->valuestring);
+
+    cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
+    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+      dError("failed to read %s, dnodeFqdn not found", pDnode->file);
+      goto PRASE_DNODE_OVER;
+    }
+    tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
+    if (!dnodePort || dnodePort->type != cJSON_String) {
+      dError("failed to read %s, dnodePort not found", pDnode->file);
+      goto PRASE_DNODE_OVER;
+    }
+    pDnodeEp->port = atoi(dnodePort->valuestring);
+  }
+
+  dInfo("succcessed to read file %s", pDnode->file);
+  dndPrintDnodes(pDnd);
+
+PRASE_DNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+
+  if (dndIsEpChanged(pDnd, pDnode->dnodeId)) {
+    dError("localEp %s different with %s and need reconfigured", pDnd->opt.localEp, pDnode->file);
+    return -1;
+  }
+
+  if (pDnode->dnodeEps == NULL) {
+    pDnode->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
+    pDnode->dnodeEps->num = 1;
+    pDnode->dnodeEps->eps[0].port = pDnd->opt.serverPort;
+    tstrncpy(pDnode->dnodeEps->eps[0].fqdn, pDnd->opt.localFqdn, TSDB_FQDN_LEN);
+  }
+
+  dndResetDnodes(pDnd, pDnode->dnodeEps);
+
+  terrno = 0;
+  return 0;
+}
+
+static int32_t dndWriteDnodes(SDnode *pDnd) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  FILE *fp = fopen(pDnode->file, "w");
+  if (!fp) {
+    dError("failed to write %s since %s", pDnode->file, strerror(errno));
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 30000;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": \"%d\",\n", pDnode->dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": \"%d\",\n", pDnode->dropped);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
+  for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) {
+    SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+    len += snprintf(content + len, maxLen - len, "    \"dnodeId\": \"%d\",\n", pDnodeEp->id);
+    len += snprintf(content + len, maxLen - len, "    \"isMnode\": \"%d\",\n", pDnodeEp->isMnode);
+    len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", pDnodeEp->fqdn);
+    len += snprintf(content + len, maxLen - len, "    \"dnodePort\": \"%u\"\n", pDnodeEp->port);
+    if (i < pDnode->dnodeEps->num - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  taosFsyncFile(fileno(fp));
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  dInfo("successed to write %s", pDnode->file);
+  return 0;
+}
+
+static void dndSendStatusMsg(SDnode *pDnd) {
+  int32_t     contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SStatusMsg *pStatus = rpcMallocCont(contLen);
+  if (pStatus == NULL) {
+    dError("failed to malloc status message");
+    return;
+  }
+
+  dndLockDnode(pDnd);
+  pStatus->sversion = htonl(pDnd->opt.sver);
+  pStatus->dnodeId = htonl(pDnd->d.dnodeId);
+  pStatus->clusterId = htobe64(pDnd->d.clusterId);
+  pStatus->rebootTime = htonl(pDnd->d.rebootTime);
+  pStatus->numOfCores = htonl(pDnd->opt.numOfCores);
+  tstrncpy(pStatus->dnodeEp, pDnd->opt.localEp, TSDB_EP_LEN);
+  pStatus->clusterCfg.statusInterval = htonl(pDnd->opt.statusInterval);
+  tstrncpy(pStatus->clusterCfg.timezone, pDnd->opt.timezone, TSDB_TIMEZONE_LEN);
+  tstrncpy(pStatus->clusterCfg.locale, pDnd->opt.locale, TSDB_LOCALE_LEN);
+  tstrncpy(pStatus->clusterCfg.charset, pDnd->opt.charset, TSDB_LOCALE_LEN);
+  pStatus->clusterCfg.checkTime = 0;
+  char timestr[32] = "1970-01-01 00:00:00.00";
+  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  dndUnLockDnode(pDnd);
+
+  dndGetVnodeLoads(pDnd, &pStatus->vnodeLoads);
+  contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad);
+
+  SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS};
+  dndSendMsgToMnode(pDnd, &rpcMsg);
+}
+
+static void dndUpdateDnodeCfg(SDnode *pDnd, SDnodeCfg *pCfg) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+  if (pDnode->dnodeId != 0 && pDnode->dropped != pCfg->dropped) return;
+
+  dndLockDnode(pDnd);
+
+  pDnode->dnodeId = pCfg->dnodeId;
+  pDnode->clusterId = pCfg->clusterId;
+  pDnode->dropped = pCfg->dropped;
+  dInfo("set dnodeId:%d clusterId:%" PRId64 " dropped:%d", pCfg->dnodeId, pCfg->clusterId, pCfg->dropped);
+
+  dndWriteDnodes(pDnd);
+  dndUnLockDnode(pDnd);
+}
+
+static void dndUpdateDnodeEps(SDnode *pDnd, SDnodeEps *pDnodeEps) {
+  if (pDnodeEps == NULL || pDnodeEps->num <= 0) return;
+
+  dndLockDnode(pDnd);
+
+  if (pDnodeEps->num != pDnd->d.dnodeEps->num) {
+    dndResetDnodes(pDnd, pDnodeEps);
+    dndWriteDnodes(pDnd);
+  } else {
+    int32_t size = pDnodeEps->num * sizeof(SDnodeEp) + sizeof(SDnodeEps);
+    if (memcmp(pDnd->d.dnodeEps, pDnodeEps, size) != 0) {
+      dndResetDnodes(pDnd, pDnodeEps);
+      dndWriteDnodes(pDnd);
+    }
+  }
+
+  dndUnLockDnode(pDnd);
+}
+
+static void dndProcessStatusRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  if (pEpSet && pEpSet->numOfEps > 0) {
+    dndUpdateMnodeEpSet(pDnd, pEpSet);
+  }
+
+  if (pMsg->code != TSDB_CODE_SUCCESS) return;
+
+  SStatusRsp *pStatusRsp = pMsg->pCont;
+  SDnodeCfg  *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
+  pCfg->clusterId = htobe64(pCfg->clusterId);
+  dndUpdateDnodeCfg(pDnd, pCfg);
+
+  if (pCfg->dropped) return;
+
+  SDnodeEps *pDnodeEps = &pStatusRsp->dnodeEps;
+  pDnodeEps->num = htonl(pDnodeEps->num);
+  for (int32_t i = 0; i < pDnodeEps->num; ++i) {
+    pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id);
+    pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port);
+  }
+
+  dndUpdateDnodeEps(pDnd, pDnodeEps);
+}
+
+static void dndProcessAuthRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
+
+static void dndProcessGrantRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
+
+static void dndProcessConfigDnodeReq(SDnode *pDnd, SRpcMsg *pMsg) {
+  SCfgDnodeMsg *pCfg = pMsg->pCont;
+
+  int32_t code = TSDB_CODE_OPS_NOT_SUPPORT;
+  SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code};
+  rpcSendResponse(&rspMsg);
+  rpcFreeCont(pMsg->pCont);
+}
+
+static void dndProcessStartupReq(SDnode *pDnd, SRpcMsg *pMsg) {
+  dInfo("startup msg is received");
+
+  SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg));
+  dndGetStartup(pDnd, pStartup);
+
+  dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
+
+  SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)};
+  rpcSendResponse(&rpcRsp);
+  rpcFreeCont(pMsg->pCont);
+}
+
+static void *dnodeThreadRoutine(void *param) {
+  SDnode *pDnd = param;
+  int32_t ms = pDnd->opt.statusInterval * 1000;
+
+  while (true) {
+    taosMsleep(ms);
+    if (dndGetStat(pDnd) != DND_STAT_RUNNING) {
+      continue;
+    }
+
+    pthread_testcancel();
+    dndSendStatusMsg(pDnd);
+  }
+}
+
+int32_t dndInitDnode(SDnode *pDnd) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  pDnode->dnodeId = 0;
+  pDnode->rebootTime = taosGetTimestampSec();
+  pDnode->dropped = 0;
+  pDnode->clusterId = 0;
+
+  char path[PATH_MAX];
+  snprintf(path, PATH_MAX, "%s/dnode.json", pDnd->dir.dnode);
+  pDnode->file = strdup(path);
+  if (pDnode->file == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pDnode->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (pDnode->dnodeHash == NULL) {
+    dError("failed to init dnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dndReadDnodes(pDnd) != 0) {
+    dError("failed to read file:%s since %s", pDnode->file, terrstr());
+    return -1;
+  }
+
+  pthread_mutex_init(&pDnode->mutex, NULL);
+
+  pDnode->threadId = taosCreateThread(dnodeThreadRoutine, pDnd);
+  if (pDnode->threadId == NULL) {
+    dError("failed to init dnode thread");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  dInfo("dnd-dnode is initialized");
+  return 0;
+}
+
+void dndCleanupDnode(SDnode *pDnd) {
+  SDnodeMgmt *pDnode = &pDnd->d;
+
+  if (pDnode->threadId != NULL) {
+    taosDestoryThread(pDnode->threadId);
+    pDnode->threadId = NULL;
+  }
+
+  dndLockDnode(pDnd);
+
+  if (pDnode->dnodeEps != NULL) {
+    free(pDnode->dnodeEps);
+    pDnode->dnodeEps = NULL;
+  }
+
+  if (pDnode->dnodeHash != NULL) {
+    taosHashCleanup(pDnode->dnodeHash);
+    pDnode->dnodeHash = NULL;
+  }
+
+  if (pDnode->file != NULL) {
+    free(pDnode->file);
+    pDnode->file = NULL;
+  }
+
+  dndUnLockDnode(pDnd);
+  pthread_mutex_destroy(&pDnode->mutex);
+
+  dInfo("dnd-dnode is cleaned up");
+}
+
+void dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  switch (pMsg->msgType) {
+    case TSDB_MSG_TYPE_NETWORK_TEST:
+      dndProcessStartupReq(pDnd, pMsg);
+      break;
+    case TSDB_MSG_TYPE_CONFIG_DNODE_IN:
+      dndProcessConfigDnodeReq(pDnd, pMsg);
+      break;
+    default:
+      dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+      SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
+      rpcSendResponse(&rspMsg);
+      rpcFreeCont(pMsg->pCont);
+  }
+}
+
+void dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  switch (pMsg->msgType) {
+    case TSDB_MSG_TYPE_STATUS_RSP:
+      dndProcessStatusRsp(pDnd, pMsg, pEpSet);
+      break;
+    case TSDB_MSG_TYPE_AUTH_RSP:
+      dndProcessAuthRsp(pDnd, pMsg, pEpSet);
+      break;
+    case TSDB_MSG_TYPE_GRANT_RSP:
+      dndProcessGrantRsp(pDnd, pMsg, pEpSet);
+      break;
+    default:
+      dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+  }
+}
diff --git a/source/dnode/mgmt/impl/src/dndInt.c b/source/dnode/mgmt/impl/src/dndInt.c
new file mode 100644
index 0000000000..f4cee3f6fd
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndInt.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndDnode.h"
+#include "dndMnode.h"
+#include "dndTransport.h"
+#include "dndVnodes.h"
+#include "sync.h"
+#include "tcache.h"
+#include "wal.h"
+#include "tcrc32c.h"
+
+EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
+
+void dndSetStat(SDnode *pDnode, EStat stat) {
+  dDebug("dnode stat set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat));
+  pDnode->stat = stat;
+}
+
+char *dndStatStr(EStat stat) {
+  switch (stat) {
+    case DND_STAT_INIT:
+      return "init";
+    case DND_STAT_RUNNING:
+      return "running";
+    case DND_STAT_STOPPED:
+      return "stopped";
+    default:
+      return "unknown";
+  }
+}
+
+void dndReportStartup(SDnode *pDnode, char *name, char *desc) {
+  SStartupMsg *pStartup = &pDnode->startup;
+  tstrncpy(pStartup->name, name, strlen(pStartup->name));
+  tstrncpy(pStartup->desc, desc, strlen(pStartup->desc));
+  pStartup->finished = 0;
+}
+
+void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) {
+  memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg));
+  pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
+}
+
+static int32_t dndCheckRunning(char *dataDir) {
+  char filepath[PATH_MAX] = {0};
+  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
+
+  FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
+  if (fd < 0) {
+    dError("failed to open lock file:%s since %s, quit", filepath, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  int32_t ret = taosLockFile(fd);
+  if (ret != 0) {
+    dError("failed to lock file:%s since %s, quit", filepath, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    taosCloseFile(fd);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOptions) {
+  if (dndCheckRunning(pOptions->dataDir) != 0) {
+    return -1;
+  }
+
+  char path[PATH_MAX + 100];
+  snprintf(path, sizeof(path), "%s%smnode", pOptions->dataDir, TD_DIRSEP);
+  pDnode->dir.mnode = strdup(path);
+
+  snprintf(path, sizeof(path), "%s%svnode", pOptions->dataDir, TD_DIRSEP);
+  pDnode->dir.vnodes = strdup(path);
+
+  snprintf(path, sizeof(path), "%s%sdnode", pOptions->dataDir, TD_DIRSEP);
+  pDnode->dir.dnode = strdup(path);
+
+  if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) {
+    dError("failed to malloc dir object");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (taosMkDir(pDnode->dir.dnode) != 0) {
+    dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (taosMkDir(pDnode->dir.mnode) != 0) {
+    dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (taosMkDir(pDnode->dir.vnodes) != 0) {
+    dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupEnv(SDnode *pDnode) {
+  if (pDnode->dir.mnode != NULL) {
+    tfree(pDnode->dir.mnode);
+  }
+
+  if (pDnode->dir.vnodes != NULL) {
+    tfree(pDnode->dir.vnodes);
+  }
+
+  if (pDnode->dir.dnode != NULL) {
+    tfree(pDnode->dir.dnode);
+  }
+
+  taosStopCacheRefreshWorker();
+}
+
+SDnode *dndInit(SDnodeOpt *pOptions) {
+  taosIgnSIGPIPE();
+  taosBlockSIGPIPE();
+  taosResolveCRC();
+
+  SDnode *pDnode = calloc(1, sizeof(pDnode));
+  if (pDnode == NULL) {
+    dError("failed to create dnode object");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  dInfo("start to initialize TDengine");
+  dndSetStat(pDnode, DND_STAT_INIT);
+
+  if (dndInitEnv(pDnode, pOptions) != 0) {
+    dError("failed to init env");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (rpcInit() != 0) {
+    dError("failed to init rpc env");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (walInit() != 0) {
+    dError("failed to init wal env");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (dndInitDnode(pDnode) != 0) {
+    dError("failed to init dnode");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (dndInitVnodes(pDnode) != 0) {
+    dError("failed to init vnodes");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (dndInitMnode(pDnode) != 0) {
+    dError("failed to init mnode");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  if (dndInitTrans(pDnode) != 0) {
+    dError("failed to init transport");
+    dndCleanup(pDnode);
+    return NULL;
+  }
+
+  dndSetStat(pDnode, DND_STAT_RUNNING);
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+  dInfo("TDengine is initialized successfully");
+
+  return 0;
+}
+
+void dndCleanup(SDnode *pDnode) {
+  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
+    dError("dnode is shutting down");
+    return;
+  }
+
+  dInfo("start to cleanup TDengine");
+  dndSetStat(pDnode, DND_STAT_STOPPED);
+  dndCleanupTrans(pDnode);
+  dndCleanupMnode(pDnode);
+  dndCleanupVnodes(pDnode);
+  dndCleanupDnode(pDnode);
+  walCleanUp();
+  rpcCleanup();
+
+  dInfo("TDengine is cleaned up successfully");
+  dndCleanupEnv(pDnode);
+  free(pDnode);
+}
diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c
new file mode 100644
index 0000000000..5f3e48d8a1
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndMnode.c
@@ -0,0 +1,816 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndMnode.h"
+#include "dndDnode.h"
+#include "dndTransport.h"
+
+static int32_t dndInitMnodeReadWorker(SDnode *pDnode);
+static int32_t dndInitMnodeWriteWorker(SDnode *pDnode);
+static int32_t dndInitMnodeSyncWorker(SDnode *pDnode);
+static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode);
+static void    dndCleanupMnodeReadWorker(SDnode *pDnode);
+static void    dndCleanupMnodeWriteWorker(SDnode *pDnode);
+static void    dndCleanupMnodeSyncWorker(SDnode *pDnode);
+static void    dndCleanupMnodeMgmtWorker(SDnode *pDnode);
+static int32_t dndAllocMnodeReadQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode);
+static void    dndFreeMnodeReadQueue(SDnode *pDnode);
+static void    dndFreeMnodeWriteQueue(SDnode *pDnode);
+static void    dndFreeMnodeApplyQueue(SDnode *pDnode);
+static void    dndFreeMnodeSyncQueue(SDnode *pDnode);
+static void    dndFreeMnodeMgmtQueue(SDnode *pDnode);
+
+static void    dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void    dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void    dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void    dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void    dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
+static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg);
+void           dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+
+static int32_t dndStartMnodeWorker(SDnode *pDnode);
+static void    dndStopMnodeWorker(SDnode *pDnode);
+
+static SMnode *dndAcquireMnode(SDnode *pDnode);
+static void    dndReleaseMnode(SDnode *pDnode, SMnode *pMnode);
+
+static int32_t dndReadMnodeFile(SDnode *pDnode);
+static int32_t dndWriteMnodeFile(SDnode *pDnode);
+
+static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions);
+static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions);
+static int32_t dndDropMnode(SDnode *pDnode);
+
+static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+
+static SMnode *dndAcquireMnode(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  SMnode     *pMnode = NULL;
+  int32_t     refCount = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+  if (pMgmt->deployed && !pMgmt->dropped) {
+    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
+    pMnode = pMgmt->pMnode;
+  } else {
+    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
+  }
+  taosRUnLockLatch(&pMgmt->latch);
+
+  dTrace("acquire mnode, refCount:%d", refCount);
+  return pMnode;
+}
+
+static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  int32_t     refCount = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+  if (pMnode != NULL) {
+    refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
+  }
+  taosRUnLockLatch(&pMgmt->latch);
+
+  dTrace("release mnode, refCount:%d", refCount);
+}
+
+static int32_t dndReadMnodeFile(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  int32_t     code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR;
+  int32_t     len = 0;
+  int32_t     maxLen = 300;
+  char       *content = calloc(1, maxLen + 1);
+  cJSON      *root = NULL;
+
+  FILE *fp = fopen(pMgmt->file, "r");
+  if (fp == NULL) {
+    dDebug("file %s not exist", pMgmt->file);
+    code = 0;
+    goto PRASE_MNODE_OVER;
+  }
+
+  len = (int32_t)fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", pMgmt->file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", pMgmt->file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_String) {
+    dError("failed to read %s since deployed not found", pMgmt->file);
+    goto PRASE_MNODE_OVER;
+  }
+  pMgmt->deployed = atoi(deployed->valuestring);
+
+  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
+  if (!dropped || dropped->type != cJSON_String) {
+    dError("failed to read %s since dropped not found", pMgmt->file);
+    goto PRASE_MNODE_OVER;
+  }
+  pMgmt->dropped = atoi(dropped->valuestring);
+
+  code = 0;
+  dInfo("succcessed to read file %s", pMgmt->file);
+
+PRASE_MNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+
+  terrno = code;
+  return code;
+}
+
+static int32_t dndWriteMnodeFile(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  char        file[PATH_MAX + 20] = {0};
+  snprintf(file, sizeof(file), "%s.bak", pMgmt->file);
+
+  FILE *fp = fopen(file, "w");
+  if (fp != NULL) {
+    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 300;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": \"%d\",\n", pMgmt->deployed);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": \"%d\"\n", pMgmt->dropped);
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  taosFsyncFile(fileno(fp));
+  fclose(fp);
+  free(content);
+
+  if (taosRenameFile(file, pMgmt->file) != 0) {
+    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
+    dError("failed to rename %s since %s", pMgmt->file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s", pMgmt->file);
+  return 0;
+}
+
+static int32_t dndStartMnodeWorker(SDnode *pDnode) {
+  if (dndAllocMnodeReadQueue(pDnode) != 0) {
+    dError("failed to alloc mnode read queue since %s", terrstr());
+    return -1;
+  }
+
+  if (dndAllocMnodeWriteQueue(pDnode) != 0) {
+    dError("failed to alloc mnode write queue since %s", terrstr());
+    return -1;
+  }
+
+  if (dndAllocMnodeApplyQueue(pDnode) != 0) {
+    dError("failed to alloc mnode apply queue since %s", terrstr());
+    return -1;
+  }
+
+  if (dndAllocMnodeSyncQueue(pDnode) != 0) {
+    dError("failed to alloc mnode sync queue since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndStopMnodeWorker(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  taosWLockLatch(&pMgmt->latch);
+  pMgmt->deployed = 0;
+  pMgmt->pMnode = NULL;
+  taosWUnLockLatch(&pMgmt->latch);
+
+  while (pMgmt->refCount > 1) taosMsleep(10);
+  while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pMgmt->pApplyQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10);
+
+  dndFreeMnodeReadQueue(pDnode);
+  dndFreeMnodeWriteQueue(pDnode);
+  dndFreeMnodeApplyQueue(pDnode);
+  dndFreeMnodeSyncQueue(pDnode);
+
+  dndCleanupMnodeReadWorker(pDnode);
+  dndCleanupMnodeWriteWorker(pDnode);
+  dndCleanupMnodeSyncWorker(pDnode);
+}
+
+static bool dndNeedDeployMnode(SDnode *pDnode) {
+  if (dndGetDnodeId(pDnode) > 0) {
+    return false;
+  }
+
+  if (dndGetClusterId(pDnode) > 0) {
+    return false;
+  }
+  if (strcmp(pDnode->opt.localEp, pDnode->opt.firstEp) != 0) {
+    return false;
+  }
+
+  return true;
+}
+
+static void dndInitMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions) {
+  pOptions->pDnode = pDnode;
+  pOptions->sendMsgToDnodeFp = dndSendMsgToDnode;
+  pOptions->sendMsgToMnodeFp = dndSendMsgToMnode;
+  pOptions->sendRedirectMsgFp = dndSendRedirectMsg;
+  pOptions->putMsgToApplyMsgFp = dndPutMsgIntoMnodeApplyQueue;
+}
+
+static int32_t dndBuildMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions, SCreateMnodeMsg *pMsg) {
+  dndInitMnodeOptions(pDnode, pOptions);
+
+  if (pMsg == NULL) {
+    pOptions->dnodeId = 1;
+    pOptions->clusterId = 1234;
+    pOptions->replica = 1;
+    pOptions->selfIndex = 0;
+    SReplica *pReplica = &pOptions->replicas[0];
+    pReplica->id = 1;
+    pReplica->port = pDnode->opt.serverPort;
+    tstrncpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN);
+  } else {
+    pOptions->dnodeId = dndGetDnodeId(pDnode);
+    pOptions->clusterId = dndGetClusterId(pDnode);
+    pOptions->selfIndex = -1;
+    pOptions->replica = pMsg->replica;
+    for (int32_t index = 0; index < pMsg->replica; ++index) {
+      SReplica *pReplica = &pOptions->replicas[index];
+      pReplica->id = pMsg->replicas[index].id;
+      pReplica->port = pMsg->replicas[index].port;
+      tstrncpy(pReplica->fqdn, pMsg->replicas[index].fqdn, TSDB_FQDN_LEN);
+      if (pReplica->id == pOptions->dnodeId) {
+        pOptions->selfIndex = index;
+      }
+    }
+  }
+
+  if (pOptions->selfIndex == -1) {
+    terrno = TSDB_CODE_DND_MNODE_ID_NOT_FOUND;
+    dError("failed to build mnode options since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  int32_t code = dndStartMnodeWorker(pDnode);
+  if (code != 0) {
+    dError("failed to start mnode worker since %s", terrstr());
+    return code;
+  }
+
+  SMnode *pMnode = mnodeOpen(pDnode->dir.mnode, pOptions);
+  if (pMnode == NULL) {
+    dError("failed to open mnode since %s", terrstr());
+    code = terrno;
+    dndStopMnodeWorker(pDnode);
+    terrno = code;
+    return code;
+  }
+
+  if (dndWriteMnodeFile(pDnode) != 0) {
+    dError("failed to write mnode file since %s", terrstr());
+    code = terrno;
+    dndStopMnodeWorker(pDnode);
+    mnodeClose(pMnode);
+    mnodeDestroy(pDnode->dir.mnode);
+    terrno = code;
+    return code;
+  }
+
+  taosWLockLatch(&pMgmt->latch);
+  pMgmt->pMnode = pMnode;
+  pMgmt->deployed = 1;
+  taosWUnLockLatch(&pMgmt->latch);
+
+  return 0;
+}
+
+static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL) {
+    dError("failed to alter mnode since %s", terrstr());
+    return -1;
+  }
+
+  if (mnodeAlter(pMnode, pOptions) != 0) {
+    dError("failed to alter mnode since %s", terrstr());
+    dndReleaseMnode(pDnode, pMnode);
+    return -1;
+  }
+
+  dndReleaseMnode(pDnode, pMnode);
+  return 0;
+}
+
+static int32_t dndDropMnode(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL) {
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  }
+
+  taosRLockLatch(&pMgmt->latch);
+  pMgmt->dropped = 1;
+  taosRUnLockLatch(&pMgmt->latch);
+
+  if (dndWriteMnodeFile(pDnode) != 0) {
+    taosRLockLatch(&pMgmt->latch);
+    pMgmt->dropped = 0;
+    taosRUnLockLatch(&pMgmt->latch);
+
+    dndReleaseMnode(pDnode, pMnode);
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  }
+
+  dndStopMnodeWorker(pDnode);
+  dndWriteMnodeFile(pDnode);
+  mnodeClose(pMnode);
+  mnodeDestroy(pDnode->dir.mnode);
+
+  return 0;
+}
+
+static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) {
+  SCreateMnodeMsg *pMsg = pRpcMsg->pCont;
+  pMsg->dnodeId = htonl(pMsg->dnodeId);
+  for (int32_t i = 0; i < pMsg->replica; ++i) {
+    pMsg->replicas[i].id = htonl(pMsg->replicas[i].id);
+    pMsg->replicas[i].port = htons(pMsg->replicas[i].port);
+  }
+
+  return pMsg;
+}
+
+static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+  SCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+  if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+    terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+    return -1;
+  } else {
+    SMnodeOptions option = {0};
+    if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) {
+      return -1;
+    }
+    return dndOpenMnode(pDnode, &option);
+  }
+}
+
+static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+  SAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+  if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+    terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+    return -1;
+  } else {
+    SMnodeOptions option = {0};
+    if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) {
+      return -1;
+    }
+    return dndAlterMnode(pDnode, &option);
+  }
+}
+
+static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+  SDropMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+  if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+    terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+    return -1;
+  } else {
+    return dndDropMnode(pDnode);
+  }
+}
+
+static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+  int32_t code = 0;
+
+  switch (pMsg->msgType) {
+    case TSDB_MSG_TYPE_CREATE_MNODE_IN:
+      code = dndProcessCreateMnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_ALTER_MNODE_IN:
+      code = dndProcessAlterMnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_DROP_MNODE_IN:
+      code = dndProcessDropMnodeReq(pDnode, pMsg);
+      break;
+    default:
+      code = TSDB_CODE_MSG_NOT_PROCESSED;
+      break;
+  }
+
+  SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
+  rpcSendResponse(&rsp);
+  rpcFreeCont(pMsg->pCont);
+  taosFreeQitem(pMsg);
+}
+
+static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode != NULL) {
+    mnodeProcessReadMsg(pMnode, pMsg);
+    dndReleaseMnode(pDnode, pMnode);
+  } else {
+    mnodeSendRsp(pMsg, terrno);
+  }
+
+  mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode != NULL) {
+    mnodeProcessWriteMsg(pMnode, pMsg);
+    dndReleaseMnode(pDnode, pMnode);
+  } else {
+    mnodeSendRsp(pMsg, terrno);
+  }
+
+  mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode != NULL) {
+    mnodeProcessApplyMsg(pMnode, pMsg);
+    dndReleaseMnode(pDnode, pMnode);
+  } else {
+    mnodeSendRsp(pMsg, terrno);
+  }
+
+  mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode != NULL) {
+    mnodeProcessSyncMsg(pMnode, pMsg);
+    dndReleaseMnode(pDnode, pMnode);
+  } else {
+    mnodeSendRsp(pMsg, terrno);
+  }
+
+  mnodeCleanupMsg(pMsg);
+}
+
+static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
+  assert(pQueue);
+
+  SMnodeMsg *pMsg = mnodeInitMsg(pMnode, pRpcMsg);
+  if (pMsg == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (taosWriteQitem(pQueue, pMsg) != 0) {
+    mnodeCleanupMsg(pMsg);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  SMnode     *pMnode = dndAcquireMnode(pDnode);
+
+  SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
+  if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) {
+    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pRpcMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  SMnode     *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pMsg->pCont);
+  }
+
+  dndReleaseMnode(pDnode, pMnode);
+}
+
+void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  SMnode     *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pMsg->pCont);
+  }
+
+  dndReleaseMnode(pDnode, pMnode);
+}
+
+void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  SMnode     *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pMsg->pCont);
+  }
+
+  dndReleaseMnode(pDnode, pMnode);
+}
+
+static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL) {
+    return -1;
+  }
+
+  int32_t code = taosWriteQitem(pMgmt->pApplyQ, pMsg);
+  dndReleaseMnode(pDnode, pMnode);
+  return code;
+}
+
+static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  pMgmt->pMgmtQ = tWorkerAllocQueue(&pMgmt->mgmtPool, NULL, (FProcessItem)dndProcessMnodeMgmtQueue);
+  if (pMgmt->pMgmtQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeMnodeMgmtQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ);
+  pMgmt->pMgmtQ = NULL;
+}
+
+static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode) {
+  SMnodeMgmt  *pMgmt = &pDnode->m;
+  SWorkerPool *pPool = &pMgmt->mgmtPool;
+  pPool->name = "mnode-mgmt";
+  pPool->min = 1;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupMnodeMgmtWorker(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  ;
+  tWorkerCleanup(&pMgmt->mgmtPool);
+}
+
+static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, NULL, (FProcessItem)dndProcessMnodeReadQueue);
+  if (pMgmt->pReadQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeMnodeReadQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ);
+  pMgmt->pReadQ = NULL;
+}
+
+static int32_t dndInitMnodeReadWorker(SDnode *pDnode) {
+  SMnodeMgmt  *pMgmt = &pDnode->m;
+  SWorkerPool *pPool = &pMgmt->readPool;
+  pPool->name = "mnode-read";
+  pPool->min = 0;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupMnodeReadWorker(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerCleanup(&pMgmt->readPool);
+}
+
+static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeWriteQueue);
+  if (pMgmt->pWriteQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeMnodeWriteQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ);
+  pMgmt->pWriteQ = NULL;
+}
+
+static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  pMgmt->pApplyQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeApplyQueue);
+  if (pMgmt->pApplyQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeMnodeApplyQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pApplyQ);
+  pMgmt->pApplyQ = NULL;
+}
+
+static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) {
+  SMnodeMgmt  *pMgmt = &pDnode->m;
+  SWorkerPool *pPool = &pMgmt->writePool;
+  pPool->name = "mnode-write";
+  pPool->min = 0;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupMnodeWriteWorker(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerCleanup(&pMgmt->writePool);
+}
+
+static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, NULL, (FProcessItem)dndProcessMnodeSyncQueue);
+  if (pMgmt->pSyncQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeMnodeSyncQueue(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ);
+  pMgmt->pSyncQ = NULL;
+}
+
+static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) {
+  SMnodeMgmt  *pMgmt = &pDnode->m;
+  SWorkerPool *pPool = &pMgmt->syncPool;
+  pPool->name = "mnode-sync";
+  pPool->min = 0;
+  pPool->max = 1;
+  return tWorkerInit(pPool);
+}
+
+static void dndCleanupMnodeSyncWorker(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  tWorkerCleanup(&pMgmt->syncPool);
+}
+
+int32_t dndInitMnode(SDnode *pDnode) {
+  dInfo("dnode-mnode start to init");
+  SMnodeMgmt *pMgmt = &pDnode->m;
+  taosInitRWLatch(&pMgmt->latch);
+
+  if (dndInitMnodeMgmtWorker(pDnode) != 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  char path[PATH_MAX];
+  snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode);
+  pMgmt->file = strdup(path);
+  if (pMgmt->file == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dndReadMnodeFile(pDnode) != 0) {
+    return -1;
+  }
+
+  if (pMgmt->dropped) {
+    dInfo("mnode has been deployed and needs to be deleted");
+    mnodeDestroy(pDnode->dir.mnode);
+    return 0;
+  }
+
+  if (!pMgmt->deployed) {
+    bool needDeploy = dndNeedDeployMnode(pDnode);
+    if (!needDeploy) {
+      dDebug("mnode does not need to be deployed");
+      return 0;
+    }
+
+    dInfo("start to deploy mnode");
+  } else {
+    dInfo("start to open mnode");
+  }
+
+  SMnodeOptions option = {0};
+  dndInitMnodeOptions(pDnode, &option);
+  return dndOpenMnode(pDnode, &option);
+}
+
+void dndCleanupMnode(SDnode *pDnode) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  dInfo("dnode-mnode start to clean up");
+  dndStopMnodeWorker(pDnode);
+  dndCleanupMnodeMgmtWorker(pDnode);
+  tfree(pMgmt->file);
+  dInfo("dnode-mnode is cleaned up");
+}
+
+int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SMnodeMgmt *pMgmt = &pDnode->m;
+
+  SMnode *pMnode = dndAcquireMnode(pDnode);
+  if (pMnode == NULL) {
+    terrno = TSDB_CODE_APP_NOT_READY;
+    dTrace("failed to get user auth since %s", terrstr());
+    return -1;
+  }
+
+  int32_t code = mnodeRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
+  dndReleaseMnode(pDnode, pMnode);
+  return code;
+}
diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c
new file mode 100644
index 0000000000..679e3ef5f9
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndTransport.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* this file is mainly responsible for the communication between DNODEs. Each
+ * dnode works as both server and client. Dnode may send status, grant, config
+ * messages to mnode, mnode may send create/alter/drop table/vnode messages
+ * to dnode. All theses messages are handled from here
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndTransport.h"
+#include "dndDnode.h"
+#include "dndMnode.h"
+#include "dndVnodes.h"
+
+static void dndInitMsgFp(STransMgmt *pMgmt) {
+  // msg from client to dnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_SUBMIT] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_QUERY] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_FETCH] = dndProcessVnodeFetchMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg;
+
+  // msg from client to mnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_USER] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_USER] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_USER] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DB] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DB] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg;
+
+  // message from client to dnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq;
+
+  // message from mnode to vnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+
+  // message from mnode to dnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dndProcessVnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dndProcessMnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dndProcessMnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dndProcessMnodeMgmtMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dndProcessDnodeReq;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+
+  // message from dnode to mnode
+  pMgmt->msgFp[TSDB_MSG_TYPE_AUTH] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dndProcessDnodeRsp;
+  pMgmt->msgFp[TSDB_MSG_TYPE_GRANT] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dndProcessDnodeRsp;
+  pMgmt->msgFp[TSDB_MSG_TYPE_STATUS] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dndProcessDnodeRsp;
+}
+
+static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SDnode     *pDnode = parent;
+  STransMgmt *pMgmt = &pDnode->t;
+
+  int32_t msgType = pMsg->msgType;
+
+  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
+    if (pMsg == NULL || pMsg->pCont == NULL) return;
+    dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]);
+    rpcFreeCont(pMsg->pCont);
+    return;
+  }
+
+  DndMsgFp fp = pMgmt->msgFp[msgType];
+  if (fp != NULL) {
+    dTrace("RPC %p, rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code));
+    (*fp)(pDnode, pMsg, pEpSet);
+  } else {
+    dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
+    rpcFreeCont(pMsg->pCont);
+  }
+}
+
+static int32_t dndInitClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->t;
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.label = "DND-C";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp = dndProcessResponse;
+  rpcInit.sessions = TSDB_MAX_VNODES << 4;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000;
+  rpcInit.user = "-internal";
+  rpcInit.ckey = "-key";
+  rpcInit.secret = "-secret";
+
+  pMgmt->clientRpc = rpcOpen(&rpcInit);
+  if (pMgmt->clientRpc == NULL) {
+    dError("failed to init rpc client");
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->t;
+  if (pMgmt->clientRpc) {
+    rpcClose(pMgmt->clientRpc);
+    pMgmt->clientRpc = NULL;
+    dInfo("dnode peer rpc client is closed");
+  }
+}
+
+static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SDnode *pDnode = param;
+  STransMgmt *pMgmt = &pDnode->t;
+
+  int32_t msgType = pMsg->msgType;
+  if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
+    dndProcessDnodeReq(pDnode, pMsg, pEpSet);
+    return;
+  }
+
+  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
+    dError("RPC %p, req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]);
+    SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_EXITING};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pMsg->pCont);
+    return;
+  } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) {
+    dError("RPC %p, req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
+    SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pMsg->pCont);
+    return;
+  }
+
+  if (pMsg->pCont == NULL) {
+    SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN};
+    rpcSendResponse(&rspMsg);
+    return;
+  }
+
+  DndMsgFp fp = pMgmt->msgFp[msgType];
+  if (fp != NULL) {
+    dTrace("RPC %p, req:%s will be processed", pMsg->handle, taosMsg[msgType]);
+    (*fp)(pDnode, pMsg, pEpSet);
+  } else {
+    dError("RPC %p, req:%s is not processed", pMsg->handle, taosMsg[msgType]);
+    SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pMsg->pCont);
+  }
+}
+
+static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
+  STransMgmt *pMgmt = &pDnode->t;
+
+  SEpSet epSet = {0};
+  dndGetMnodeEpSet(pDnode, &epSet);
+  rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
+}
+
+static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SDnode *pDnode = parent;
+
+  if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) != 0) {
+    if (terrno != TSDB_CODE_APP_NOT_READY) {
+      dTrace("failed to get user auth from mnode since %s", terrstr());
+      return -1;
+    }
+  }
+
+  dDebug("user:%s, send auth msg to mnodes", user);
+
+  SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
+  tstrncpy(pMsg->user, user, TSDB_USER_LEN);
+
+  SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH};
+  SRpcMsg rpcRsp = {0};
+  dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
+
+  if (rpcRsp.code != 0) {
+    terrno = rpcRsp.code;
+    dError("user:%s, failed to get user auth from mnodes since %s", user, terrstr());
+  } else {
+    SAuthRsp *pRsp = rpcRsp.pCont;
+    memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
+    memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
+    *spi = pRsp->spi;
+    *encrypt = pRsp->encrypt;
+    dDebug("user:%s, success to get user auth from mnodes", user);
+  }
+
+  rpcFreeCont(rpcRsp.pCont);
+  return rpcRsp.code;
+}
+
+static int32_t dndInitServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->t;
+  dndInitMsgFp(pMgmt);
+
+  int32_t numOfThreads = (int32_t)((pDnode->opt.numOfCores * pDnode->opt.numOfThreadsPerCore) / 2.0);
+  if (numOfThreads < 1) {
+    numOfThreads = 1;
+  }
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort = pDnode->opt.serverPort;
+  rpcInit.label = "DND-S";
+  rpcInit.numOfThreads = numOfThreads;
+  rpcInit.cfp = dndProcessRequest;
+  rpcInit.sessions = pDnode->opt.maxShellConns;
+  rpcInit.connType = TAOS_CONN_SERVER;
+  rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000;
+  rpcInit.afp = dndRetrieveUserAuthInfo;
+
+  pMgmt->serverRpc = rpcOpen(&rpcInit);
+  if (pMgmt->serverRpc == NULL) {
+    dError("failed to init rpc server");
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->t;
+  if (pMgmt->serverRpc) {
+    rpcClose(pMgmt->serverRpc);
+    pMgmt->serverRpc = NULL;
+  }
+}
+
+int32_t dndInitTrans(SDnode *pDnode) {
+  if (dndInitClient(pDnode) != 0) {
+    return -1;
+  }
+
+  if (dndInitServer(pDnode) != 0) {
+    return -1;
+  }
+
+  dInfo("dnode-transport is initialized");
+  return 0;
+}
+
+void dndCleanupTrans(SDnode *pDnode) {
+  dndCleanupServer(pDnode);
+  dndCleanupClient(pDnode);
+  dInfo("dnode-transport is cleaned up");
+}
+
+void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pMsg) {
+  STransMgmt *pMgmt = &pDnode->t;
+  rpcSendRequest(pMgmt->clientRpc, pEpSet, pMsg, NULL);
+}
+
+void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) {
+  SEpSet epSet = {0};
+  dndGetMnodeEpSet(pDnode, &epSet);
+  dndSendMsgToDnode(pDnode, &epSet, pMsg);
+}
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c
new file mode 100644
index 0000000000..d5e94106a7
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndVnodes.c
@@ -0,0 +1,1119 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndVnodes.h"
+#include "dndTransport.h"
+
+typedef struct {
+  int32_t    vgId;
+  int32_t    refCount;
+  int8_t     dropped;
+  int8_t     accessState;
+  SVnode    *pImpl;
+  taos_queue pWriteQ;
+  taos_queue pSyncQ;
+  taos_queue pApplyQ;
+  taos_queue pQueryQ;
+  taos_queue pFetchQ;
+} SVnodeObj;
+
+typedef struct {
+  int32_t    vnodeNum;
+  int32_t    opened;
+  int32_t    failed;
+  int32_t    threadIndex;
+  pthread_t *pThreadId;
+  SVnodeObj *pVnodes;
+  SDnode    *pDnode;
+} SVnodeThread;
+
+static int32_t dndInitVnodeReadWorker(SDnode *pDnode);
+static int32_t dndInitVnodeWriteWorker(SDnode *pDnode);
+static int32_t dndInitVnodeSyncWorker(SDnode *pDnode);
+static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode);
+static void    dndCleanupVnodeReadWorker(SDnode *pDnode);
+static void    dndCleanupVnodeWriteWorker(SDnode *pDnode);
+static void    dndCleanupVnodeSyncWorker(SDnode *pDnode);
+static void    dndCleanupVnodeMgmtWorker(SDnode *pDnode);
+static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void    dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void    dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void    dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void    dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void    dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode);
+
+static void    dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
+static void    dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
+static void    dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void    dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void    dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void    dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
+void           dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void           dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg);
+
+static SVnodeObj  *dndAcquireVnode(SDnode *pDnode, int32_t vgId);
+static void        dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t     dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl);
+static void        dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode);
+static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes);
+static int32_t     dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes);
+static int32_t     dndWriteVnodesToFile(SDnode *pDnode);
+
+static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg);
+static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndOpenVnodes(SDnode *pDnode);
+static void    dndCloseVnodes(SDnode *pDnode);
+
+static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+
+static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  SVnodeObj   *pVnode = NULL;
+  int32_t      refCount = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+  taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
+  if (pVnode == NULL) {
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+  } else {
+    refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+  }
+  taosRUnLockLatch(&pMgmt->latch);
+
+  dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+  return pVnode;
+}
+
+static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  int32_t      refCount = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+  if (pVnode != NULL) {
+    refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+  }
+  taosRUnLockLatch(&pMgmt->latch);
+
+  if (pVnode != NULL) {
+    dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
+  }
+}
+
+static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  SVnodeObj   *pVnode = calloc(1, sizeof(SVnodeObj));
+  if (pVnode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pVnode->vgId = vgId;
+  pVnode->refCount = 0;
+  pVnode->dropped = 0;
+  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
+  pVnode->pImpl = pImpl;
+
+  if (dndAllocVnodeQueryQueue(pDnode, pVnode) != 0) {
+    return -1;
+  }
+
+  if (dndAllocVnodeFetchQueue(pDnode, pVnode) != 0) {
+    return -1;
+  }
+
+  if (dndAllocVnodeWriteQueue(pDnode, pVnode) != 0) {
+    return -1;
+  }
+
+  if (dndAllocVnodeApplyQueue(pDnode, pVnode) != 0) {
+    return -1;
+  }
+
+  if (dndAllocVnodeSyncQueue(pDnode, pVnode) != 0) {
+    return -1;
+  }
+
+  taosWLockLatch(&pMgmt->latch);
+  int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+  taosWUnLockLatch(&pMgmt->latch);
+
+  if (code != 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return code;
+}
+
+static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  taosWLockLatch(&pMgmt->latch);
+  taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
+  taosWUnLockLatch(&pMgmt->latch);
+
+  dndReleaseVnode(pDnode, pVnode);
+  while (pVnode->refCount > 0) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+
+  dndFreeVnodeQueryQueue(pDnode, pVnode);
+  dndFreeVnodeFetchQueue(pDnode, pVnode);
+  dndFreeVnodeWriteQueue(pDnode, pVnode);
+  dndFreeVnodeApplyQueue(pDnode, pVnode);
+  dndFreeVnodeSyncQueue(pDnode, pVnode);
+}
+
+static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  taosRLockLatch(&pMgmt->latch);
+
+  int32_t     num = 0;
+  int32_t     size = taosHashGetSize(pMgmt->hash);
+  SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *));
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode) {
+      num++;
+      if (num < size) {
+        int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+        dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+        pVnodes[num] = (*ppVnode);
+      }
+    }
+    pIter = taosHashIterate(pMgmt->hash, pIter);
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+  *numOfVnodes = num;
+
+  return pVnodes;
+}
+
+static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) {
+  int32_t    code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR;
+  int32_t    len = 0;
+  int32_t    maxLen = 30000;
+  char      *content = calloc(1, maxLen + 1);
+  cJSON     *root = NULL;
+  FILE      *fp = NULL;
+  char       file[PATH_MAX + 20] = {0};
+  SVnodeObj *pVnodes = NULL;
+
+  snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
+
+  fp = fopen(file, "r");
+  if (!fp) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_VNODE_OVER;
+  }
+
+  len = (int32_t)fread(content, 1, maxLen, fp);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes");
+  if (!vnodes || vnodes->type != cJSON_Array) {
+    dError("failed to read %s since vnodes not found", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  int32_t vnodesNum = cJSON_GetArraySize(vnodes);
+  if (vnodesNum <= 0) {
+    dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum);
+    goto PRASE_VNODE_OVER;
+  }
+
+  pVnodes = calloc(vnodesNum, sizeof(SVnodeObj));
+  if (pVnodes == NULL) {
+    dError("failed to read %s since out of memory", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  for (int32_t i = 0; i < vnodesNum; ++i) {
+    cJSON     *vnode = cJSON_GetArrayItem(vnodes, i);
+    SVnodeObj *pVnode = &pVnodes[i];
+
+    cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
+    if (!vgId || vgId->type != cJSON_String) {
+      dError("failed to read %s since vgId not found", file);
+      goto PRASE_VNODE_OVER;
+    }
+    pVnode->vgId = atoi(vgId->valuestring);
+
+    cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
+    if (!dropped || dropped->type != cJSON_String) {
+      dError("failed to read %s since dropped not found", file);
+      goto PRASE_VNODE_OVER;
+    }
+    pVnode->dropped = atoi(vnode->valuestring);
+  }
+
+  code = 0;
+  dInfo("succcessed to read file %s", file);
+
+PRASE_VNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (fp != NULL) fclose(fp);
+
+  return code;
+}
+
+static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
+  char file[PATH_MAX + 20] = {0};
+  char realfile[PATH_MAX + 20] = {0};
+  snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
+  snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
+
+  FILE *fp = fopen(file, "w");
+  if (fp != NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t     len = 0;
+  int32_t     maxLen = 30000;
+  char       *content = calloc(1, maxLen + 1);
+  int32_t     numOfVnodes = 0;
+  SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"vnodes\": [{\n");
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    SVnodeObj *pVnode = pVnodes[i];
+    len += snprintf(content + len, maxLen - len, "    \"vgId\": \"%d\",\n", pVnode->vgId);
+    len += snprintf(content + len, maxLen - len, "    \"dropped\": \"%d\"\n", pVnode->dropped);
+    if (i < numOfVnodes - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  fwrite(content, 1, len, fp);
+  taosFsyncFile(fileno(fp));
+  fclose(fp);
+  free(content);
+  terrno = 0;
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    SVnodeObj *pVnode = pVnodes[i];
+    dndReleaseVnode(pDnode, pVnode);
+  }
+
+  if (pVnodes != NULL) {
+    free(pVnodes);
+  }
+
+  dInfo("successed to write %s", file);
+  return taosRenameFile(file, realfile);
+}
+
+static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) {
+  char path[PATH_MAX + 20] = {0};
+  snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId);
+  SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
+
+  if (pImpl == NULL) {
+    return -1;
+  }
+
+  int32_t code = dndCreateVnodeWrapper(pDnode, vgId, pImpl);
+  if (code != 0) {
+    vnodeDrop(pImpl);
+    terrno = code;
+    return code;
+  }
+
+  code = dndWriteVnodesToFile(pDnode);
+  if (code != 0) {
+    vnodeDrop(pImpl);
+    terrno = code;
+    return code;
+  }
+
+  return 0;
+}
+
+static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) {
+  pVnode->dropped = 1;
+  if (dndWriteVnodesToFile(pDnode) != 0) {
+    pVnode->dropped = 0;
+    return -1;
+  }
+
+  dndDropVnodeWrapper(pDnode, pVnode);
+  vnodeDrop(pVnode->pImpl);
+  dndWriteVnodesToFile(pDnode);
+  return 0;
+}
+
+static void *dnodeOpenVnodeFunc(void *param) {
+  SVnodeThread *pThread = param;
+  SDnode       *pDnode = pThread->pDnode;
+  SVnodesMgmt  *pMgmt = &pDnode->vmgmt;
+
+  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+  setThreadName("open-vnodes");
+
+  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+    SVnodeObj *pVnode = &pThread->pVnodes[v];
+
+    char stepDesc[TSDB_STEP_DESC_LEN] = {0};
+    snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId,
+             pMgmt->openVnodes, pMgmt->totalVnodes);
+    dndReportStartup(pDnode, "open-vnodes", stepDesc);
+
+    char path[PATH_MAX + 20] = {0};
+    snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId);
+    SVnode *pImpl = vnodeOpen(path, NULL);
+    if (pImpl == NULL) {
+      dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
+      pThread->failed++;
+    } else {
+      dndCreateVnodeWrapper(pDnode, pVnode->vgId, pImpl);
+      dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex);
+      pThread->opened++;
+    }
+
+    atomic_add_fetch_32(&pMgmt->openVnodes, 1);
+  }
+
+  dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+         pThread->failed);
+  return NULL;
+}
+
+static int32_t dndOpenVnodes(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  taosInitRWLatch(&pMgmt->latch);
+
+  pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pMgmt->hash == NULL) {
+    dError("failed to init vnode hash");
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  SVnodeObj *pVnodes = NULL;
+  int32_t    numOfVnodes = 0;
+  if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) {
+    dInfo("failed to get vnode list from disk since %s", terrstr());
+    return -1;
+  }
+
+  pMgmt->totalVnodes = numOfVnodes;
+
+  int32_t threadNum = tsNumOfCores;
+  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+
+  SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
+  for (int32_t t = 0; t < threadNum; ++t) {
+    threads[t].threadIndex = t;
+    threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj));
+  }
+
+  for (int32_t v = 0; v < numOfVnodes; ++v) {
+    int32_t       t = v % threadNum;
+    SVnodeThread *pThread = &threads[t];
+    pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v];
+  }
+
+  dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum == 0) continue;
+
+    pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread);
+    if (pThread->pThreadId == NULL) {
+      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+    }
+  }
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SVnodeThread *pThread = &threads[t];
+    taosDestoryThread(pThread->pThreadId);
+    pThread->pThreadId = NULL;
+    free(pThread->pVnodes);
+  }
+  free(threads);
+
+  if (pMgmt->openVnodes != pMgmt->totalVnodes) {
+    dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes);
+    return -1;
+  } else {
+    dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes);
+    return 0;
+  }
+}
+
+static void dndCloseVnodes(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+  int32_t     numOfVnodes = 0;
+  SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    dndDropVnodeWrapper(pDnode, pVnodes[i]);
+  }
+
+  if (pVnodes != NULL) {
+    free(pVnodes);
+  }
+
+  if (pMgmt->hash != NULL) {
+    taosHashCleanup(pMgmt->hash);
+    pMgmt->hash = NULL;
+  }
+
+  dInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  *vgId = htonl(pCreate->vgId);
+
+  tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
+  pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
+  pCfg->totalBlocks = htonl(pCreate->totalBlocks);
+  pCfg->daysPerFile = htonl(pCreate->daysPerFile);
+  pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
+  pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
+  pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
+  pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
+  pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
+  pCfg->precision = pCreate->precision;
+  pCfg->compression = pCreate->compression;
+  pCfg->cacheLastRow = pCreate->cacheLastRow;
+  pCfg->update = pCreate->update;
+  pCfg->quorum = pCreate->quorum;
+  pCfg->replica = pCreate->replica;
+  pCfg->walLevel = pCreate->walLevel;
+  pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
+
+  for (int32_t i = 0; i < pCfg->replica; ++i) {
+    pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
+    tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
+  }
+
+  return 0;
+}
+
+static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
+  pDrop->vgId = htonl(pDrop->vgId);
+  return pDrop;
+}
+
+static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
+  SAuthVnodeMsg *pAuth = rpcMsg->pCont;
+  pAuth->vgId = htonl(pAuth->vgId);
+  return pAuth;
+}
+
+static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SVnodeCfg vnodeCfg = {0};
+  int32_t   vgId = 0;
+
+  dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
+  dDebug("vgId:%d, create vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode != NULL) {
+    dDebug("vgId:%d, already exist, return success", vgId);
+    dndReleaseVnode(pDnode, pVnode);
+    return 0;
+  }
+
+  if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) {
+    dError("vgId:%d, failed to create vnode since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  return 0;
+}
+
+static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SVnodeCfg vnodeCfg = {0};
+  int32_t   vgId = 0;
+
+  dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
+  dDebug("vgId:%d, alter vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) {
+    dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
+    dndReleaseVnode(pDnode, pVnode);
+    return terrno;
+  }
+
+  dndReleaseVnode(pDnode, pVnode);
+  return 0;
+}
+
+static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg);
+
+  int32_t vgId = pDrop->vgId;
+  dDebug("vgId:%d, drop vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  if (dndDropVnode(pDnode, pVnode) != 0) {
+    dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr());
+    dndReleaseVnode(pDnode, pVnode);
+    return terrno;
+  }
+
+  return 0;
+}
+
+static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
+
+  int32_t code = 0;
+  int32_t vgId = pAuth->vgId;
+  dDebug("vgId:%d, auth vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to auth since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  pVnode->accessState = pAuth->accessState;
+  dndReleaseVnode(pDnode, pVnode);
+  return 0;
+}
+
+static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
+
+  int32_t vgId = pAuth->vgId;
+  dDebug("vgId:%d, auth vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to auth since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  if (vnodeSync(pVnode->pImpl) != 0) {
+    dError("vgId:%d, failed to auth vnode since %s", vgId, terrstr());
+    dndReleaseVnode(pDnode, pVnode);
+    return terrno;
+  }
+
+  dndReleaseVnode(pDnode, pVnode);
+  return 0;
+}
+
+static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+  SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg);
+
+  int32_t vgId = pCompact->vgId;
+  dDebug("vgId:%d, compact vnode req is received", vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to compact since %s", vgId, terrstr());
+    return terrno;
+  }
+
+  if (vnodeCompact(pVnode->pImpl) != 0) {
+    dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr());
+    dndReleaseVnode(pDnode, pVnode);
+    return terrno;
+  }
+
+  dndReleaseVnode(pDnode, pVnode);
+  return 0;
+}
+
+static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+  int32_t code = 0;
+
+  switch (pMsg->msgType) {
+    case TSDB_MSG_TYPE_CREATE_VNODE_IN:
+      code = dndProcessCreateVnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_ALTER_VNODE_IN:
+      code = dndProcessAlterVnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_DROP_VNODE_IN:
+      code = dndProcessDropVnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_AUTH_VNODE_IN:
+      code = dndProcessAuthVnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_SYNC_VNODE_IN:
+      code = dndProcessSyncVnodeReq(pDnode, pMsg);
+      break;
+    case TSDB_MSG_TYPE_COMPACT_VNODE_IN:
+      code = dndProcessCompactVnodeReq(pDnode, pMsg);
+      break;
+    default:
+      code = TSDB_CODE_MSG_NOT_PROCESSED;
+      break;
+  }
+
+  if (code != 0) {
+    SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
+  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY);
+}
+
+static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
+  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH);
+}
+
+static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+  SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
+  SRpcMsg   *pRpcMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    taosGetQitem(qall, (void **)&pRpcMsg);
+    vnodeAppendMsg(pMsg, pRpcMsg);
+    taosFreeQitem(pRpcMsg);
+  }
+
+  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE);
+}
+
+static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+  SVnodeMsg *pMsg = NULL;
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    taosGetQitem(qall, (void **)&pMsg);
+    vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY);
+  }
+}
+
+static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+  SVnodeMsg *pMsg = NULL;
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    taosGetQitem(qall, (void **)&pMsg);
+    vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC);
+  }
+}
+
+static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
+  int32_t code = 0;
+
+  if (pQueue == NULL) {
+    code = TSDB_CODE_MSG_NOT_PROCESSED;
+  } else {
+    SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
+    if (pMsg == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
+      *pMsg = *pRpcMsg;
+      if (taosWriteQitem(pQueue, pMsg) != 0) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+      }
+    }
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pRpcMsg->pCont);
+  }
+}
+
+static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
+  int32_t code = 0;
+
+  if (pQueue == NULL) {
+    code = TSDB_CODE_MSG_NOT_PROCESSED;
+  } else {
+    SVnodeMsg *pMsg = vnodeInitMsg(1);
+    if (pMsg == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
+      if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) {
+        code = terrno;
+      } else {
+        if (taosWriteQitem(pQueue, pMsg) != 0) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+        }
+      }
+    }
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pRpcMsg->pCont);
+  }
+}
+
+static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) {
+  SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
+  pHead->vgId = htonl(pHead->vgId);
+
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId);
+  if (pVnode == NULL) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+    rpcSendResponse(&rsp);
+    rpcFreeCont(pMsg->pCont);
+  }
+
+  return pVnode;
+}
+
+void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  dndWriteRpcMsgToVnodeQueue(pMgmt->pMgmtQ, pMsg);
+}
+
+void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+  if (pVnode != NULL) {
+    dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg);
+    dndReleaseVnode(pDnode, pVnode);
+  }
+}
+
+void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+  if (pVnode != NULL) {
+    dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
+    dndReleaseVnode(pDnode, pVnode);
+  }
+}
+
+void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+  if (pVnode != NULL) {
+    dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
+    dndReleaseVnode(pDnode, pVnode);
+  }
+}
+
+void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+  if (pVnode != NULL) {
+    dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
+    dndReleaseVnode(pDnode, pVnode);
+  }
+}
+
+static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) {
+  SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+  if (pVnode == NULL) {
+    return -1;
+  }
+
+  int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg);
+  dndReleaseVnode(pDnode, pVnode);
+  return code;
+}
+
+static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  SWorkerPool *pPool = &pMgmt->mgmtPool;
+  pPool->name = "vnode-mgmt";
+  pPool->min = 1;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->pMgmtQ = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)dndProcessVnodeMgmtQueue);
+  if (pMgmt->pMgmtQ == NULL) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupVnodeMgmtWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ);
+  tWorkerCleanup(&pMgmt->mgmtPool);
+  pMgmt->pMgmtQ = NULL;
+}
+
+static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue);
+  if (pVnode->pQueryQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
+  pVnode->pQueryQ = NULL;
+}
+
+static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue);
+  if (pVnode->pFetchQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
+  pVnode->pFetchQ = NULL;
+}
+
+static int32_t dndInitVnodeReadWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+  int32_t maxFetchThreads = 4;
+  float   threadsForQuery = MAX(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores, 1);
+
+  SWorkerPool *pPool = &pMgmt->queryPool;
+  pPool->name = "vnode-query";
+  pPool->min = (int32_t)threadsForQuery;
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  pPool = &pMgmt->fetchPool;
+  pPool->name = "vnode-fetch";
+  pPool->min = MIN(maxFetchThreads, pDnode->opt.numOfCores);
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) {
+    TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  return 0;
+}
+
+static void dndCleanupVnodeReadWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tWorkerCleanup(&pMgmt->fetchPool);
+  tWorkerCleanup(&pMgmt->queryPool);
+}
+
+static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue);
+  if (pVnode->pWriteQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
+  pVnode->pWriteQ = NULL;
+}
+
+static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue);
+  if (pVnode->pApplyQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
+  pVnode->pApplyQ = NULL;
+}
+
+static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
+  SVnodesMgmt  *pMgmt = &pDnode->vmgmt;
+  SMWorkerPool *pPool = &pMgmt->writePool;
+  pPool->name = "vnode-write";
+  pPool->max = tsNumOfCores;
+  if (tMWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupVnodeWriteWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tMWorkerCleanup(&pMgmt->writePool);
+}
+
+static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue);
+  if (pVnode->pSyncQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ);
+  pVnode->pSyncQ = NULL;
+}
+
+static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) {
+  int32_t maxThreads = tsNumOfCores / 2;
+  if (maxThreads < 1) maxThreads = 1;
+
+  SVnodesMgmt  *pMgmt = &pDnode->vmgmt;
+  SMWorkerPool *pPool = &pMgmt->writePool;
+  pPool->name = "vnode-sync";
+  pPool->max = maxThreads;
+  if (tMWorkerInit(pPool) != 0) {
+    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static void dndCleanupVnodeSyncWorker(SDnode *pDnode) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+  tMWorkerCleanup(&pMgmt->syncPool);
+}
+
+int32_t dndInitVnodes(SDnode *pDnode) {
+  dInfo("dnode-vnodes start to init");
+
+  if (dndInitVnodeReadWorker(pDnode) != 0) {
+    dError("failed to init vnodes read worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitVnodeWriteWorker(pDnode) != 0) {
+    dError("failed to init vnodes write worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitVnodeSyncWorker(pDnode) != 0) {
+    dError("failed to init vnodes sync worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitVnodeMgmtWorker(pDnode) != 0) {
+    dError("failed to init vnodes mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndOpenVnodes(pDnode) != 0) {
+    dError("failed to open vnodes since %s", terrstr());
+    return -1;
+  }
+
+  dInfo("dnode-vnodes is initialized");
+  return 0;
+}
+
+void dndCleanupVnodes(SDnode *pDnode) {
+  dInfo("dnode-vnodes start to clean up");
+  dndCloseVnodes(pDnode);
+  dndCleanupVnodeReadWorker(pDnode);
+  dndCleanupVnodeWriteWorker(pDnode);
+  dndCleanupVnodeSyncWorker(pDnode);
+  dndCleanupVnodeMgmtWorker(pDnode);
+  dInfo("dnode-vnodes is cleaned up");
+}
+
+void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) {
+  SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+  taosRLockLatch(&pMgmt->latch);
+  pLoads->num = taosHashGetSize(pMgmt->hash);
+
+  int32_t v = 0;
+  void   *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    if (ppVnode == NULL || *ppVnode == NULL) continue;
+
+    SVnodeObj  *pVnode = *ppVnode;
+    SVnodeLoad *pLoad = &pLoads->data[v++];
+
+    vnodeGetLoad(pVnode->pImpl, pLoad);
+    pLoad->vgId = htonl(pLoad->vgId);
+    pLoad->totalStorage = htobe64(pLoad->totalStorage);
+    pLoad->compStorage = htobe64(pLoad->compStorage);
+    pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
+    pLoad->tablesNum = htobe64(pLoad->tablesNum);
+
+    pIter = taosHashIterate(pMgmt->hash, pIter);
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+}
diff --git a/source/dnode/mgmt/inc/dnodeInt.h b/source/dnode/mgmt/inc/dnodeInt.h
deleted file mode 100644
index 906455dce4..0000000000
--- a/source/dnode/mgmt/inc/dnodeInt.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DNODE_INT_H_
-#define _TD_DNODE_INT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "os.h"
-#include "taosmsg.h"
-#include "tglobal.h"
-#include "tlog.h"
-#include "trpc.h"
-
-extern int32_t dDebugFlag;
-
-#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
-#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}
-#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }}
-#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("DND ", 255, __VA_ARGS__); }}
-#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
-#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
-
-typedef enum { DN_RUN_STAT_INIT, DN_RUN_STAT_RUNNING, DN_RUN_STAT_STOPPED } EDnStat;
-typedef void (*MsgFp)(SRpcMsg *pMsg, SEpSet *pEpSet);
-
-int32_t dnodeInit();
-void    dnodeCleanup();
-
-EDnStat dnodeGetRunStat();
-void    dnodeSetRunStat();
-
-void dnodeReportStartup(char *name, char *desc);
-void dnodeReportStartupFinished(char *name, char *desc);
-void dnodeGetStartup(SStartupMsg *);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DNODE_INT_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/src/dnode.c b/source/dnode/mgmt/src/dnode.c
deleted file mode 100644
index 1fbeb1e732..0000000000
--- a/source/dnode/mgmt/src/dnode.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeInt.h"
-
-static bool stop = false;
-static void sigintHandler(int32_t signum, void *info, void *ctx) { stop = true; }
-static void setSignalHandler() {
-  taosSetSignal(SIGTERM, sigintHandler);
-  taosSetSignal(SIGHUP, sigintHandler);
-  taosSetSignal(SIGINT, sigintHandler);
-  taosSetSignal(SIGABRT, sigintHandler);
-  taosSetSignal(SIGBREAK, sigintHandler);
-}
-
-int main(int argc, char const *argv[]) {
-  setSignalHandler();
-
-  int32_t code = dnodeInit();
-  if (code != 0) {
-    dInfo("Failed to start TDengine, please check the log at:%s", tsLogDir);
-    exit(EXIT_FAILURE);
-  }
-
-  while (!stop) {
-    taosMsleep(100);
-  }
-
-  dInfo("TDengine is shut down!");
-  dnodeCleanup();
-
-  return 0;
-}
diff --git a/source/dnode/mgmt/src/dnodeDnode.c b/source/dnode/mgmt/src/dnodeDnode.c
deleted file mode 100644
index 7843075b30..0000000000
--- a/source/dnode/mgmt/src/dnodeDnode.c
+++ /dev/null
@@ -1,560 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeDnode.h"
-#include "dnodeTransport.h"
-#include "dnodeVnodes.h"
-#include "cJSON.h"
-#include "thash.h"
-#include "tthread.h"
-#include "ttime.h"
-
-static struct {
-  int32_t         dnodeId;
-  int64_t         clusterId;
-  SDnodeEps      *dnodeEps;
-  SHashObj       *dnodeHash;
-  SEpSet          mnodeEpSetForShell;
-  SEpSet          mnodeEpSetForPeer;
-  char            file[PATH_MAX + 20];
-  uint32_t        rebootTime;
-  int8_t          dropped;
-  int8_t          threadStop;
-  pthread_t      *threadId;
-  pthread_mutex_t mutex;
-} tsDnode = {0};
-
-int32_t dnodeGetDnodeId() {
-  int32_t dnodeId = 0;
-  pthread_mutex_lock(&tsDnode.mutex);
-  dnodeId = tsDnode.dnodeId;
-  pthread_mutex_unlock(&tsDnode.mutex);
-  return dnodeId;
-}
-
-int64_t dnodeGetClusterId() {
-  int64_t clusterId = 0;
-  pthread_mutex_lock(&tsDnode.mutex);
-  clusterId = tsDnode.clusterId;
-  pthread_mutex_unlock(&tsDnode.mutex);
-  return clusterId;
-}
-
-void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) {
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  SDnodeEp *pEp = taosHashGet(tsDnode.dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pEp != NULL) {
-    if (port) *port = pEp->dnodePort;
-    if (fqdn) tstrncpy(fqdn, pEp->dnodeFqdn, TSDB_FQDN_LEN);
-    if (ep) snprintf(ep, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort);
-  }
-
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-void dnodeGetMnodeEpSetForPeer(SEpSet *pEpSet) {
-  pthread_mutex_lock(&tsDnode.mutex);
-  *pEpSet = tsDnode.mnodeEpSetForPeer;
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-void dnodeGetMnodeEpSetForShell(SEpSet *pEpSet) {
-  pthread_mutex_lock(&tsDnode.mutex);
-  *pEpSet = tsDnode.mnodeEpSetForShell;
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-void dnodeSendRedirectMsg(SRpcMsg *pMsg, bool forShell) {
-  int32_t msgType = pMsg->msgType;
-
-  SEpSet epSet = {0};
-  if (forShell) {
-    dnodeGetMnodeEpSetForShell(&epSet);
-  } else {
-    dnodeGetMnodeEpSetForPeer(&epSet);
-  }
-
-  dDebug("RPC %p, msg:%s is redirected, num:%d use:%d", pMsg->handle, taosMsg[msgType], epSet.numOfEps, epSet.inUse);
-
-  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]);
-    if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) {
-      if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) ||
-          (epSet.port[i] == tsServerPort && forShell)) {
-        epSet.inUse = (i + 1) % epSet.numOfEps;
-        dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse);
-      }
-    }
-
-    epSet.port[i] = htons(epSet.port[i]);
-  }
-
-  rpcSendRedirectRsp(pMsg->handle, &epSet);
-}
-
-static void dnodeUpdateMnodeEpSet(SEpSet *pEpSet) {
-  if (pEpSet == NULL || pEpSet->numOfEps <= 0) {
-    dError("mnode is changed, but content is invalid, discard it");
-    return;
-  } else {
-    dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
-  }
-
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  tsDnode.mnodeEpSetForPeer = *pEpSet;
-  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
-    pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
-    dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
-  }
-  tsDnode.mnodeEpSetForShell = *pEpSet;
-
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-static void dnodePrintDnodes() {
-  dDebug("print dnode endpoint list, num:%d", tsDnode.dnodeEps->dnodeNum);
-  for (int32_t i = 0; i < tsDnode.dnodeEps->dnodeNum; i++) {
-    SDnodeEp *ep = &tsDnode.dnodeEps->dnodeEps[i];
-    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort, ep->isMnode);
-  }
-}
-
-static void dnodeResetDnodes(SDnodeEps *pEps) {
-  assert(pEps != NULL);
-  int32_t size = sizeof(SDnodeEps) + pEps->dnodeNum * sizeof(SDnodeEp);
-
-  if (pEps->dnodeNum > tsDnode.dnodeEps->dnodeNum) {
-    SDnodeEps *tmp = calloc(1, size);
-    if (tmp == NULL) return;
-
-    tfree(tsDnode.dnodeEps);
-    tsDnode.dnodeEps = tmp;
-  }
-
-  if (tsDnode.dnodeEps != pEps) {
-    memcpy(tsDnode.dnodeEps, pEps, size);
-  }
-
-  tsDnode.mnodeEpSetForPeer.inUse = 0;
-  tsDnode.mnodeEpSetForShell.inUse = 0;
-
-  int32_t mIndex = 0;
-  for (int32_t i = 0; i < tsDnode.dnodeEps->dnodeNum; i++) {
-    SDnodeEp *ep = &tsDnode.dnodeEps->dnodeEps[i];
-    if (!ep->isMnode) continue;
-    if (mIndex >= TSDB_MAX_REPLICA) continue;
-    strcpy(tsDnode.mnodeEpSetForShell.fqdn[mIndex], ep->dnodeFqdn);
-    strcpy(tsDnode.mnodeEpSetForPeer.fqdn[mIndex], ep->dnodeFqdn);
-    tsDnode.mnodeEpSetForShell.port[mIndex] = ep->dnodePort;
-    tsDnode.mnodeEpSetForShell.port[mIndex] = ep->dnodePort + tsDnodeDnodePort;
-    mIndex++;
-  }
-
-  for (int32_t i = 0; i < tsDnode.dnodeEps->dnodeNum; ++i) {
-    SDnodeEp *ep = &tsDnode.dnodeEps->dnodeEps[i];
-    taosHashPut(tsDnode.dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
-  }
-
-  dnodePrintDnodes();
-}
-
-static bool dnodeIsEpChanged(int32_t dnodeId, char *epStr) {
-  bool changed = false;
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  SDnodeEp *pEp = taosHashGet(tsDnode.dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pEp != NULL) {
-    char epSaved[TSDB_EP_LEN + 1];
-    snprintf(epSaved, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort);
-    changed = strcmp(epStr, epSaved) != 0;
-  }
-
-  pthread_mutex_unlock(&tsDnode.mutex);
-  return changed;
-}
-
-static int32_t dnodeReadDnodes() {
-  int32_t len = 0;
-  int32_t maxLen = 30000;
-  char   *content = calloc(1, maxLen + 1);
-  cJSON  *root = NULL;
-  FILE   *fp = NULL;
-
-  fp = fopen(tsDnode.file, "r");
-  if (!fp) {
-    dDebug("file %s not exist", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  len = (int32_t)fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_String) {
-    dError("failed to read %s since dnodeId not found", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-  tsDnode.dnodeId = atoi(dnodeId->valuestring);
-
-  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read %s since clusterId not found", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-  tsDnode.clusterId = atoll(clusterId->valuestring);
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_String) {
-    dError("failed to read %s since dropped not found", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-  tsDnode.dropped = atoi(dropped->valuestring);
-
-  cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
-  if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
-    dError("failed to read %s since dnodeInfos not found", tsDnode.file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
-  if (dnodeInfosSize <= 0) {
-    dError("failed to read %s since dnodeInfos size:%d invalid", tsDnode.file, dnodeInfosSize);
-    goto PRASE_DNODE_OVER;
-  }
-
-  tsDnode.dnodeEps = calloc(1, dnodeInfosSize * sizeof(SDnodeEp) + sizeof(SDnodeEps));
-  if (tsDnode.dnodeEps == NULL) {
-    dError("failed to calloc dnodeEpList since %s", strerror(errno));
-    goto PRASE_DNODE_OVER;
-  }
-  tsDnode.dnodeEps->dnodeNum = dnodeInfosSize;
-
-  for (int32_t i = 0; i < dnodeInfosSize; ++i) {
-    cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
-    if (dnodeInfo == NULL) break;
-
-    SDnodeEp *pEp = &tsDnode.dnodeEps->dnodeEps[i];
-
-    cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
-    if (!dnodeId || dnodeId->type != cJSON_String) {
-      dError("failed to read %s, dnodeId not found", tsDnode.file);
-      goto PRASE_DNODE_OVER;
-    }
-    pEp->dnodeId = atoi(dnodeId->valuestring);
-
-    cJSON *isMnode = cJSON_GetObjectItem(dnodeInfo, "isMnode");
-    if (!isMnode || isMnode->type != cJSON_String) {
-      dError("failed to read %s, isMnode not found", tsDnode.file);
-      goto PRASE_DNODE_OVER;
-    }
-    pEp->isMnode = atoi(isMnode->valuestring);
-
-    cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
-    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
-      dError("failed to read %s, dnodeFqdn not found", tsDnode.file);
-      goto PRASE_DNODE_OVER;
-    }
-    tstrncpy(pEp->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
-
-    cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
-    if (!dnodePort || dnodePort->type != cJSON_String) {
-      dError("failed to read %s, dnodePort not found", tsDnode.file);
-      goto PRASE_DNODE_OVER;
-    }
-    pEp->dnodePort = atoi(dnodePort->valuestring);
-  }
-
-  dInfo("succcessed to read file %s", tsDnode.file);
-  dnodePrintDnodes();
-
-PRASE_DNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
-
-  if (dnodeIsEpChanged(tsDnode.dnodeId, tsLocalEp)) {
-    dError("localEp %s different with %s and need reconfigured", tsLocalEp, tsDnode.file);
-    return -1;
-  }
-
-  if (tsDnode.dnodeEps == NULL) {
-    tsDnode.dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
-    tsDnode.dnodeEps->dnodeNum = 1;
-    tsDnode.dnodeEps->dnodeEps[0].dnodePort = tsServerPort;
-    tstrncpy(tsDnode.dnodeEps->dnodeEps[0].dnodeFqdn, tsLocalFqdn, TSDB_FQDN_LEN);
-  }
-
-  dnodeResetDnodes(tsDnode.dnodeEps);
-
-  terrno = 0;
-  return 0;
-}
-
-static int32_t dnodeWriteDnodes() {
-  FILE *fp = fopen(tsDnode.file, "w");
-  if (!fp) {
-    dError("failed to write %s since %s", tsDnode.file, strerror(errno));
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 30000;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": \"%d\",\n", tsDnode.dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", tsDnode.clusterId);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": \"%d\",\n", tsDnode.dropped);
-  len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
-  for (int32_t i = 0; i < tsDnode.dnodeEps->dnodeNum; ++i) {
-    SDnodeEp *ep = &tsDnode.dnodeEps->dnodeEps[i];
-    len += snprintf(content + len, maxLen - len, "    \"dnodeId\": \"%d\",\n", ep->dnodeId);
-    len += snprintf(content + len, maxLen - len, "    \"isMnode\": \"%d\",\n", ep->isMnode);
-    len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
-    len += snprintf(content + len, maxLen - len, "    \"dnodePort\": \"%u\"\n", ep->dnodePort);
-    if (i < tsDnode.dnodeEps->dnodeNum - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  fwrite(content, 1, len, fp);
-  taosFsyncFile(fileno(fp));
-  fclose(fp);
-  free(content);
-  terrno = 0;
-
-  dInfo("successed to write %s", tsDnode.file);
-  return 0;
-}
-
-static void dnodeSendStatusMsg() {
-  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-
-  SStatusMsg *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    dError("failed to malloc status message");
-    return;
-  }
-
-  pStatus->sversion = htonl(tsVersion);
-  pStatus->dnodeId = htonl(dnodeGetDnodeId());
-  pStatus->clusterId = htobe64(dnodeGetClusterId());
-  pStatus->rebootTime = htonl(tsDnode.rebootTime);
-  pStatus->numOfCores = htonl(tsNumOfCores);
-  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
-
-  pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
-  pStatus->clusterCfg.checkTime = 0;
-  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, TSDB_TIMEZONE_LEN);
-  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
-  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-
-  dnodeGetVnodeLoads(&pStatus->vnodeLoads);
-  contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad);
-
-  SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS};
-  dnodeSendMsgToMnode(&rpcMsg);
-}
-
-static void dnodeUpdateCfg(SDnodeCfg *pCfg) {
-  if (tsDnode.dnodeId == 0) return;
-  if (tsDnode.dropped) return;
-
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  tsDnode.dnodeId = pCfg->dnodeId;
-  tsDnode.clusterId = pCfg->clusterId;
-  tsDnode.dropped = pCfg->dropped;
-  dInfo("dnodeId is set to %d, clusterId is set to %" PRId64, pCfg->dnodeId, pCfg->clusterId);
-
-  dnodeWriteDnodes();
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-static void dnodeUpdateDnodeEps(SDnodeEps *pEps) {
-  if (pEps == NULL || pEps->dnodeNum <= 0) return;
-
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  if (pEps->dnodeNum != tsDnode.dnodeEps->dnodeNum) {
-    dnodeResetDnodes(pEps);
-    dnodeWriteDnodes();
-  } else {
-    int32_t size = pEps->dnodeNum * sizeof(SDnodeEp) + sizeof(SDnodeEps);
-    if (memcmp(tsDnode.dnodeEps, pEps, size) != 0) {
-      dnodeResetDnodes(pEps);
-      dnodeWriteDnodes();
-    }
-  }
-
-  pthread_mutex_unlock(&tsDnode.mutex);
-}
-
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  if (pMsg->code != TSDB_CODE_SUCCESS) return;
-
-  SStatusRsp *pStatusRsp = pMsg->pCont;
-
-  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  pCfg->clusterId = htobe64(pCfg->clusterId);
-  dnodeUpdateCfg(pCfg);
-
-  if (pCfg->dropped) return;
-
-  SDnodeEps *pEps = &pStatusRsp->dnodeEps;
-  pEps->dnodeNum = htonl(pEps->dnodeNum);
-  for (int32_t i = 0; i < pEps->dnodeNum; ++i) {
-    pEps->dnodeEps[i].dnodeId = htonl(pEps->dnodeEps[i].dnodeId);
-    pEps->dnodeEps[i].dnodePort = htons(pEps->dnodeEps[i].dnodePort);
-  }
-
-  dnodeUpdateDnodeEps(pEps);
-}
-
-static void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) {
-  SCfgDnodeMsg *pCfg = pMsg->pCont;
-
-  int32_t code = taosCfgDynamicOptions(pCfg->config);
-  SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code};
-  rpcSendResponse(&rspMsg);
-  rpcFreeCont(pMsg->pCont);
-}
-
-static void dnodeProcessStartupReq(SRpcMsg *pMsg) {
-  dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont);
-
-  SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg));
-  dnodeGetStartup(pStartup);
-
-  dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-
-  SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)};
-  rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pMsg->pCont);
-}
-
-static void *dnodeThreadRoutine(void *param) {
-  int32_t ms = tsStatusInterval * 1000;
-
-  while (!tsDnode.threadStop) {
-    if (dnodeGetRunStat() != DN_RUN_STAT_RUNNING) {
-      continue;
-    } else {
-      dnodeSendStatusMsg();
-    }
-    taosMsleep(ms);
-  }
-}
-
-int32_t dnodeInitDnode() {
-  tsDnode.dnodeId = 0;
-  tsDnode.clusterId = 0;
-  tsDnode.dnodeEps = NULL;
-  snprintf(tsDnode.file, sizeof(tsDnode.file), "%s/dnode.json", tsDnodeDir);
-  tsDnode.rebootTime = taosGetTimestampSec();
-  tsDnode.dropped = 0;
-  pthread_mutex_init(&tsDnode.mutex, NULL);
-  tsDnode.threadStop = false;
-
-  tsDnode.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (tsDnode.dnodeHash == NULL) {
-    dError("failed to init dnode hash");
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-
-  tsDnode.threadId = taosCreateThread(dnodeThreadRoutine, NULL);
-  if (tsDnode.threadId == NULL) {
-    dError("failed to init dnode thread");
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-
-  int32_t code = dnodeReadDnodes();
-  if (code != 0) {
-    dError("failed to read file:%s since %s", tsDnode.file, tstrerror(code));
-    return code;
-  }
-
-  dInfo("dnode-dnode is initialized");
-  return 0;
-}
-
-void dnodeCleanupDnode() {
-  if (tsDnode.threadId != NULL) {
-    tsDnode.threadStop = true;
-    taosDestoryThread(tsDnode.threadId);
-    tsDnode.threadId = NULL;
-  }
-
-  pthread_mutex_lock(&tsDnode.mutex);
-
-  if (tsDnode.dnodeEps != NULL) {
-    free(tsDnode.dnodeEps);
-    tsDnode.dnodeEps = NULL;
-  }
-
-  if (tsDnode.dnodeHash) {
-    taosHashCleanup(tsDnode.dnodeHash);
-    tsDnode.dnodeHash = NULL;
-  }
-
-  pthread_mutex_unlock(&tsDnode.mutex);
-  pthread_mutex_destroy(&tsDnode.mutex);
-
-  dInfo("dnode-dnode is cleaned up");
-}
-
-void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  int32_t msgType = pMsg->msgType;
-
-  if (msgType == TSDB_MSG_TYPE_STATUS_RSP && pEpSet) {
-    dnodeUpdateMnodeEpSet(pEpSet);
-  }
-
-  switch (msgType) {
-    case TSDB_MSG_TYPE_NETWORK_TEST:
-      dnodeProcessStartupReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_CONFIG_DNODE_IN:
-      dnodeProcessConfigDnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_STATUS_RSP:
-      dnodeProcessStatusRsp(pMsg);
-      break;
-    default:
-      dError("RPC %p, %s not processed", pMsg->handle, taosMsg[msgType]);
-      SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_MSG_NOT_PROCESSED};
-      rpcSendResponse(&rspMsg);
-      rpcFreeCont(pMsg->pCont);
-  }
-}
diff --git a/source/dnode/mgmt/src/dnodeInt.c b/source/dnode/mgmt/src/dnodeInt.c
deleted file mode 100644
index 2674e107fd..0000000000
--- a/source/dnode/mgmt/src/dnodeInt.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeDnode.h"
-#include "dnodeMnode.h"
-#include "dnodeTransport.h"
-#include "dnodeVnodes.h"
-#include "sync.h"
-#include "tcache.h"
-#include "tconfig.h"
-#include "tnote.h"
-#include "tstep.h"
-#include "wal.h"
-
-static struct {
-  SStartupMsg startup;
-  EDnStat     runStat;
-  SSteps     *steps;
-} tsInt;
-
-EDnStat dnodeGetRunStat() { return tsInt.runStat; }
-
-void dnodeSetRunStat(EDnStat stat) { tsInt.runStat = stat; }
-
-void dnodeReportStartup(char *name, char *desc) {
-  SStartupMsg *pStartup = &tsInt.startup;
-  tstrncpy(pStartup->name, name, strlen(pStartup->name));
-  tstrncpy(pStartup->desc, desc, strlen(pStartup->desc));
-  pStartup->finished = 0;
-}
-
-void dnodeReportStartupFinished(char *name, char *desc) {
-  SStartupMsg *pStartup = &tsInt.startup;
-  tstrncpy(pStartup->name, name, strlen(pStartup->name));
-  tstrncpy(pStartup->desc, desc, strlen(pStartup->desc));
-  pStartup->finished = 1;
-}
-
-void dnodeGetStartup(SStartupMsg *pStartup) { memcpy(pStartup, &tsInt.startup, sizeof(SStartupMsg)); }
-
-static int32_t dnodeCheckRunning(char *dir) {
-  char filepath[256] = {0};
-  snprintf(filepath, sizeof(filepath), "%s/.running", dir);
-
-  FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
-  if (fd < 0) {
-    dError("failed to open lock file:%s since %s, quit", filepath, strerror(errno));
-    return -1;
-  }
-
-  int32_t ret = taosLockFile(fd);
-  if (ret != 0) {
-    dError("failed to lock file:%s since %s, quit", filepath, strerror(errno));
-    taosCloseFile(fd);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t dnodeInitDir() {
-  sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
-  sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
-  sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
-
-  if (!taosMkDir(tsDnodeDir)) {
-    dError("failed to create dir:%s since %s", tsDnodeDir, strerror(errno));
-    return -1;
-  }
-
-  if (!taosMkDir(tsMnodeDir)) {
-    dError("failed to create dir:%s since %s", tsMnodeDir, strerror(errno));
-    return -1;
-  }
-
-  if (!taosMkDir(tsVnodeDir)) {
-    dError("failed to create dir:%s since %s", tsVnodeDir, strerror(errno));
-    return -1;
-  }
-
-  if (dnodeCheckRunning(tsDnodeDir) != 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t dnodeInitMain() {
-  tsInt.runStat = DN_RUN_STAT_STOPPED;
-  tscEmbedded = 1;
-  taosIgnSIGPIPE();
-  taosBlockSIGPIPE();
-  taosResolveCRC();
-  taosInitGlobalCfg();
-  taosReadGlobalLogCfg();
-  taosSetCoreDump(tsEnableCoreFile);
-
-  if (!taosMkDir(tsLogDir)) {
-    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
-    return -1;
-  }
-
-  char temp[TSDB_FILENAME_LEN];
-  sprintf(temp, "%s/taosdlog", tsLogDir);
-  if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) {
-    printf("failed to init log file\n");
-  }
-
-  if (!taosReadGlobalCfg()) {
-    taosPrintGlobalCfg();
-    dError("TDengine read global config failed");
-    return -1;
-  }
-
-  dInfo("start to initialize TDengine");
-
-  taosInitNotes();
-
-  if (taosCheckGlobalCfg() != 0) {
-    return -1;
-  }
-
-  dnodeInitDir();
-
-  return 0;
-}
-
-static void dnodeCleanupMain() {
-  taos_cleanup();
-  taosCloseLog();
-  taosStopCacheRefreshWorker();
-}
-
-int32_t dnodeInit() {
-  SSteps *steps = taosStepInit(10, dnodeReportStartup);
-  if (steps == NULL) return -1;
-
-  taosStepAdd(steps, "dnode-main", dnodeInitMain, dnodeCleanupMain);
-  taosStepAdd(steps, "dnode-rpc", rpcInit, rpcCleanup);
-  taosStepAdd(steps, "dnode-tfs", NULL, NULL);
-  taosStepAdd(steps, "dnode-wal", walInit, walCleanUp);
-  //taosStepAdd(steps, "dnode-sync", syncInit, syncCleanUp);
-  taosStepAdd(steps, "dnode-dnode", dnodeInitDnode, dnodeCleanupDnode);
-  taosStepAdd(steps, "dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes);
-  taosStepAdd(steps, "dnode-mnode", dnodeInitMnode, dnodeCleanupMnode);
-  taosStepAdd(steps, "dnode-trans", dnodeInitTrans, dnodeCleanupTrans);
-
-  tsInt.steps = steps;
-  taosStepExec(tsInt.steps);
-
-  dnodeSetRunStat(DN_RUN_STAT_RUNNING);
-  dnodeReportStartupFinished("TDengine", "initialized successfully");
-  dInfo("TDengine is initialized successfully");
-
-  return 0;
-}
-
-void dnodeCleanup() {
-  if (dnodeGetRunStat() != DN_RUN_STAT_STOPPED) {
-    dnodeSetRunStat(DN_RUN_STAT_STOPPED);
-    taosStepCleanup(tsInt.steps);
-    tsInt.steps = NULL;
-  }
-}
diff --git a/source/dnode/mgmt/src/dnodeMnode.c b/source/dnode/mgmt/src/dnodeMnode.c
deleted file mode 100644
index 48cc1cb20d..0000000000
--- a/source/dnode/mgmt/src/dnodeMnode.c
+++ /dev/null
@@ -1,592 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeMnode.h"
-#include "cJSON.h"
-#include "dnodeDnode.h"
-#include "dnodeTransport.h"
-#include "mnode.h"
-#include "tlockfree.h"
-#include "tqueue.h"
-#include "tstep.h"
-#include "tworker.h"
-
-static struct {
-  int32_t     refCount;
-  int8_t      deployed;
-  int8_t      dropped;
-  SWorkerPool mgmtPool;
-  SWorkerPool readPool;
-  SWorkerPool writePool;
-  SWorkerPool syncPool;
-  taos_queue  pReadQ;
-  taos_queue  pWriteQ;
-  taos_queue  pApplyQ;
-  taos_queue  pSyncQ;
-  taos_queue  pMgmtQ;
-  SSteps     *pSteps;
-  SRWLatch    latch;
-} tsMnode = {0};
-
-static int32_t dnodeAllocMnodeReadQueue();
-static void    dnodeFreeMnodeReadQueue();
-static int32_t dnodeAllocMnodeWriteQueue();
-static void    dnodeFreeMnodeWriteQueue();
-static int32_t dnodeAllocMnodeApplyQueue();
-static void    dnodeFreeMnodeApplyQueue();
-static int32_t dnodeAllocMnodeSyncQueue();
-static void    dnodeFreeMnodeSyncQueue();
-
-static int32_t dnodeAcquireMnode() {
-  taosRLockLatch(&tsMnode.latch);
-
-  int32_t code = tsMnode.deployed ? 0 : TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-  if (code == 0) {
-    atomic_add_fetch_32(&tsMnode.refCount, 1);
-  }
-
-  taosRUnLockLatch(&tsMnode.latch);
-  return code;
-}
-
-static void dnodeReleaseMnode() { atomic_sub_fetch_32(&tsMnode.refCount, 1); }
-
-static int32_t dnodeReadMnodeFile() {
-  int32_t code = TSDB_CODE_DND_READ_MNODE_FILE_ERROR;
-  int32_t len = 0;
-  int32_t maxLen = 300;
-  char   *content = calloc(1, maxLen + 1);
-  cJSON  *root = NULL;
-  FILE   *fp = NULL;
-  char    file[PATH_MAX + 20] = {0};
-
-  snprintf(file, sizeof(file), "%s/mnode.json", tsDnodeDir);
-  fp = fopen(file, "r");
-  if (!fp) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_MNODE_OVER;
-  }
-
-  len = (int32_t)fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_String) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  tsMnode.deployed = atoi(deployed->valuestring);
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_String) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  tsMnode.dropped = atoi(dropped->valuestring);
-
-  code = 0;
-  dInfo("succcessed to read file %s", file);
-
-PRASE_MNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
-
-  return code;
-}
-
-static int32_t dnodeWriteMnodeFile() {
-  char file[PATH_MAX + 20] = {0};
-  char realfile[PATH_MAX + 20] = {0};
-  snprintf(file, sizeof(file), "%s/mnode.json.bak", tsDnodeDir);
-  snprintf(realfile, sizeof(realfile), "%s/mnode.json", tsDnodeDir);
-
-  FILE *fp = fopen(file, "w");
-  if (!fp) {
-    dError("failed to write %s since %s", file, strerror(errno));
-    return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 300;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": \"%d\",\n", tsMnode.deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": \"%d\"\n", tsMnode.dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  fwrite(content, 1, len, fp);
-  taosFsyncFile(fileno(fp));
-  fclose(fp);
-  free(content);
-
-  int32_t code = taosRenameFile(file, realfile);
-  if (code != 0) {
-    dError("failed to rename %s since %s", file, tstrerror(code));
-    return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR;
-  }
-
-  dInfo("successed to write %s", realfile);
-  return 0;
-}
-
-static int32_t dnodeStartMnode() {
-  int32_t code = dnodeAllocMnodeReadQueue();
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocMnodeWriteQueue();
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocMnodeApplyQueue();
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocMnodeSyncQueue();
-  if (code != 0) {
-    return code;
-  }
-
-  taosWLockLatch(&tsMnode.latch);
-  tsMnode.deployed = 1;
-  taosWUnLockLatch(&tsMnode.latch);
-
-  return mnodeStart(NULL);
-}
-
-static void dnodeStopMnode() {
-  taosWLockLatch(&tsMnode.latch);
-  tsMnode.deployed = 0;
-  taosWUnLockLatch(&tsMnode.latch);
-
-  dnodeReleaseMnode();
-
-  while (tsMnode.refCount > 0) taosMsleep(10);
-  while (!taosQueueEmpty(tsMnode.pReadQ)) taosMsleep(10);
-  while (!taosQueueEmpty(tsMnode.pApplyQ)) taosMsleep(10);
-  while (!taosQueueEmpty(tsMnode.pWriteQ)) taosMsleep(10);
-  while (!taosQueueEmpty(tsMnode.pSyncQ)) taosMsleep(10);
-
-  dnodeFreeMnodeReadQueue();
-  dnodeFreeMnodeWriteQueue();
-  dnodeFreeMnodeApplyQueue();
-  dnodeFreeMnodeSyncQueue();
-}
-
-static int32_t dnodeUnDeployMnode() {
-  tsMnode.dropped = 1;
-  int32_t code = dnodeWriteMnodeFile();
-  if (code != 0) {
-    tsMnode.dropped = 0;
-    dError("failed to undeploy mnode since %s", tstrerror(code));
-    return code;
-  }
-
-  dnodeStopMnode();
-  mnodeUnDeploy();
-  dnodeWriteMnodeFile();
-
-  return code;
-}
-
-static int32_t dnodeDeployMnode(SMnodeCfg *pCfg) {
-  int32_t code = mnodeDeploy(pCfg);
-  if (code != 0) {
-    dError("failed to deploy mnode since %s", tstrerror(code));
-    return code;
-  }
-
-  code = dnodeStartMnode();
-  if (code != 0) {
-    dnodeUnDeployMnode();
-    dError("failed to deploy mnode since %s", tstrerror(code));
-    return code;
-  }
-
-  code = dnodeWriteMnodeFile();
-  if (code != 0) {
-    dnodeUnDeployMnode();
-    dError("failed to deploy mnode since %s", tstrerror(code));
-    return code;
-  }
-
-  dInfo("deploy mnode success");
-  return code;
-}
-
-static int32_t dnodeAlterMnode(SMnodeCfg *pCfg) {
-  int32_t code = dnodeAcquireMnode();
-  if (code == 0) {
-    code = mnodeAlter(pCfg);
-    dnodeReleaseMnode();
-  }
-  return code;
-}
-
-static SCreateMnodeMsg *dnodeParseCreateMnodeMsg(SRpcMsg *pRpcMsg) {
-  SCreateMnodeMsg *pMsg = pRpcMsg->pCont;
-  pMsg->dnodeId = htonl(pMsg->dnodeId);
-  for (int32_t i = 0; i < pMsg->replica; ++i) {
-    pMsg->replicas[i].port = htons(pMsg->replicas[i].port);
-  }
-  return pMsg;
-}
-
-static int32_t dnodeProcessCreateMnodeReq(SRpcMsg *pRpcMsg) {
-  SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont);
-
-  if (pMsg->dnodeId != dnodeGetDnodeId()) {
-    return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
-  } else {
-    SMnodeCfg cfg = {0};
-    cfg.replica = pMsg->replica;
-    memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA));
-    return dnodeDeployMnode(&cfg);
-  }
-}
-
-static int32_t dnodeProcessAlterMnodeReq(SRpcMsg *pRpcMsg) {
-  SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont);
-  if (pMsg->dnodeId != dnodeGetDnodeId()) {
-    return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
-  } else {
-    SMnodeCfg cfg = {0};
-    cfg.replica = pMsg->replica;
-    memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA));
-    return dnodeAlterMnode(&cfg);
-  }
-}
-
-static int32_t dnodeProcessDropMnodeReq(SRpcMsg *pMsg) {
-  SAlterMnodeMsg *pCfg = pMsg->pCont;
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-
-  if (pCfg->dnodeId != dnodeGetDnodeId()) {
-    return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
-  } else {
-    return dnodeUnDeployMnode();
-  }
-}
-
-static void dnodeProcessMnodeMgmtQueue(void *unused, SRpcMsg *pMsg) {
-  int32_t code = 0;
-
-  switch (pMsg->msgType) {
-    case TSDB_MSG_TYPE_CREATE_MNODE_IN:
-      code = dnodeProcessCreateMnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_ALTER_MNODE_IN:
-      code = dnodeProcessAlterMnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_DROP_MNODE_IN:
-      code = dnodeProcessDropMnodeReq(pMsg);
-      break;
-    default:
-      code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-      break;
-  }
-
-  SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
-  rpcSendResponse(&rsp);
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dnodeProcessMnodeReadQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_READ); }
-
-static void dnodeProcessMnodeWriteQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_WRITE); }
-
-static void dnodeProcessMnodeApplyQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_APPLY); }
-
-static void dnodeProcessMnodeSyncQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_SYNC); }
-
-static int32_t dnodeWriteMnodeMsgToQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
-  int32_t code = 0;
-
-  if (pQueue == NULL) {
-    code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-  } else {
-    SMnodeMsg *pMsg = mnodeInitMsg(pRpcMsg);
-    if (pMsg == NULL) {
-      code = terrno;
-    }
-  }
-
-  if (code != TSDB_CODE_SUCCESS) {
-    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pRpcMsg->pCont);
-  }
-}
-
-void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteMnodeMsgToQueue(tsMnode.pMgmtQ, pMsg); }
-
-void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  if (dnodeAcquireMnode() == 0) {
-    dnodeWriteMnodeMsgToQueue(tsMnode.pWriteQ, pMsg);
-    dnodeReleaseMnode();
-  } else {
-    dnodeSendRedirectMsg(pMsg, 0);
-  }
-}
-
-void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  int32_t code = dnodeAcquireMnode();
-  if (code == 0) {
-    dnodeWriteMnodeMsgToQueue(tsMnode.pSyncQ, pMsg);
-    dnodeReleaseMnode();
-  } else {
-    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  if (dnodeAcquireMnode() == 0) {
-    dnodeWriteMnodeMsgToQueue(tsMnode.pReadQ, pMsg);
-    dnodeReleaseMnode();
-  } else {
-    dnodeSendRedirectMsg(pMsg, 0);
-  }
-}
-
-static int32_t dnodePutMsgIntoMnodeApplyQueue(SMnodeMsg *pMsg) {
-  int32_t code = dnodeAcquireMnode();
-  if (code != 0) return code;
-
-  code = taosWriteQitem(tsMnode.pApplyQ, pMsg);
-  dnodeReleaseMnode();
-  return code;
-}
-
-static int32_t dnodeAllocMnodeMgmtQueue() {
-  tsMnode.pMgmtQ = tWorkerAllocQueue(&tsMnode.mgmtPool, NULL, (FProcessItem)dnodeProcessMnodeMgmtQueue);
-  if (tsMnode.pMgmtQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeMnodeMgmtQueue() {
-  tWorkerFreeQueue(&tsMnode.mgmtPool, tsMnode.pMgmtQ);
-  tsMnode.pMgmtQ = NULL;
-}
-
-static int32_t dnodeInitMnodeMgmtWorker() {
-  SWorkerPool *pPool = &tsMnode.mgmtPool;
-  pPool->name = "mnode-mgmt";
-  pPool->min = 1;
-  pPool->max = 1;
-  return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeMgmtWorker() { tWorkerCleanup(&tsMnode.mgmtPool); }
-
-static int32_t dnodeAllocMnodeReadQueue() {
-  tsMnode.pReadQ = tWorkerAllocQueue(&tsMnode.readPool, NULL, (FProcessItem)dnodeProcessMnodeReadQueue);
-  if (tsMnode.pReadQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeMnodeReadQueue() {
-  tWorkerFreeQueue(&tsMnode.readPool, tsMnode.pReadQ);
-  tsMnode.pReadQ = NULL;
-}
-
-static int32_t dnodeInitMnodeReadWorker() {
-  SWorkerPool *pPool = &tsMnode.readPool;
-  pPool->name = "mnode-read";
-  pPool->min = 0;
-  pPool->max = 1;
-  return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeReadWorker() { tWorkerCleanup(&tsMnode.readPool); }
-
-static int32_t dnodeAllocMnodeWriteQueue() {
-  tsMnode.pWriteQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeWriteQueue);
-  if (tsMnode.pWriteQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeMnodeWriteQueue() {
-  tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pWriteQ);
-  tsMnode.pWriteQ = NULL;
-}
-
-static int32_t dnodeAllocMnodeApplyQueue() {
-  tsMnode.pApplyQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeApplyQueue);
-  if (tsMnode.pApplyQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeMnodeApplyQueue() {
-  tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pApplyQ);
-  tsMnode.pApplyQ = NULL;
-}
-
-static int32_t dnodeInitMnodeWriteWorker() {
-  SWorkerPool *pPool = &tsMnode.writePool;
-  pPool->name = "mnode-write";
-  pPool->min = 0;
-  pPool->max = 1;
-  return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeWriteWorker() { tWorkerCleanup(&tsMnode.writePool); }
-
-static int32_t dnodeAllocMnodeSyncQueue() {
-  tsMnode.pSyncQ = tWorkerAllocQueue(&tsMnode.syncPool, NULL, (FProcessItem)dnodeProcessMnodeSyncQueue);
-  if (tsMnode.pSyncQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeMnodeSyncQueue() {
-  tWorkerFreeQueue(&tsMnode.syncPool, tsMnode.pSyncQ);
-  tsMnode.pSyncQ = NULL;
-}
-
-static int32_t dnodeInitMnodeSyncWorker() {
-  SWorkerPool *pPool = &tsMnode.syncPool;
-  pPool->name = "mnode-sync";
-  pPool->min = 0;
-  pPool->max = 1;
-  return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeSyncWorker() { tWorkerCleanup(&tsMnode.syncPool); }
-
-static int32_t dnodeInitMnodeModule() {
-  taosInitRWLatch(&tsMnode.latch);
-
-  SMnodePara para;
-  para.dnodeId = dnodeGetDnodeId();
-  para.clusterId = dnodeGetClusterId();
-  para.SendMsgToDnode = dnodeSendMsgToDnode;
-  para.SendMsgToMnode = dnodeSendMsgToMnode;
-  para.SendRedirectMsg = dnodeSendRedirectMsg;
-
-  return mnodeInit(para);
-}
-
-static void dnodeCleanupMnodeModule() { mnodeCleanup(); }
-
-static bool dnodeNeedDeployMnode() {
-  if (dnodeGetDnodeId() > 0) return false;
-  if (dnodeGetClusterId() > 0) return false;
-  if (strcmp(tsFirst, tsLocalEp) != 0) return false;
-  return true;
-}
-
-static int32_t dnodeOpenMnode() {
-  int32_t code = dnodeReadMnodeFile();
-  if (code != 0) {
-    dError("failed to read open mnode since %s", tstrerror(code));
-    return code;
-  }
-
-  if (tsMnode.dropped) {
-    dInfo("mnode already dropped, undeploy it");
-    return dnodeUnDeployMnode();
-  }
-
-  if (!tsMnode.deployed) {
-    bool needDeploy = dnodeNeedDeployMnode();
-    if (!needDeploy) return 0;
-
-    dInfo("start to deploy mnode");
-    SMnodeCfg cfg = {.replica = 1};
-    cfg.replicas[0].port = tsServerPort;
-    tstrncpy(cfg.replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
-    code = dnodeDeployMnode(&cfg);
-  } else {
-    dInfo("start to open mnode");
-    return dnodeStartMnode();
-  }
-}
-
-static void dnodeCloseMnode() {
-  if (dnodeAcquireMnode() == 0) {
-    dnodeStopMnode();
-  }
-}
-
-int32_t dnodeInitMnode() {
-  dInfo("dnode-mnode start to init");
-
-  SSteps *pSteps = taosStepInit(6, dnodeReportStartup);
-  taosStepAdd(pSteps, "dnode-mnode-env", dnodeInitMnodeModule, dnodeCleanupMnodeModule);
-  taosStepAdd(pSteps, "dnode-mnode-mgmt", dnodeInitMnodeMgmtWorker, dnodeCleanupMnodeMgmtWorker);
-  taosStepAdd(pSteps, "dnode-mnode-read", dnodeInitMnodeReadWorker, dnodeCleanupMnodeReadWorker);
-  taosStepAdd(pSteps, "dnode-mnode-write", dnodeInitMnodeWriteWorker, dnodeCleanupMnodeWriteWorker);
-  taosStepAdd(pSteps, "dnode-mnode-sync", dnodeInitMnodeSyncWorker, dnodeCleanupMnodeSyncWorker);
-  taosStepAdd(pSteps, "dnode-mnode", dnodeOpenMnode, dnodeCloseMnode);
-
-  tsMnode.pSteps = pSteps;
-  int32_t code = taosStepExec(pSteps);
-
-  if (code != 0) {
-    dError("dnode-mnode init failed since %s", tstrerror(code));
-  } else {
-    dInfo("dnode-mnode is initialized");
-  }
-}
-
-void dnodeCleanupMnode() {
-  if (tsMnode.pSteps == NULL) {
-    dInfo("dnode-mnode start to clean up");
-    taosStepCleanup(tsMnode.pSteps);
-    tsMnode.pSteps = NULL;
-    dInfo("dnode-mnode is cleaned up");
-  }
-}
-
-int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  int32_t code = dnodeAcquireMnode();
-  if (code != 0) {
-    dTrace("failed to get user auth since mnode not deployed");
-    return code;
-  }
-
-  code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
-  dnodeReleaseMnode();
-  return code;
-}
\ No newline at end of file
diff --git a/source/dnode/mgmt/src/dnodeTransport.c b/source/dnode/mgmt/src/dnodeTransport.c
deleted file mode 100644
index 475470b574..0000000000
--- a/source/dnode/mgmt/src/dnodeTransport.c
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-/* this file is mainly responsible for the communication between DNODEs. Each 
- * dnode works as both server and client. Dnode may send status, grant, config
- * messages to mnode, mnode may send create/alter/drop table/vnode messages 
- * to dnode. All theses messages are handled from here
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeTransport.h"
-#include "dnodeDnode.h"
-#include "dnodeMnode.h"
-#include "dnodeVnodes.h"
-
-static struct {
-  void *peerRpc;
-  void *shellRpc;
-  void *clientRpc;
-  MsgFp msgFp[TSDB_MSG_TYPE_MAX];
-} tsTrans;
-
-static void dnodeInitMsgFp() {
-  // msg from client to dnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_QUERY] = dnodeProcessVnodeQueryMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_FETCH] = dnodeProcessVnodeFetchMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_TABLE_META] = dnodeProcessVnodeQueryMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_TABLES_META] = dnodeProcessVnodeQueryMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dnodeProcessVnodeQueryMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dnodeProcessVnodeQueryMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_ACK] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_MQ_RESET] = dnodeProcessVnodeWriteMsg;
-  
-  // msg from client to mnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_CONNECT] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_USER] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_USER] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_USER] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DB] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DB] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_USE_DB] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_DB] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_DB] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_KILL_CONN] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SHOW] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dnodeProcessMnodeWriteMsg;
-
-  // message from client to dnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessDnodeMsg;
-
-  // message from mnode to vnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessVnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessMnodeWriteMsg;
-
-  // message from mnode to dnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dnodeProcessDnodeMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-
-  // message from dnode to mnode
-  tsTrans.msgFp[TSDB_MSG_TYPE_AUTH] = dnodeProcessMnodeReadMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dnodeProcessDnodeMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_GRANT] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dnodeProcessDnodeMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_STATUS] = dnodeProcessMnodeWriteMsg;
-  tsTrans.msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dnodeProcessDnodeMsg;
-}
-
-static void dnodeProcessPeerReq(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SRpcMsg rspMsg = {.handle = pMsg->handle};
-  int32_t msgType = pMsg->msgType;
-
-  if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
-    dnodeProcessDnodeMsg(pMsg, pEpSet);
-    return;
-  }
-
-  if (dnodeGetRunStat() != DN_RUN_STAT_RUNNING) {
-    rspMsg.code = TSDB_CODE_APP_NOT_READY;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-    dTrace("RPC %p, peer req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
-    return;
-  }
-
-  if (pMsg->pCont == NULL) {
-    rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN;
-    rpcSendResponse(&rspMsg);
-    return;
-  }
-
-  MsgFp fp = tsTrans.msgFp[msgType];
-  if (fp != NULL) {
-    dTrace("RPC %p, peer req:%s will be processed", pMsg->handle, taosMsg[msgType]);
-    (*fp)(pMsg, pEpSet);
-  } else {
-    dError("RPC %p, peer req:%s not processed", pMsg->handle, taosMsg[msgType]);
-    rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static int32_t dnodeInitPeerServer() {
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.localPort = tsDnodeDnodePort;
-  rpcInit.label = "DND-S";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp = dnodeProcessPeerReq;
-  rpcInit.sessions = TSDB_MAX_VNODES << 4;
-  rpcInit.connType = TAOS_CONN_SERVER;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-
-  tsTrans.peerRpc = rpcOpen(&rpcInit);
-  if (tsTrans.peerRpc == NULL) {
-    dError("failed to init peer rpc server");
-    return -1;
-  }
-
-  dInfo("dnode peer rpc server is initialized");
-  return 0;
-}
-
-static void dnodeCleanupPeerServer() {
-  if (tsTrans.peerRpc) {
-    rpcClose(tsTrans.peerRpc);
-    tsTrans.peerRpc = NULL;
-    dInfo("dnode peer server is closed");
-  }
-}
-
-static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  int32_t msgType = pMsg->msgType;
-
-  if (dnodeGetRunStat() == DN_RUN_STAT_STOPPED) {
-    if (pMsg == NULL || pMsg->pCont == NULL) return;
-    dTrace("RPC %p, peer rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]);
-    rpcFreeCont(pMsg->pCont);
-    return;
-  }
-
-  MsgFp fp = tsTrans.msgFp[msgType];
-  if (fp != NULL) {
-    dTrace("RPC %p, peer rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code));
-    (*fp)(pMsg, pEpSet);
-  } else {
-    dDebug("RPC %p, peer rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-}
-
-static int32_t dnodeInitClient() {
-  char secret[TSDB_KEY_LEN] = "secret";
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.label = "DND-C";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp = dnodeProcessPeerRsp;
-  rpcInit.sessions = TSDB_MAX_VNODES << 4;
-  rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.user = "t";
-  rpcInit.ckey = "key";
-  rpcInit.secret = secret;
-
-  tsTrans.clientRpc = rpcOpen(&rpcInit);
-  if (tsTrans.clientRpc == NULL) {
-    dError("failed to init peer rpc client");
-    return -1;
-  }
-
-  dInfo("dnode peer rpc client is initialized");
-  return 0;
-}
-
-static void dnodeCleanupClient() {
-  if (tsTrans.clientRpc) {
-    rpcClose(tsTrans.clientRpc);
-    tsTrans.clientRpc = NULL;
-    dInfo("dnode peer rpc client is closed");
-  }
-}
-
-static void dnodeProcessShellReq(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SRpcMsg rspMsg = {.handle = pMsg->handle};
-  int32_t msgType = pMsg->msgType;
-
-  if (dnodeGetRunStat() == DN_RUN_STAT_STOPPED) {
-    dError("RPC %p, shell req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]);
-    rspMsg.code = TSDB_CODE_DND_EXITING;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-    return;
-  } else if (dnodeGetRunStat() != DN_RUN_STAT_RUNNING) {
-    dError("RPC %p, shell req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
-    rspMsg.code = TSDB_CODE_APP_NOT_READY;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-    return;
-  }
-
-  if (pMsg->pCont == NULL) {
-    rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN;
-    rpcSendResponse(&rspMsg);
-    return;
-  }
-
-  MsgFp fp = tsTrans.msgFp[msgType];
-  if (fp != NULL) {
-    dTrace("RPC %p, shell req:%s will be processed", pMsg->handle, taosMsg[msgType]);
-    (*fp)(pMsg, pEpSet);
-  } else {
-    dError("RPC %p, shell req:%s is not processed", pMsg->handle, taosMsg[msgType]);
-    rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
-  SEpSet epSet = {0};
-  dnodeGetMnodeEpSetForPeer(&epSet);
-  rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp);
-}
-
-static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  int32_t code = dnodeGetUserAuthFromMnode(user, spi, encrypt, secret, ckey);
-  if (code != TSDB_CODE_APP_NOT_READY) return code;
-
-  SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
-  tstrncpy(pMsg->user, user, sizeof(pMsg->user));
-
-  dDebug("user:%s, send auth msg to mnodes", user);
-  SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH};
-  SRpcMsg rpcRsp = {0};
-  dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
-
-  if (rpcRsp.code != 0) {
-    dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
-  } else {
-    dDebug("user:%s, auth msg received from mnodes", user);
-    SAuthRsp *pRsp = rpcRsp.pCont;
-    memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
-    memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
-    *spi = pRsp->spi;
-    *encrypt = pRsp->encrypt;
-  }
-
-  rpcFreeCont(rpcRsp.pCont);
-  return rpcRsp.code;
-}
-
-static int32_t dnodeInitShellServer() {
-  int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
-  if (numOfThreads < 1) {
-    numOfThreads = 1;
-  }
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.localPort = tsDnodeShellPort;
-  rpcInit.label = "SHELL";
-  rpcInit.numOfThreads = numOfThreads;
-  rpcInit.cfp = dnodeProcessShellReq;
-  rpcInit.sessions = tsMaxShellConns;
-  rpcInit.connType = TAOS_CONN_SERVER;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.afp = dnodeRetrieveUserAuthInfo;
-
-  tsTrans.shellRpc = rpcOpen(&rpcInit);
-  if (tsTrans.shellRpc == NULL) {
-    dError("failed to init shell rpc server");
-    return -1;
-  }
-
-  dInfo("dnode shell rpc server is initialized");
-  return 0;
-}
-
-static void dnodeCleanupShellServer() {
-  if (tsTrans.shellRpc) {
-    rpcClose(tsTrans.shellRpc);
-    tsTrans.shellRpc = NULL;
-  }
-}
-
-int32_t dnodeInitTrans() {
-  if (dnodeInitClient() != 0) {
-    return -1;
-  }
-
-  if (dnodeInitPeerServer() != 0) {
-    return -1;
-  }
-
-  if (dnodeInitShellServer() != 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-void dnodeCleanupTrans() {
-  dnodeCleanupShellServer();
-  dnodeCleanupPeerServer();
-  dnodeCleanupClient();
-}
-
-void dnodeSendMsgToDnode(SEpSet *epSet, SRpcMsg *rpcMsg) { rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL); }
-
-void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
-  SEpSet epSet = {0};
-  dnodeGetMnodeEpSetForPeer(&epSet);
-  dnodeSendMsgToDnode(&epSet, rpcMsg);
-}
\ No newline at end of file
diff --git a/source/dnode/mgmt/src/dnodeVnodes.c b/source/dnode/mgmt/src/dnodeVnodes.c
deleted file mode 100644
index 36a8060161..0000000000
--- a/source/dnode/mgmt/src/dnodeVnodes.c
+++ /dev/null
@@ -1,1031 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeVnodes.h"
-#include "cJSON.h"
-#include "dnodeTransport.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tqueue.h"
-#include "tstep.h"
-#include "tthread.h"
-#include "tworker.h"
-#include "vnode.h"
-
-typedef struct {
-  int32_t    vgId;
-  int32_t    refCount;
-  int8_t     dropped;
-  int8_t     accessState;
-  SVnode *   pImpl;
-  taos_queue pWriteQ;
-  taos_queue pSyncQ;
-  taos_queue pApplyQ;
-  taos_queue pQueryQ;
-  taos_queue pFetchQ;
-} SVnodeObj;
-
-typedef struct {
-  pthread_t *threadId;
-  int32_t    threadIndex;
-  int32_t    failed;
-  int32_t    opened;
-  int32_t    vnodeNum;
-  SVnodeObj *pVnodes;
-} SVThread;
-
-static struct {
-  SHashObj *   hash;
-  SWorkerPool  mgmtPool;
-  SWorkerPool  queryPool;
-  SWorkerPool  fetchPool;
-  SMWorkerPool syncPool;
-  SMWorkerPool writePool;
-  taos_queue   pMgmtQ;
-  SSteps *     pSteps;
-  int32_t      openVnodes;
-  int32_t      totalVnodes;
-  SRWLatch     latch;
-} tsVnodes;
-
-static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode);
-static void    dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode);
-static void    dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode);
-static void    dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode);
-static void    dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode);
-static void    dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode);
-
-static SVnodeObj *dnodeAcquireVnode(int32_t vgId) {
-  SVnodeObj *pVnode = NULL;
-  int32_t    refCount = 0;
-
-  taosRLockLatch(&tsVnodes.latch);
-  taosHashGetClone(tsVnodes.hash, &vgId, sizeof(int32_t), (void *)&pVnode);
-  if (pVnode == NULL) {
-    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
-  } else {
-    refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
-  }
-  taosRUnLockLatch(&tsVnodes.latch);
-
-  dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount);
-  return pVnode;
-}
-
-static void dnodeReleaseVnode(SVnodeObj *pVnode) {
-  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
-}
-
-static int32_t dnodeCreateVnodeWrapper(int32_t vgId, SVnode *pImpl) {
-  SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
-  if (pVnode == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-
-  pVnode->vgId = vgId;
-  pVnode->refCount = 0;
-  pVnode->dropped = 0;
-  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
-  pVnode->pImpl = pImpl;
-
-  int32_t code = dnodeAllocVnodeQueryQueue(pVnode);
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocVnodeFetchQueue(pVnode);
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocVnodeWriteQueue(pVnode);
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocVnodeApplyQueue(pVnode);
-  if (code != 0) {
-    return code;
-  }
-
-  code = dnodeAllocVnodeSyncQueue(pVnode);
-  if (code != 0) {
-    return code;
-  }
-
-  taosWLockLatch(&tsVnodes.latch);
-  code = taosHashPut(tsVnodes.hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
-  taosWUnLockLatch(&tsVnodes.latch);
-
-  return code;
-}
-
-static void dnodeDropVnodeWrapper(SVnodeObj *pVnode) {
-  taosWLockLatch(&tsVnodes.latch);
-  taosHashRemove(tsVnodes.hash, &pVnode->vgId, sizeof(int32_t));
-  taosWUnLockLatch(&tsVnodes.latch);
-
-  // wait all queue empty
-  dnodeReleaseVnode(pVnode);
-  while (pVnode->refCount > 0) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
-
-  dnodeFreeVnodeQueryQueue(pVnode);
-  dnodeFreeVnodeFetchQueue(pVnode);
-  dnodeFreeVnodeWriteQueue(pVnode);
-  dnodeFreeVnodeApplyQueue(pVnode);
-  dnodeFreeVnodeSyncQueue(pVnode);
-}
-
-static SVnodeObj **dnodeGetVnodesFromHash(int32_t *numOfVnodes) {
-  taosRLockLatch(&tsVnodes.latch);
-
-  int32_t     num = 0;
-  int32_t     size = taosHashGetSize(tsVnodes.hash);
-  SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *));
-
-  void *pIter = taosHashIterate(tsVnodes.hash, NULL);
-  while (pIter) {
-    SVnodeObj **ppVnode = pIter;
-    SVnodeObj * pVnode = *ppVnode;
-    if (pVnode) {
-      num++;
-      if (num < size) {
-        int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
-        dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount);
-        pVnodes[num] = (*ppVnode);
-      }
-    }
-    pIter = taosHashIterate(tsVnodes.hash, pIter);
-  }
-
-  taosRUnLockLatch(&tsVnodes.latch);
-  *numOfVnodes = num;
-
-  return pVnodes;
-}
-
-static int32_t dnodeGetVnodesFromFile(SVnodeObj **ppVnodes, int32_t *numOfVnodes) {
-  int32_t    code = TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR;
-  int32_t    len = 0;
-  int32_t    maxLen = 30000;
-  char *     content = calloc(1, maxLen + 1);
-  cJSON *    root = NULL;
-  FILE *     fp = NULL;
-  char       file[PATH_MAX + 20] = {0};
-  SVnodeObj *pVnodes = NULL;
-
-  snprintf(file, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir);
-
-  fp = fopen(file, "r");
-  if (!fp) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_VNODE_OVER;
-  }
-
-  len = (int32_t)fread(content, 1, maxLen, fp);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_VNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_VNODE_OVER;
-  }
-
-  cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes");
-  if (!vnodes || vnodes->type != cJSON_Array) {
-    dError("failed to read %s since vnodes not found", file);
-    goto PRASE_VNODE_OVER;
-  }
-
-  int32_t vnodesNum = cJSON_GetArraySize(vnodes);
-  if (vnodesNum <= 0) {
-    dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum);
-    goto PRASE_VNODE_OVER;
-  }
-
-  pVnodes = calloc(vnodesNum, sizeof(SVnodeObj));
-  if (pVnodes == NULL) {
-    dError("failed to read %s since out of memory", file);
-    goto PRASE_VNODE_OVER;
-  }
-
-  for (int32_t i = 0; i < vnodesNum; ++i) {
-    cJSON *    vnode = cJSON_GetArrayItem(vnodes, i);
-    SVnodeObj *pVnode = &pVnodes[i];
-
-    cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
-    if (!vgId || vgId->type != cJSON_String) {
-      dError("failed to read %s since vgId not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    pVnode->vgId = atoi(vgId->valuestring);
-
-    cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
-    if (!dropped || dropped->type != cJSON_String) {
-      dError("failed to read %s since dropped not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    pVnode->dropped = atoi(vnode->valuestring);
-  }
-
-  code = 0;
-  dInfo("succcessed to read file %s", file);
-
-PRASE_VNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (fp != NULL) fclose(fp);
-
-  return code;
-}
-
-static int32_t dnodeWriteVnodesToFile() {
-  char file[PATH_MAX + 20] = {0};
-  char realfile[PATH_MAX + 20] = {0};
-  snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", tsVnodeDir);
-  snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir);
-
-  FILE *fp = fopen(file, "w");
-  if (!fp) {
-    dError("failed to write %s since %s", file, strerror(errno));
-    return -1;
-  }
-
-  int32_t     len = 0;
-  int32_t     maxLen = 30000;
-  char *      content = calloc(1, maxLen + 1);
-  int32_t     numOfVnodes = 0;
-  SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"vnodes\": [{\n");
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    SVnodeObj *pVnode = pVnodes[i];
-    len += snprintf(content + len, maxLen - len, "    \"vgId\": \"%d\",\n", pVnode->vgId);
-    len += snprintf(content + len, maxLen - len, "    \"dropped\": \"%d\"\n", pVnode->dropped);
-    if (i < numOfVnodes - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  fwrite(content, 1, len, fp);
-  taosFsyncFile(fileno(fp));
-  fclose(fp);
-  free(content);
-  terrno = 0;
-
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    SVnodeObj *pVnode = pVnodes[i];
-    dnodeReleaseVnode(pVnode);
-  }
-
-  if (pVnodes != NULL) {
-    free(pVnodes);
-  }
-
-  dInfo("successed to write %s", file);
-  return taosRenameFile(file, realfile);
-}
-
-static int32_t dnodeCreateVnode(int32_t vgId, SVnodeCfg *pCfg) {
-  int32_t code = 0;
-
-  char path[PATH_MAX + 20] = {0};
-  snprintf(path, sizeof(path), "%s/vnode%d", tsVnodeDir, vgId);
-  SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
-
-  if (pImpl == NULL) {
-    code = terrno;
-    return code;
-  }
-
-  code = dnodeCreateVnodeWrapper(vgId, pImpl);
-  if (code != 0) {
-    vnodeDrop(pImpl);
-    return code;
-  }
-
-  code = dnodeWriteVnodesToFile();
-  if (code != 0) {
-    vnodeDrop(pImpl);
-    return code;
-  }
-
-  return code;
-}
-
-static int32_t dnodeDropVnode(SVnodeObj *pVnode) {
-  pVnode->dropped = 1;
-
-  int32_t code = dnodeWriteVnodesToFile();
-  if (code != 0) {
-    pVnode->dropped = 0;
-    return code;
-  }
-
-  dnodeDropVnodeWrapper(pVnode);
-  vnodeDrop(pVnode->pImpl);
-  dnodeWriteVnodesToFile();
-  return 0;
-}
-
-static void *dnodeOpenVnodeFunc(void *param) {
-  SVThread *pThread = param;
-
-  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
-  setThreadName("open-vnodes");
-
-  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
-    SVnodeObj *pVnode = &pThread->pVnodes[v];
-
-    char stepDesc[TSDB_STEP_DESC_LEN] = {0};
-    snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId,
-             tsVnodes.openVnodes, tsVnodes.totalVnodes);
-    dnodeReportStartup("open-vnodes", stepDesc);
-
-    char path[PATH_MAX + 20] = {0};
-    snprintf(path, sizeof(path), "%s/vnode%d", tsVnodeDir, pVnode->vgId);
-    SVnode *pImpl = vnodeOpen(path, NULL);
-    if (pImpl == NULL) {
-      dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
-      pThread->failed++;
-    } else {
-      dnodeCreateVnodeWrapper(pVnode->vgId, pImpl);
-      dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex);
-      pThread->opened++;
-    }
-
-    atomic_add_fetch_32(&tsVnodes.openVnodes, 1);
-  }
-
-  dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
-         pThread->failed);
-  return NULL;
-}
-
-static int32_t dnodeOpenVnodes() {
-  taosInitRWLatch(&tsVnodes.latch);
-
-  tsVnodes.hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (tsVnodes.hash == NULL) {
-    dError("failed to init vnode hash");
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  SVnodeObj *pVnodes = NULL;
-  int32_t    numOfVnodes = 0;
-  int32_t    code = dnodeGetVnodesFromFile(&pVnodes, &numOfVnodes);
-  if (code != TSDB_CODE_SUCCESS) {
-    dInfo("failed to get vnode list from disk since %s", tstrerror(code));
-    return code;
-  }
-
-  tsVnodes.totalVnodes = numOfVnodes;
-
-  int32_t threadNum = tsNumOfCores;
-  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
-
-  SVThread *threads = calloc(threadNum, sizeof(SVThread));
-  for (int32_t t = 0; t < threadNum; ++t) {
-    threads[t].threadIndex = t;
-    threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj));
-  }
-
-  for (int32_t v = 0; v < numOfVnodes; ++v) {
-    int32_t   t = v % threadNum;
-    SVThread *pThread = &threads[t];
-    pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v];
-  }
-
-  dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
-
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SVThread *pThread = &threads[t];
-    if (pThread->vnodeNum == 0) continue;
-
-    pThread->threadId = taosCreateThread(dnodeOpenVnodeFunc, pThread);
-    if (pThread->threadId == NULL) {
-      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
-    }
-  }
-
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SVThread *pThread = &threads[t];
-    taosDestoryThread(pThread->threadId);
-    pThread->threadId = NULL;
-    free(pThread->pVnodes);
-  }
-  free(threads);
-
-  if (tsVnodes.openVnodes != tsVnodes.totalVnodes) {
-    dError("there are total vnodes:%d, opened:%d", tsVnodes.totalVnodes, tsVnodes.openVnodes);
-    return -1;
-  } else {
-    dInfo("total vnodes:%d open successfully", tsVnodes.totalVnodes);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeCloseVnodes() {
-  int32_t     numOfVnodes = 0;
-  SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes);
-
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    dnodeDropVnodeWrapper(pVnodes[i]);
-  }
-  if (pVnodes != NULL) {
-    free(pVnodes);
-  }
-
-  if (tsVnodes.hash != NULL) {
-    taosHashCleanup(tsVnodes.hash);
-    tsVnodes.hash = NULL;
-  }
-
-  dInfo("total vnodes:%d are all closed", numOfVnodes);
-}
-
-static int32_t dnodeParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
-  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
-  *vgId = htonl(pCreate->vgId);
-
-#if 0
-  tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
-  pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
-  pCfg->totalBlocks = htonl(pCreate->totalBlocks);
-  pCfg->daysPerFile = htonl(pCreate->daysPerFile);
-  pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
-  pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
-  pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
-  pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
-  pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
-  pCfg->precision = pCreate->precision;
-  pCfg->compression = pCreate->compression;
-  pCfg->cacheLastRow = pCreate->cacheLastRow;
-  pCfg->update = pCreate->update;
-  pCfg->quorum = pCreate->quorum;
-  pCfg->replica = pCreate->replica;
-  pCfg->walLevel = pCreate->walLevel;
-  pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
-
-  for (int32_t i = 0; i < pCfg->replica; ++i) {
-    pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
-    tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-  }
-#endif
-
-  return 0;
-}
-
-static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
-  SDropVnodeMsg *pDrop = rpcMsg->pCont;
-  pDrop->vgId = htonl(pDrop->vgId);
-  return pDrop;
-}
-
-static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
-  SAuthVnodeMsg *pAuth = rpcMsg->pCont;
-  pAuth->vgId = htonl(pAuth->vgId);
-  return pAuth;
-}
-
-static int32_t vnodeProcessCreateVnodeReq(SRpcMsg *rpcMsg) {
-  SVnodeCfg vnodeCfg = {0};
-  int32_t   vgId = 0;
-
-  dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
-  dDebug("vgId:%d, create vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode != NULL) {
-    dDebug("vgId:%d, already exist, return success", vgId);
-    dnodeReleaseVnode(pVnode);
-    return 0;
-  }
-
-  int32_t code = dnodeCreateVnode(vgId, &vnodeCfg);
-  if (code != 0) {
-    dError("vgId:%d, failed to create vnode since %s", vgId, tstrerror(code));
-  }
-
-  return code;
-}
-
-static int32_t vnodeProcessAlterVnodeReq(SRpcMsg *rpcMsg) {
-  SVnodeCfg vnodeCfg = {0};
-  int32_t   vgId = 0;
-  int32_t   code = 0;
-
-  dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
-  dDebug("vgId:%d, alter vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    code = terrno;
-    dDebug("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code));
-    return code;
-  }
-
-  code = vnodeAlter(pVnode->pImpl, &vnodeCfg);
-  if (code != 0) {
-    dError("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code));
-  }
-
-  dnodeReleaseVnode(pVnode);
-  return code;
-}
-
-static int32_t vnodeProcessDropVnodeReq(SRpcMsg *rpcMsg) {
-  SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg);
-
-  int32_t code = 0;
-  int32_t vgId = pDrop->vgId;
-  dDebug("vgId:%d, drop vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    code = terrno;
-    dDebug("vgId:%d, failed to drop since %s", vgId, tstrerror(code));
-    return code;
-  }
-
-  code = dnodeDropVnode(pVnode);
-  if (code != 0) {
-    dnodeReleaseVnode(pVnode);
-    dError("vgId:%d, failed to drop vnode since %s", vgId, tstrerror(code));
-  }
-
-  return code;
-}
-
-static int32_t vnodeProcessAuthVnodeReq(SRpcMsg *rpcMsg) {
-  SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
-
-  int32_t code = 0;
-  int32_t vgId = pAuth->vgId;
-  dDebug("vgId:%d, auth vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    code = terrno;
-    dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code));
-    return code;
-  }
-
-  pVnode->accessState = pAuth->accessState;
-  dnodeReleaseVnode(pVnode);
-  return code;
-}
-
-static int32_t vnodeProcessSyncVnodeReq(SRpcMsg *rpcMsg) {
-  SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
-
-  int32_t code = 0;
-  int32_t vgId = pAuth->vgId;
-  dDebug("vgId:%d, auth vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    code = terrno;
-    dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code));
-    return code;
-  }
-
-  code = vnodeSync(pVnode->pImpl);
-  if (code != 0) {
-    dError("vgId:%d, failed to auth vnode since %s", vgId, tstrerror(code));
-  }
-
-  dnodeReleaseVnode(pVnode);
-  return code;
-}
-
-static int32_t vnodeProcessCompactVnodeReq(SRpcMsg *rpcMsg) {
-  SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg);
-
-  int32_t code = 0;
-  int32_t vgId = pCompact->vgId;
-  dDebug("vgId:%d, compact vnode req is received", vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    code = terrno;
-    dDebug("vgId:%d, failed to compact since %s", vgId, tstrerror(code));
-    return code;
-  }
-
-  code = vnodeCompact(pVnode->pImpl);
-  if (code != 0) {
-    dError("vgId:%d, failed to compact vnode since %s", vgId, tstrerror(code));
-  }
-
-  dnodeReleaseVnode(pVnode);
-  return code;
-}
-
-static void dnodeProcessVnodeMgmtQueue(void *unused, SRpcMsg *pMsg) {
-  int32_t code = 0;
-
-  switch (pMsg->msgType) {
-    case TSDB_MSG_TYPE_CREATE_VNODE_IN:
-      code = vnodeProcessCreateVnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_ALTER_VNODE_IN:
-      code = vnodeProcessAlterVnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_DROP_VNODE_IN:
-      code = vnodeProcessDropVnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_AUTH_VNODE_IN:
-      code = vnodeProcessAuthVnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_SYNC_VNODE_IN:
-      code = vnodeProcessSyncVnodeReq(pMsg);
-      break;
-    case TSDB_MSG_TYPE_COMPACT_VNODE_IN:
-      code = vnodeProcessCompactVnodeReq(pMsg);
-      break;
-    default:
-      code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-      break;
-  }
-
-  SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
-  rpcSendResponse(&rsp);
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dnodeProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
-  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY);
-}
-
-static void dnodeProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
-  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH);
-}
-
-static void dnodeProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
-  SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
-  SRpcMsg *  pRpcMsg = NULL;
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    taosGetQitem(qall, (void **)&pRpcMsg);
-    vnodeAppendMsg(pMsg, pRpcMsg);
-    taosFreeQitem(pRpcMsg);
-  }
-
-  vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE);
-}
-
-static void dnodeProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
-  SVnodeMsg *pMsg = NULL;
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    taosGetQitem(qall, (void **)&pMsg);
-    vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY);
-  }
-}
-
-static void dnodeProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
-  SVnodeMsg *pMsg = NULL;
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    taosGetQitem(qall, (void **)&pMsg);
-    vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC);
-  }
-}
-
-static int32_t dnodeWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
-  int32_t code = 0;
-
-  if (pQueue == NULL) {
-    code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-  } else {
-    SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
-    if (pMsg == NULL) {
-      code = TSDB_CODE_DND_OUT_OF_MEMORY;
-    } else {
-      *pMsg = *pRpcMsg;
-      code = taosWriteQitem(pQueue, pMsg);
-    }
-  }
-
-  if (code != TSDB_CODE_SUCCESS) {
-    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pRpcMsg->pCont);
-  }
-}
-
-static int32_t dnodeWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
-  int32_t code = 0;
-
-  if (pQueue == NULL) {
-    code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
-  } else {
-    SVnodeMsg *pMsg = vnodeInitMsg(1);
-    if (pMsg == NULL) {
-      code = TSDB_CODE_DND_OUT_OF_MEMORY;
-    } else {
-      vnodeAppendMsg(pMsg, pRpcMsg);
-      code = taosWriteQitem(pQueue, pMsg);
-    }
-  }
-
-  if (code != TSDB_CODE_SUCCESS) {
-    SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pRpcMsg->pCont);
-  }
-}
-
-static SVnodeObj *dnodeAcquireVnodeFromMsg(SRpcMsg *pMsg) {
-  SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
-  pHead->vgId = htonl(pHead->vgId);
-
-  SVnodeObj *pVnode = dnodeAcquireVnode(pHead->vgId);
-  if (pVnode == NULL) {
-    SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
-    rpcSendResponse(&rsp);
-    rpcFreeCont(pMsg->pCont);
-  }
-
-  return pVnode;
-}
-
-void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteRpcMsgToVnodeQueue(tsVnodes.pMgmtQ, pMsg); }
-
-void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
-  if (pVnode != NULL) {
-    dnodeWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg);
-    dnodeReleaseVnode(pVnode);
-  }
-}
-
-void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
-  if (pVnode != NULL) {
-    dnodeWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
-    dnodeReleaseVnode(pVnode);
-  }
-}
-
-void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
-  if (pVnode != NULL) {
-    dnodeWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
-    dnodeReleaseVnode(pVnode);
-  }
-}
-
-void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
-  if (pVnode != NULL) {
-    dnodeWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
-    dnodeReleaseVnode(pVnode);
-  }
-}
-
-static int32_t dnodePutMsgIntoVnodeApplyQueue(int32_t vgId, SVnodeMsg *pMsg) {
-  SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
-  if (pVnode == NULL) {
-    return terrno;
-  }
-
-  int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg);
-  dnodeReleaseVnode(pVnode);
-  return code;
-}
-
-static int32_t dnodeInitVnodeMgmtWorker() {
-  SWorkerPool *pPool = &tsVnodes.mgmtPool;
-  pPool->name = "vnode-mgmt";
-  pPool->min = 1;
-  pPool->max = 1;
-  if (tWorkerInit(pPool) != 0) {
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  tsVnodes.pMgmtQ = tWorkerAllocQueue(pPool, NULL, (FProcessItem)dnodeProcessVnodeMgmtQueue);
-  if (tsVnodes.pMgmtQ == NULL) {
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  return 0;
-}
-
-static void dnodeCleanupVnodeMgmtWorker() {
-  tWorkerFreeQueue(&tsVnodes.mgmtPool, tsVnodes.pMgmtQ);
-  tWorkerCleanup(&tsVnodes.mgmtPool);
-  tsVnodes.pMgmtQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode) {
-  pVnode->pQueryQ = tWorkerAllocQueue(&tsVnodes.queryPool, pVnode, (FProcessItem)dnodeProcessVnodeQueryQueue);
-  if (pVnode->pQueryQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode) {
-  tWorkerFreeQueue(&tsVnodes.queryPool, pVnode->pQueryQ);
-  pVnode->pQueryQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode) {
-  pVnode->pFetchQ = tWorkerAllocQueue(&tsVnodes.fetchPool, pVnode, (FProcessItem)dnodeProcessVnodeFetchQueue);
-  if (pVnode->pFetchQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode) {
-  tWorkerFreeQueue(&tsVnodes.fetchPool, pVnode->pFetchQ);
-  pVnode->pFetchQ = NULL;
-}
-
-static int32_t dnodeInitVnodeReadWorker() {
-  int32_t maxFetchThreads = 4;
-  float   threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
-
-  SWorkerPool *pPool = &tsVnodes.queryPool;
-  pPool->name = "vnode-query";
-  pPool->min = (int32_t)threadsForQuery;
-  pPool->max = pPool->min;
-  if (tWorkerInit(pPool) != 0) {
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  pPool = &tsVnodes.fetchPool;
-  pPool->name = "vnode-fetch";
-  pPool->min = MIN(maxFetchThreads, tsNumOfCores);
-  pPool->max = pPool->min;
-  if (tWorkerInit(pPool) != 0) {
-    TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  return 0;
-}
-
-static void dnodeCleanupVnodeReadWorker() {
-  tWorkerCleanup(&tsVnodes.fetchPool);
-  tWorkerCleanup(&tsVnodes.queryPool);
-}
-
-static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode) {
-  pVnode->pWriteQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeWriteQueue);
-  if (pVnode->pWriteQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode) {
-  tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pWriteQ);
-  pVnode->pWriteQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode) {
-  pVnode->pApplyQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeApplyQueue);
-  if (pVnode->pApplyQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode) {
-  tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pApplyQ);
-  pVnode->pApplyQ = NULL;
-}
-
-static int32_t dnodeInitVnodeWriteWorker() {
-  SMWorkerPool *pPool = &tsVnodes.writePool;
-  pPool->name = "vnode-write";
-  pPool->max = tsNumOfCores;
-  if (tMWorkerInit(pPool) != 0) {
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  return 0;
-}
-
-static void dnodeCleanupVnodeWriteWorker() { tMWorkerCleanup(&tsVnodes.writePool); }
-
-static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode) {
-  pVnode->pSyncQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeSyncQueue);
-  if (pVnode->pSyncQ == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
-  return 0;
-}
-
-static void dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode) {
-  tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pSyncQ);
-  pVnode->pSyncQ = NULL;
-}
-
-static int32_t dnodeInitVnodeSyncWorker() {
-  int32_t maxThreads = tsNumOfCores / 2;
-  if (maxThreads < 1) maxThreads = 1;
-
-  SMWorkerPool *pPool = &tsVnodes.writePool;
-  pPool->name = "vnode-sync";
-  pPool->max = maxThreads;
-  if (tMWorkerInit(pPool) != 0) {
-    return TSDB_CODE_VND_OUT_OF_MEMORY;
-  }
-
-  return 0;
-}
-
-static void dnodeCleanupVnodeSyncWorker() { tMWorkerCleanup(&tsVnodes.syncPool); }
-
-static int32_t dnodeInitVnodeModule() {
-  SVnodePara para;
-  para.SendMsgToDnode = dnodeSendMsgToDnode;
-  para.SendMsgToMnode = dnodeSendMsgToMnode;
-  para.PutMsgIntoApplyQueue = dnodePutMsgIntoVnodeApplyQueue;
-
-  return vnodeInit(para);
-}
-
-int32_t dnodeInitVnodes() {
-  dInfo("dnode-vnodes start to init");
-
-  SSteps *pSteps = taosStepInit(6, dnodeReportStartup);
-  taosStepAdd(pSteps, "dnode-vnode-env", dnodeInitVnodeModule, vnodeCleanup);
-  taosStepAdd(pSteps, "dnode-vnode-mgmt", dnodeInitVnodeMgmtWorker, dnodeCleanupVnodeMgmtWorker);
-  taosStepAdd(pSteps, "dnode-vnode-read", dnodeInitVnodeReadWorker, dnodeCleanupVnodeReadWorker);
-  taosStepAdd(pSteps, "dnode-vnode-write", dnodeInitVnodeWriteWorker, dnodeCleanupVnodeWriteWorker);
-  taosStepAdd(pSteps, "dnode-vnode-sync", dnodeInitVnodeSyncWorker, dnodeCleanupVnodeSyncWorker);
-  taosStepAdd(pSteps, "dnode-vnodes", dnodeOpenVnodes, dnodeCleanupVnodes);
-
-  tsVnodes.pSteps = pSteps;
-  return taosStepExec(pSteps);
-}
-
-void dnodeCleanupVnodes() {
-  if (tsVnodes.pSteps != NULL) {
-    dInfo("dnode-vnodes start to clean up");
-    taosStepCleanup(tsVnodes.pSteps);
-    tsVnodes.pSteps = NULL;
-    dInfo("dnode-vnodes is cleaned up");
-  }
-}
-
-void dnodeGetVnodeLoads(SVnodeLoads *pLoads) {
-  pLoads->num = taosHashGetSize(tsVnodes.hash);
-
-  int32_t v = 0;
-  void *  pIter = taosHashIterate(tsVnodes.hash, NULL);
-  while (pIter) {
-    SVnodeObj **ppVnode = pIter;
-    if (ppVnode == NULL) continue;
-
-    SVnodeObj *pVnode = *ppVnode;
-    if (pVnode == NULL) continue;
-
-    SVnodeLoad *pLoad = &pLoads->data[v++];
-    vnodeGetLoad(pVnode->pImpl, pLoad);
-    pLoad->vgId = htonl(pLoad->vgId);
-    pLoad->totalStorage = htobe64(pLoad->totalStorage);
-    pLoad->compStorage = htobe64(pLoad->compStorage);
-    pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
-    pLoad->tablesNum = htobe64(pLoad->tablesNum);
-
-    pIter = taosHashIterate(tsVnodes.hash, pIter);
-  }
-}
diff --git a/source/dnode/mnode/CMakeLists.txt b/source/dnode/mnode/CMakeLists.txt
index 6de5f06476..45a5af7566 100644
--- a/source/dnode/mnode/CMakeLists.txt
+++ b/source/dnode/mnode/CMakeLists.txt
@@ -1,3 +1,2 @@
 add_subdirectory(impl)
 add_subdirectory(sdb)
-add_subdirectory(transaction)
diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt
index 4c9d44b39e..49c9d54513 100644
--- a/source/dnode/mnode/impl/CMakeLists.txt
+++ b/source/dnode/mnode/impl/CMakeLists.txt
@@ -8,7 +8,6 @@ target_include_directories(
 target_link_libraries(
     mnode
     PRIVATE sdb
-    PRIVATE transaction
     PUBLIC transport
     PUBLIC cjson
 )
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/inc/mnodeDef.h b/source/dnode/mnode/impl/inc/mnodeDef.h
index b6449ecfe7..4b4c4abdb3 100644
--- a/source/dnode/mnode/impl/inc/mnodeDef.h
+++ b/source/dnode/mnode/impl/inc/mnodeDef.h
@@ -76,6 +76,28 @@ typedef enum {
   MN_AUTH_MAX
 } EMnAuthOp;
 
+typedef enum {
+  TRN_STAGE_PREPARE = 1,
+  TRN_STAGE_EXECUTE = 2,
+  TRN_STAGE_COMMIT = 3,
+  TRN_STAGE_ROLLBACK = 4,
+  TRN_STAGE_RETRY = 5
+} ETrnStage;
+
+typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy;
+
+
+typedef struct STrans {
+  int32_t    id;
+  ETrnStage  stage;
+  ETrnPolicy policy;
+  void      *rpcHandle;
+  SArray    *redoLogs;
+  SArray    *undoLogs;
+  SArray    *commitLogs;
+  SArray    *redoActions;
+  SArray    *undoActions;
+} STrans;
 
 
 typedef struct SClusterObj {
@@ -109,7 +131,7 @@ typedef struct SMnodeObj {
   int64_t    roleTime;
   int64_t    createdTime;
   int64_t    updateTime;
-  SDnodeObj *pDnode;
+  SDnodeObj *pDnd;
 } SMnodeObj;
 
 typedef struct {
@@ -193,7 +215,7 @@ typedef struct SDbObj {
 typedef struct {
   int32_t    dnodeId;
   int8_t     role;
-  SDnodeObj *pDnode;
+  SDnodeObj *pDnd;
 } SVnodeGid;
 
 typedef struct SVgObj {
diff --git a/source/dnode/mnode/impl/inc/mnodeInt.h b/source/dnode/mnode/impl/inc/mnodeInt.h
index 96803ba4a5..43af281f27 100644
--- a/source/dnode/mnode/impl/inc/mnodeInt.h
+++ b/source/dnode/mnode/impl/inc/mnodeInt.h
@@ -18,21 +18,49 @@
 
 #include "mnodeDef.h"
 #include "sdb.h"
-#include "trn.h"
+#include "tstep.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef void (*MnodeRpcFp)(SMnodeMsg *pMsg);
+typedef int32_t (*MnodeRpcFp)(SMnodeMsg *pMsg);
+
+typedef struct SMnodeBak {
+  int32_t    dnodeId;
+  int64_t    clusterId;
+  tmr_h      timer;
+  SSteps    *pInitSteps;
+  SSteps    *pStartSteps;
+  SMnodeOptions para;
+  MnodeRpcFp msgFp[TSDB_MSG_TYPE_MAX];
+} SMnodeBak;
+
+typedef struct SMnode {
+  int32_t           dnodeId;
+  int64_t           clusterId;
+  int8_t            replica;
+  int8_t            selfIndex;
+  SReplica          replicas[TSDB_MAX_REPLICA];
+  tmr_h             timer;
+  SSteps           *pInitSteps;
+  SSteps           *pStartSteps;
+  struct SSdb      *pSdb;
+  struct SDnode    *pServer;
+  MnodeRpcFp        msgFp[TSDB_MSG_TYPE_MAX];
+  PutMsgToMnodeQFp  putMsgToApplyMsgFp;
+  SendMsgToDnodeFp  sendMsgToDnodeFp;
+  SendMsgToMnodeFp  sendMsgToMnodeFp;
+  SendRedirectMsgFp sendRedirectMsgFp;
+} SMnode;
 
 tmr_h   mnodeGetTimer();
 int32_t mnodeGetDnodeId();
 int64_t mnodeGetClusterId();
 
-void mnodeSendMsgToDnode(struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-void mnodeSendMsgToMnode(struct SRpcMsg *rpcMsg);
-void mnodeSendRedirectMsg(struct SRpcMsg *rpcMsg, bool forShell);
+void mnodeSendMsgToDnode(SMnode *pMnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
+void mnodeSendMsgToMnode(SMnode *pMnode, struct SRpcMsg *rpcMsg);
+void mnodeSendRedirectMsg(SMnode *pMnode, struct SRpcMsg *rpcMsg, bool forShell);
 
 void mnodeSetMsgFp(int32_t msgType, MnodeRpcFp fp);
 
diff --git a/include/dnode/mnode/transaction/trn.h b/source/dnode/mnode/impl/inc/mnodeTrans.h
similarity index 77%
rename from include/dnode/mnode/transaction/trn.h
rename to source/dnode/mnode/impl/inc/mnodeTrans.h
index 8ba043de12..2abe101dfd 100644
--- a/include/dnode/mnode/transaction/trn.h
+++ b/source/dnode/mnode/impl/inc/mnodeTrans.h
@@ -13,25 +13,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TRANSACTION_H_
-#define _TD_TRANSACTION_H_
+#ifndef _TD_TRANSACTION_INT_H_
+#define _TD_TRANSACTION_INT_H_
 
-#include "sdb.h"
-#include "taosmsg.h"
+#include "mnodeInt.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct STrans STrans;
-typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy;
+int32_t mnodeInitTrans();
+void    mnodeCleanupTrans();
 
-int32_t trnInit();
-void    trnCleanup();
-
-STrans *trnCreate(ETrnPolicy);
+STrans *trnCreate(ETrnPolicy policy, void *rpcHandle);
 void    trnDrop(STrans *pTrans);
-void    trnSetRpcHandle(STrans *pTrans, void *rpcHandle);
 int32_t trnAppendRedoLog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t trnAppendUndoLog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t trnAppendCommitLog(STrans *pTrans, SSdbRaw *pRaw);
@@ -42,8 +37,11 @@ int32_t trnPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)
 int32_t trnApply(SSdbRaw *pRaw, void *pData, int32_t code);
 int32_t trnExecute(int32_t tranId);
 
+SSdbRaw *trnActionEncode(STrans *pTrans);
+SSdbRow *trnActionDecode(SSdbRaw *pRaw);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TRANSACTION_H_*/
+#endif /*_TD_TRANSACTION_INT_H_*/
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index f52e60dbad..5eff8a37ca 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tglobal.h"
-#include "tstep.h"
 #include "tqueue.h"
 #include "mnodeAcct.h"
 #include "mnodeAuth.h"
@@ -34,26 +33,28 @@
 #include "mnodeTelem.h"
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
+#include "mnodeTrans.h"
 
-static struct {
-  int32_t    dnodeId;
-  int64_t    clusterId;
-  tmr_h      timer;
-  SSteps    *pInitSteps;
-  SSteps    *pStartSteps;
-  SMnodePara para;
-  MnodeRpcFp msgFp[TSDB_MSG_TYPE_MAX];
-} tsMint;
+SMnodeBak tsMint = {0};
 
 int32_t mnodeGetDnodeId() { return tsMint.para.dnodeId; }
 
 int64_t mnodeGetClusterId() { return tsMint.para.clusterId; }
 
-void mnodeSendMsgToDnode(struct SEpSet *epSet, struct SRpcMsg *rpcMsg) { (*tsMint.para.SendMsgToDnode)(epSet, rpcMsg); }
+void mnodeSendMsgToDnode(SMnode *pMnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg) {
+  assert(pMnode);
+  (*pMnode->sendMsgToDnodeFp)(pMnode->pServer, epSet, rpcMsg);
+}
 
-void mnodeSendMsgToMnode(struct SRpcMsg *rpcMsg) { return (*tsMint.para.SendMsgToMnode)(rpcMsg); }
+void mnodeSendMsgToMnode(SMnode *pMnode, struct SRpcMsg *rpcMsg) {
+  assert(pMnode);
+  (*pMnode->sendMsgToMnodeFp)(pMnode->pServer, rpcMsg);
+}
 
-void mnodeSendRedirectMsg(struct SRpcMsg *rpcMsg, bool forShell) { (*tsMint.para.SendRedirectMsg)(rpcMsg, forShell); }
+void mnodeSendRedirectMsg(SMnode *pMnode, struct SRpcMsg *rpcMsg, bool forShell) {
+  assert(pMnode);
+  (*pMnode->sendRedirectMsgFp)(pMnode->pServer, rpcMsg, forShell);
+}
 
 static int32_t mnodeInitTimer() {
   if (tsMint.timer == NULL) {
@@ -76,35 +77,20 @@ static void mnodeCleanupTimer() {
 
 tmr_h mnodeGetTimer() { return tsMint.timer; }
 
-static int32_t mnodeSetPara(SMnodePara para) {
-  tsMint.para = para;
+static int32_t mnodeSetOptions(SMnode *pMnode, const SMnodeOptions *pOptions) {
+  pMnode->dnodeId = pOptions->dnodeId;
+  pMnode->clusterId = pOptions->clusterId;
+  pMnode->replica = pOptions->replica;
+  pMnode->selfIndex = pOptions->selfIndex;
+  memcpy(&pMnode->replicas, pOptions->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+  pMnode->pServer = pOptions->pDnode;
+  pMnode->putMsgToApplyMsgFp = pOptions->putMsgToApplyMsgFp;
+  pMnode->sendMsgToDnodeFp = pOptions->sendMsgToDnodeFp;
+  pMnode->sendMsgToMnodeFp = pOptions->sendMsgToMnodeFp;
+  pMnode->sendRedirectMsgFp = pOptions->sendRedirectMsgFp;
 
-  if (tsMint.para.SendMsgToDnode == NULL) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return -1;
-  }
-
-  if (tsMint.para.SendMsgToMnode == NULL) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return -1;
-  }
-
-  if (tsMint.para.SendRedirectMsg == NULL) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return -1;
-  }
-
-  if (tsMint.para.PutMsgIntoApplyQueue == NULL) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return -1;
-  }
-
-  if (tsMint.para.dnodeId < 0) {
-    terrno = TSDB_CODE_MND_APP_ERROR;
-    return -1;
-  }
-
-  if (tsMint.para.clusterId < 0) {
+  if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL ||
+      pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) {
     terrno = TSDB_CODE_MND_APP_ERROR;
     return -1;
   }
@@ -116,7 +102,7 @@ static int32_t mnodeAllocInitSteps() {
   struct SSteps *steps = taosStepInit(16, NULL);
   if (steps == NULL) return -1;
 
-  if (taosStepAdd(steps, "mnode-trans", trnInit, trnCleanup) != 0) return -1;
+  if (taosStepAdd(steps, "mnode-trans", mnodeInitTrans, mnodeCleanupTrans) != 0) return -1;
   if (taosStepAdd(steps, "mnode-cluster", mnodeInitCluster, mnodeCleanupCluster) != 0) return -1;
   if (taosStepAdd(steps, "mnode-dnode", mnodeInitDnode, mnodeCleanupDnode) != 0) return -1;
   if (taosStepAdd(steps, "mnode-mnode", mnodeInitMnode, mnodeCleanupMnode) != 0) return -1;
@@ -150,50 +136,50 @@ static int32_t mnodeAllocStartSteps() {
   return 0;
 }
 
-int32_t mnodeInit(SMnodePara para) {
-  if (mnodeSetPara(para) != 0) {
-    mError("failed to init mnode para since %s", terrstr());
-    return -1;
+SMnode *mnodeOpen(const char *path, const SMnodeOptions *pOptions) {
+  SMnode *pMnode = calloc(1, sizeof(SMnode));
+
+  if (mnodeSetOptions(pMnode, pOptions) != 0) {
+    free(pMnode);
+    mError("failed to init mnode options since %s", terrstr());
+    return NULL;
   }
 
   if (mnodeAllocInitSteps() != 0) {
     mError("failed to alloc init steps since %s", terrstr());
-    return -1;
+    return NULL;
   }
 
   if (mnodeAllocStartSteps() != 0) {
     mError("failed to alloc start steps since %s", terrstr());
-    return -1;
+    return NULL;
   }
 
-  return taosStepExec(tsMint.pInitSteps);
-}
+  taosStepExec(tsMint.pInitSteps);
 
-void mnodeCleanup() { taosStepCleanup(tsMint.pInitSteps); }
-
-int32_t mnodeDeploy(SMnodeCfg *pCfg) {
   if (tsMint.para.dnodeId <= 0 && tsMint.para.clusterId <= 0) {
     if (sdbDeploy() != 0) {
       mError("failed to deploy sdb since %s", terrstr());
-      return -1;
+      return NULL;
+    } else {
+      mInfo("mnode is deployed");
     }
   }
 
-  mDebug("mnode is deployed");
-  return 0;
+  taosStepExec(tsMint.pStartSteps);
+
+  return pMnode;
 }
 
-void mnodeUnDeploy() { sdbUnDeploy(); }
+void mnodeClose(SMnode *pMnode) { free(pMnode); }
 
-int32_t mnodeStart(SMnodeCfg *pCfg) { return taosStepExec(tsMint.pStartSteps); }
+int32_t mnodeAlter(SMnode *pMnode, const SMnodeOptions *pOptions) { return 0; }
 
-int32_t mnodeAlter(SMnodeCfg *pCfg) { return 0; }
+void mnodeDestroy(const char *path) { sdbUnDeploy(); }
 
-void mnodeStop() { taosStepCleanup(tsMint.pStartSteps); }
+int32_t mnodeGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; }
 
-int32_t mnodeGetLoad(SMnodeLoad *pLoad) { return 0; }
-
-SMnodeMsg *mnodeInitMsg(SRpcMsg *pRpcMsg) {
+SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
   SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg));
   if (pMsg == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -221,13 +207,25 @@ void mnodeCleanupMsg(SMnodeMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
-static void mnodeProcessRpcMsg(SMnodeMsg *pMsg) {
-  int32_t msgType = pMsg->rpcMsg.msgType;
+void mnodeSendRsp(SMnodeMsg *pMsg, int32_t code) {}
 
-  if (tsMint.msgFp[msgType] == NULL) {
+static void mnodeProcessRpcMsg(SMnodeMsg *pMsg) {
+  if (!mnodeIsMaster()) {
+    mnodeSendRedirectMsg(NULL, &pMsg->rpcMsg, true);
+    mnodeCleanupMsg(pMsg);
+    return;
   }
 
-  (*tsMint.msgFp[msgType])(pMsg);
+  int32_t msgType = pMsg->rpcMsg.msgType;
+
+  MnodeRpcFp fp = tsMint.msgFp[msgType];
+  if (fp == NULL) {
+  }
+
+  int32_t code = (fp)(pMsg);
+  if (code != 0) {
+    assert(code);
+  }
 }
 
 void mnodeSetMsgFp(int32_t msgType, MnodeRpcFp fp) {
@@ -236,25 +234,13 @@ void mnodeSetMsgFp(int32_t msgType, MnodeRpcFp fp) {
   }
 }
 
-void mnodeProcessMsg(SMnodeMsg *pMsg, EMnMsgType msgType) {
-  if (!mnodeIsMaster()) {
-    mnodeSendRedirectMsg(&pMsg->rpcMsg, true);
-    mnodeCleanupMsg(pMsg);
-    return;
-  }
+void mnodeProcessReadMsg(SMnode *pMnode, SMnodeMsg *pMsg) { mnodeProcessRpcMsg(pMsg); }
 
-  switch (msgType) {
-    case MN_MSG_TYPE_READ:
-    case MN_MSG_TYPE_WRITE:
-    case MN_MSG_TYPE_SYNC:
-      mnodeProcessRpcMsg(pMsg);
-      break;
-    case MN_MSG_TYPE_APPLY:
-      break;
-    default:
-      break;
-  }
-}
+void mnodeProcessWriteMsg(SMnode *pMnode, SMnodeMsg *pMsg) { mnodeProcessRpcMsg(pMsg); }
+
+void mnodeProcessSyncMsg(SMnode *pMnode, SMnodeMsg *pMsg) { mnodeProcessRpcMsg(pMsg); }
+
+void mnodeProcessApplyMsg(SMnode *pMnode, SMnodeMsg *pMsg) {}
 
 #if 0
 
diff --git a/source/dnode/mnode/impl/src/mnodeAuth.c b/source/dnode/mnode/impl/src/mnodeAuth.c
index bb3289ebeb..ddd2b91ff3 100644
--- a/source/dnode/mnode/impl/src/mnodeAuth.c
+++ b/source/dnode/mnode/impl/src/mnodeAuth.c
@@ -20,7 +20,7 @@
 int32_t mnodeInitAuth() { return 0; }
 void    mnodeCleanupAuth() {}
 
-int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+int32_t mnodeRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
   if (strcmp(user, TSDB_NETTEST_USER) == 0) {
     char pass[32] = {0};
     taosEncryptPass((uint8_t *)user, strlen(user), pass);
diff --git a/source/dnode/mnode/impl/src/mnodeSync.c b/source/dnode/mnode/impl/src/mnodeSync.c
index fd34793172..6e4084ffa6 100644
--- a/source/dnode/mnode/impl/src/mnodeSync.c
+++ b/source/dnode/mnode/impl/src/mnodeSync.c
@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "mnodeInt.h"
+#include "mnodeTrans.h"
 
 int32_t mnodeInitSync() { return 0; }
 void    mnodeCleanUpSync() {}
diff --git a/source/dnode/mnode/impl/src/mnodeTelem.c b/source/dnode/mnode/impl/src/mnodeTelem.c
index ef1ac10eb6..206b94a6c7 100644
--- a/source/dnode/mnode/impl/src/mnodeTelem.c
+++ b/source/dnode/mnode/impl/src/mnodeTelem.c
@@ -174,7 +174,7 @@ static void mnodeAddVersionInfo(SBufferWriter* bw) {
 
 static void mnodeAddRuntimeInfo(SBufferWriter* bw) {
   SMnodeLoad load = {0};
-  if (mnodeGetLoad(&load) != 0) {
+  if (mnodeGetLoad(NULL, &load) != 0) {
     return;
   }
 
diff --git a/source/dnode/mnode/impl/src/mnodeTrans.c b/source/dnode/mnode/impl/src/mnodeTrans.c
new file mode 100644
index 0000000000..4cd6bf7bdb
--- /dev/null
+++ b/source/dnode/mnode/impl/src/mnodeTrans.c
@@ -0,0 +1,502 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mnodeTrans.h"
+#include "trpc.h"
+
+#define SDB_TRANS_VER 1
+#define TRN_DEFAULT_ARRAY_SIZE 8
+
+SSdbRaw *trnActionEncode(STrans *pTrans) {
+  int32_t rawDataLen = 10 * sizeof(int32_t);
+  int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs);
+  int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs);
+  int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs);
+  int32_t redoActionNum = taosArrayGetSize(pTrans->redoActions);
+  int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions);
+
+  for (int32_t index = 0; index < redoLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, index);
+    rawDataLen += sdbGetRawTotalSize(pTmp);
+  }
+
+  for (int32_t index = 0; index < undoLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, index);
+    rawDataLen += sdbGetRawTotalSize(pTmp);
+  }
+
+  for (int32_t index = 0; index < commitLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, index);
+    rawDataLen += sdbGetRawTotalSize(pTmp);
+  }
+
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen);
+  if (pRaw == NULL) {
+    mError("trn:%d, failed to alloc raw since %s", pTrans->id, terrstr());
+    return NULL;
+  }
+
+  int32_t dataPos = 0;
+  SDB_SET_INT32(pRaw, dataPos, pTrans->id)
+  SDB_SET_INT8(pRaw, dataPos, pTrans->stage)
+  SDB_SET_INT8(pRaw, dataPos, pTrans->policy)
+  SDB_SET_INT32(pRaw, dataPos, redoLogNum)
+  SDB_SET_INT32(pRaw, dataPos, undoLogNum)
+  SDB_SET_INT32(pRaw, dataPos, commitLogNum)
+  SDB_SET_INT32(pRaw, dataPos, redoActionNum)
+  SDB_SET_INT32(pRaw, dataPos, undoActionNum)
+
+  for (int32_t index = 0; index < redoLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, index);
+    int32_t  len = sdbGetRawTotalSize(pTmp);
+    SDB_SET_INT32(pRaw, dataPos, len)
+    SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
+  }
+
+  for (int32_t index = 0; index < undoLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, index);
+    int32_t  len = sdbGetRawTotalSize(pTmp);
+    SDB_SET_INT32(pRaw, dataPos, len)
+    SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
+  }
+
+  for (int32_t index = 0; index < commitLogNum; ++index) {
+    SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, index);
+    int32_t  len = sdbGetRawTotalSize(pTmp);
+    SDB_SET_INT32(pRaw, dataPos, len)
+    SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
+  }
+
+  mDebug("trn:%d, is encoded as raw:%p, len:%d", pTrans->id, pRaw, dataPos);
+  return pRaw;
+}
+
+SSdbRow *trnActionDecode(SSdbRaw *pRaw) {
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
+    mError("failed to get soft ver from raw:%p since %s", pRaw, terrstr());
+    return NULL;
+  }
+
+  if (sver != SDB_TRANS_VER) {
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr());
+    return NULL;
+  }
+
+  SSdbRow   *pRow = sdbAllocRow(sizeof(STrans));
+  STrans *pTrans = sdbGetRowObj(pRow);
+  if (pTrans == NULL) {
+    mError("failed to alloc trans from raw:%p since %s", pRaw, terrstr());
+    return NULL;
+  }
+
+  pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+
+  if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
+      pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mDebug("trn:%d, failed to create array while parsed from raw:%p", pTrans->id, pRaw);
+    return NULL;
+  }
+
+  int32_t redoLogNum = 0;
+  int32_t undoLogNum = 0;
+  int32_t commitLogNum = 0;
+  int32_t redoActionNum = 0;
+  int32_t undoActionNum = 0;
+
+  int32_t dataPos = 0;
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pTrans->id)
+  SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->stage)
+  SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->policy)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &redoLogNum)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &undoLogNum)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &commitLogNum)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &redoActionNum)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &undoActionNum)
+
+  int32_t code = 0;
+  for (int32_t index = 0; index < redoLogNum; ++index) {
+    int32_t dataLen = 0;
+    SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
+
+    char *pData = malloc(dataLen);
+    SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
+    void *ret = taosArrayPush(pTrans->redoLogs, pData);
+    if (ret == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      break;
+    }
+  }
+
+  if (code != 0) {
+    terrno = code;
+    mError("trn:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr());
+    trnDrop(pTrans);
+    return NULL;
+  }
+
+  mDebug("trn:%d, is parsed from raw:%p", pTrans->id, pRaw);
+  return pRow;
+}
+
+static int32_t trnActionInsert(STrans *pTrans) {
+  SArray *pArray = pTrans->redoLogs;
+  int32_t arraySize = taosArrayGetSize(pArray);
+
+  for (int32_t index = 0; index < arraySize; ++index) {
+    SSdbRaw *pRaw = taosArrayGet(pArray, index);
+    int32_t  code = sdbWrite(pRaw);
+    if (code != 0) {
+      mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr());
+      return code;
+    }
+  }
+
+  mDebug("trn:%d, write to sdb", pTrans->id);
+  return 0;
+}
+
+static int32_t trnActionDelete(STrans *pTrans) {
+  SArray *pArray = pTrans->redoLogs;
+  int32_t arraySize = taosArrayGetSize(pArray);
+
+  for (int32_t index = 0; index < arraySize; ++index) {
+    SSdbRaw *pRaw = taosArrayGet(pArray, index);
+    int32_t  code = sdbWrite(pRaw);
+    if (code != 0) {
+      mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr());
+      return code;
+    }
+  }
+
+  mDebug("trn:%d, delete from sdb", pTrans->id);
+  return 0;
+}
+
+static int32_t trnActionUpdate(STrans *pTrans, STrans *pDstTrans) {
+  assert(true);
+  SArray *pArray = pTrans->redoLogs;
+  int32_t arraySize = taosArrayGetSize(pArray);
+
+  for (int32_t index = 0; index < arraySize; ++index) {
+    SSdbRaw *pRaw = taosArrayGet(pArray, index);
+    int32_t  code = sdbWrite(pRaw);
+    if (code != 0) {
+      mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr());
+      return code;
+    }
+  }
+
+  pTrans->stage = pDstTrans->stage;
+  mDebug("trn:%d, update in sdb", pTrans->id);
+  return 0;
+}
+
+static int32_t trnGenerateTransId() { return 1; }
+
+STrans *trnCreate(ETrnPolicy policy, void *rpcHandle) {
+  STrans *pTrans = calloc(1, sizeof(STrans));
+  if (pTrans == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mError("failed to create transaction since %s", terrstr());
+    return NULL;
+  }
+
+  pTrans->id = trnGenerateTransId();
+  pTrans->stage = TRN_STAGE_PREPARE;
+  pTrans->policy = policy;
+  pTrans->rpcHandle = rpcHandle;
+  pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+  pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
+
+  if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
+      pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mError("failed to create transaction since %s", terrstr());
+    return NULL;
+  }
+
+  mDebug("trn:%d, is created, %p", pTrans->id, pTrans);
+  return pTrans;
+}
+
+static void trnDropArray(SArray *pArray) {
+  for (int32_t index = 0; index < pArray->size; ++index) {
+    SSdbRaw *pRaw = taosArrayGet(pArray, index);
+    tfree(pRaw);
+  }
+
+  taosArrayDestroy(pArray);
+}
+
+void trnDrop(STrans *pTrans) {
+  trnDropArray(pTrans->redoLogs);
+  trnDropArray(pTrans->undoLogs);
+  trnDropArray(pTrans->commitLogs);
+  trnDropArray(pTrans->redoActions);
+  trnDropArray(pTrans->undoActions);
+
+  mDebug("trn:%d, is dropped, %p", pTrans->id, pTrans);
+  tfree(pTrans);
+}
+
+void trnSetRpcHandle(STrans *pTrans, void *rpcHandle) {
+  pTrans->rpcHandle = rpcHandle;
+  mTrace("trn:%d, set rpc handle:%p", pTrans->id, rpcHandle);
+}
+
+static int32_t trnAppendArray(SArray *pArray, SSdbRaw *pRaw) {
+  if (pArray == NULL || pRaw == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  void *ptr = taosArrayPush(pArray, pRaw);
+  if (ptr == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t trnAppendRedoLog(STrans *pTrans, SSdbRaw *pRaw) {
+  int32_t code = trnAppendArray(pTrans->redoLogs, pRaw);
+  mTrace("trn:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code);
+  return code;
+}
+
+int32_t trnAppendUndoLog(STrans *pTrans, SSdbRaw *pRaw) {
+  int32_t code = trnAppendArray(pTrans->undoLogs, pRaw);
+  mTrace("trn:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code);
+  return code;
+}
+
+int32_t trnAppendCommitLog(STrans *pTrans, SSdbRaw *pRaw) {
+  int32_t code = trnAppendArray(pTrans->commitLogs, pRaw);
+  mTrace("trn:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code);
+  return code;
+}
+
+int32_t trnAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
+  int32_t code = trnAppendArray(pTrans->redoActions, pMsg);
+  mTrace("trn:%d, msg:%p append to redo actions", pTrans->id, pMsg);
+  return code;
+}
+
+int32_t trnAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
+  int32_t code = trnAppendArray(pTrans->undoActions, pMsg);
+  mTrace("trn:%d, msg:%p append to undo actions", pTrans->id, pMsg);
+  return code;
+}
+
+int32_t mnodeInitTrans() {
+  SSdbTable table = {.sdbType = SDB_TRANS,
+                     .keyType = SDB_KEY_INT32,
+                     .encodeFp = (SdbEncodeFp)trnActionEncode,
+                     .decodeFp = (SdbDecodeFp)trnActionDecode,
+                     .insertFp = (SdbInsertFp)trnActionInsert,
+                     .updateFp = (SdbUpdateFp)trnActionUpdate,
+                     .deleteFp = (SdbDeleteFp)trnActionDelete};
+  sdbSetTable(table);
+
+  mInfo("trn module is initialized");
+  return 0;
+}
+
+void mnodeCleanupTrans() { mInfo("trn module is cleaned up"); }
+
+
+int32_t trnPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)) {
+  if (syncfp == NULL) return -1;
+
+  SSdbRaw *pRaw = trnActionEncode(pTrans);
+  if (pRaw == NULL) {
+    mError("trn:%d, failed to decode trans since %s", pTrans->id, terrstr());
+    return -1;
+  }
+  sdbSetRawStatus(pRaw, SDB_STATUS_CREATING);
+
+  if (sdbWrite(pRaw) != 0) {
+    mError("trn:%d, failed to write trans since %s", pTrans->id, terrstr());
+    return -1;
+  }
+
+  if ((*syncfp)(pRaw, pTrans->rpcHandle) != 0) {
+    mError("trn:%d, failed to sync trans since %s", pTrans->id, terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void trnSendRpcRsp(void *rpcHandle, int32_t code) {
+  if (rpcHandle != NULL) {
+    SRpcMsg rspMsg = {.handle = rpcHandle, .code = terrno};
+    rpcSendResponse(&rspMsg);
+  }
+}
+
+int32_t trnApply(SSdbRaw *pRaw, void *pData, int32_t code) {
+  if (code != 0) {
+    trnSendRpcRsp(pData, terrno);
+    return 0;
+  }
+
+  if (sdbWrite(pData) != 0) {
+    code = terrno;
+    trnSendRpcRsp(pData, code);
+    terrno = code;
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t trnExecuteArray(SArray *pArray) {
+  for (int32_t index = 0; index < pArray->size; ++index) {
+    SSdbRaw *pRaw = taosArrayGetP(pArray, index);
+    if (sdbWrite(pRaw) != 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int32_t trnExecuteRedoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->redoLogs); }
+
+static int32_t trnExecuteUndoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->undoLogs); }
+
+static int32_t trnExecuteCommitLogs(STrans *pTrans) { return trnExecuteArray(pTrans->commitLogs); }
+
+static int32_t trnExecuteRedoActions(STrans *pTrans) { return trnExecuteArray(pTrans->redoActions); }
+
+static int32_t trnExecuteUndoActions(STrans *pTrans) { return trnExecuteArray(pTrans->undoActions); }
+
+static int32_t trnPerformPrepareStage(STrans *pTrans) {
+  if (trnExecuteRedoLogs(pTrans) == 0) {
+    pTrans->stage = TRN_STAGE_EXECUTE;
+    return 0;
+  } else {
+    pTrans->stage = TRN_STAGE_ROLLBACK;
+    return -1;
+  }
+}
+
+static int32_t trnPerformExecuteStage(STrans *pTrans) {
+  int32_t code = trnExecuteRedoActions(pTrans);
+
+  if (code == 0) {
+    pTrans->stage = TRN_STAGE_COMMIT;
+    return 0;
+  } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    return -1;
+  } else {
+    if (pTrans->policy == TRN_POLICY_RETRY) {
+      pTrans->stage = TRN_STAGE_RETRY;
+    } else {
+      pTrans->stage = TRN_STAGE_ROLLBACK;
+    }
+    return 0;
+  }
+}
+
+static int32_t trnPerformCommitStage(STrans *pTrans) {
+  if (trnExecuteCommitLogs(pTrans) == 0) {
+    pTrans->stage = TRN_STAGE_EXECUTE;
+    return 0;
+  } else {
+    pTrans->stage = TRN_STAGE_ROLLBACK;
+    return -1;
+  }
+}
+
+static int32_t trnPerformRollbackStage(STrans *pTrans) {
+  if (trnExecuteCommitLogs(pTrans) == 0) {
+    pTrans->stage = TRN_STAGE_EXECUTE;
+    return 0;
+  } else {
+    pTrans->stage = TRN_STAGE_ROLLBACK;
+    return -1;
+  }
+}
+
+static int32_t trnPerformRetryStage(STrans *pTrans) {
+  if (trnExecuteCommitLogs(pTrans) == 0) {
+    pTrans->stage = TRN_STAGE_EXECUTE;
+    return 0;
+  } else {
+    pTrans->stage = TRN_STAGE_ROLLBACK;
+    return -1;
+  }
+}
+
+int32_t trnExecute(int32_t tranId) {
+  int32_t code = 0;
+
+  STrans *pTrans = sdbAcquire(SDB_TRANS, &tranId);
+  if (pTrans == NULL) {
+    return -1;
+  }
+
+  if (pTrans->stage == TRN_STAGE_PREPARE) {
+    if (trnPerformPrepareStage(pTrans) != 0) {
+      sdbRelease(pTrans);
+      return -1;
+    }
+  }
+
+  if (pTrans->stage == TRN_STAGE_EXECUTE) {
+    if (trnPerformExecuteStage(pTrans) != 0) {
+      sdbRelease(pTrans);
+      return -1;
+    }
+  }
+
+  if (pTrans->stage == TRN_STAGE_COMMIT) {
+    if (trnPerformCommitStage(pTrans) != 0) {
+      sdbRelease(pTrans);
+      return -1;
+    }
+  }
+
+  if (pTrans->stage == TRN_STAGE_ROLLBACK) {
+    if (trnPerformRollbackStage(pTrans) != 0) {
+      sdbRelease(pTrans);
+      return -1;
+    }
+  }
+
+  if (pTrans->stage == TRN_STAGE_RETRY) {
+    if (trnPerformRetryStage(pTrans) != 0) {
+      sdbRelease(pTrans);
+      return -1;
+    }
+  }
+
+  sdbRelease(pTrans);
+  return 0;
+}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mnodeUser.c b/source/dnode/mnode/impl/src/mnodeUser.c
index 63aa171238..b8e5706484 100644
--- a/source/dnode/mnode/impl/src/mnodeUser.c
+++ b/source/dnode/mnode/impl/src/mnodeUser.c
@@ -18,6 +18,7 @@
 #include "os.h"
 #include "tglobal.h"
 #include "tkey.h"
+#include "mnodeTrans.h"
 
 #define SDB_USER_VER 1
 
@@ -142,12 +143,12 @@ static int32_t mnodeCreateUser(char *acct, char *user, char *pass, SMnodeMsg *pM
   userObj.updateTime = userObj.createdTime;
   userObj.rootAuth = 0;
 
-  STrans *pTrans = trnCreate(TRN_POLICY_ROLLBACK);
+  STrans *pTrans = trnCreate(TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
   if (pTrans == NULL) return -1;
-  trnSetRpcHandle(pTrans, pMsg->rpcMsg.handle);
 
   SSdbRaw *pRedoRaw = mnodeUserActionEncode(&userObj);
   if (pRedoRaw == NULL || trnAppendRedoLog(pTrans, pRedoRaw) != 0) {
+    mError("failed to append redo log since %s", terrstr());
     trnDrop(pTrans);
     return -1;
   }
@@ -155,6 +156,7 @@ static int32_t mnodeCreateUser(char *acct, char *user, char *pass, SMnodeMsg *pM
 
   SSdbRaw *pUndoRaw = mnodeUserActionEncode(&userObj);
   if (pUndoRaw == NULL || trnAppendUndoLog(pTrans, pUndoRaw) != 0) {
+    mError("failed to append undo log since %s", terrstr());
     trnDrop(pTrans);
     return -1;
   }
@@ -162,6 +164,7 @@ static int32_t mnodeCreateUser(char *acct, char *user, char *pass, SMnodeMsg *pM
 
   SSdbRaw *pCommitRaw = mnodeUserActionEncode(&userObj);
   if (pCommitRaw == NULL || trnAppendCommitLog(pTrans, pCommitRaw) != 0) {
+    mError("failed to append commit log since %s", terrstr());
     trnDrop(pTrans);
     return -1;
   }
@@ -228,6 +231,8 @@ int32_t mnodeInitUser() {
                      .deleteFp = (SdbDeleteFp)mnodeUserActionDelete};
   sdbSetTable(table);
 
+  mnodeSetMsgFp(TSDB_MSG_TYPE_CREATE_USER, mnodeProcessCreateUserMsg);
+
   return 0;
 }
 
diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h
index aa0b3c8a58..2b3c577ba9 100644
--- a/source/dnode/mnode/sdb/inc/sdbInt.h
+++ b/source/dnode/mnode/sdb/inc/sdbInt.h
@@ -52,7 +52,7 @@ typedef struct SSdbRow {
   char       pObj[];
 } SSdbRow;
 
-typedef struct {
+typedef struct SSdb {
   char       *currDir;
   char       *syncDir;
   char       *tmpDir;
@@ -67,9 +67,9 @@ typedef struct {
   SdbDeployFp deployFps[SDB_MAX];
   SdbEncodeFp encodeFps[SDB_MAX];
   SdbDecodeFp decodeFps[SDB_MAX];
-} SSdbMgr;
+} SSdb;
 
-extern SSdbMgr tsSdb;
+extern SSdb tsSdb;
 
 int32_t sdbWriteImp(SSdbRaw *pRaw);
 
diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c
index 8e9b7fbecc..83496f3794 100644
--- a/source/dnode/mnode/sdb/src/sdb.c
+++ b/source/dnode/mnode/sdb/src/sdb.c
@@ -17,7 +17,7 @@
 #include "sdbInt.h"
 #include "tglobal.h"
 
-SSdbMgr tsSdb = {0};
+SSdb tsSdb = {0};
 
 int32_t sdbInit() {
   char path[PATH_MAX + 100];
diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c
index 69c82c77f1..d77ec6ef8f 100644
--- a/source/dnode/mnode/sdb/src/sdbFile.c
+++ b/source/dnode/mnode/sdb/src/sdbFile.c
@@ -21,19 +21,19 @@
 static int32_t sdbCreateDir() {
   mDebug("start to create mnode at %s", tsMnodeDir);
   
-  if (!taosMkDir(tsSdb.currDir)) {
+  if (taosMkDir(tsSdb.currDir) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to create dir:%s since %s", tsSdb.currDir, terrstr());
     return -1;
   }
 
-  if (!taosMkDir(tsSdb.syncDir)) {
+  if (taosMkDir(tsSdb.syncDir) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to create dir:%s since %s", tsSdb.syncDir, terrstr());
     return -1;
   }
 
-  if (!taosMkDir(tsSdb.tmpDir)) {
+  if (taosMkDir(tsSdb.tmpDir) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to create dir:%s since %s", tsSdb.tmpDir, terrstr());
     return -1;
@@ -45,7 +45,7 @@ static int32_t sdbCreateDir() {
 static int32_t sdbRunDeployFp() {
   mDebug("start to run deploy functions");
 
-  for (int32_t i = SDB_START; i < SDB_MAX; ++i) {
+  for (int32_t i = SDB_MAX - 1; i > SDB_START; --i) {
     SdbDeployFp fp = tsSdb.deployFps[i];
     if (fp == NULL) continue;
     if ((*fp)() != 0) {
@@ -54,6 +54,7 @@ static int32_t sdbRunDeployFp() {
     }
   }
 
+  mDebug("end of run deploy functions");
   return 0;
 }
 
diff --git a/source/dnode/mnode/transaction/CMakeLists.txt b/source/dnode/mnode/transaction/CMakeLists.txt
deleted file mode 100644
index d35a8c9b3f..0000000000
--- a/source/dnode/mnode/transaction/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-aux_source_directory(src MNODE_SRC)
-add_library(transaction ${MNODE_SRC})
-target_include_directories(
-    transaction
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode/transaction"
-    private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-target_link_libraries(
-    transaction
-    PRIVATE os
-    PRIVATE common
-    PRIVATE util
-    PRIVATE sdb
-    PRIVATE transport
-)
diff --git a/source/dnode/mnode/transaction/inc/trnInt.h b/source/dnode/mnode/transaction/inc/trnInt.h
deleted file mode 100644
index 771217dcc0..0000000000
--- a/source/dnode/mnode/transaction/inc/trnInt.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TRANSACTION_INT_H_
-#define _TD_TRANSACTION_INT_H_
-
-#include "os.h"
-#include "trn.h"
-#include "tglobal.h"
-#include "tarray.h"
-#include "tlog.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }}
-#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }}
-#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }}
-#define mInfo(...)  { if (mDebugFlag & DEBUG_INFO)  { taosPrintLog("MND ", 255, __VA_ARGS__); }}
-#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
-#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
-
-#define TRN_VER 1
-#define TRN_DEFAULT_ARRAY_SIZE 8
-
-typedef enum {
-  TRN_STAGE_PREPARE = 1,
-  TRN_STAGE_EXECUTE = 2,
-  TRN_STAGE_COMMIT = 3,
-  TRN_STAGE_ROLLBACK = 4,
-  TRN_STAGE_RETRY = 5
-} ETrnStage;
-
-typedef struct STrans {
-  int32_t id;
-  int8_t  stage;
-  int8_t  policy;
-  void   *rpcHandle;
-  SArray *redoLogs;
-  SArray *undoLogs;
-  SArray *commitLogs;
-  SArray *redoActions;
-  SArray *undoActions;
-} STrans;
-
-SSdbRaw *trnActionEncode(STrans *pTrans);
-STrans  *trnActionDecode(SSdbRaw *pRaw);
-int32_t  trnActionInsert(STrans *pTrans);
-int32_t  trnActionDelete(STrans *pTrans);
-int32_t  trnActionUpdate(STrans *pSrcTrans, STrans *pDstTrans);
-int32_t  trnGenerateTransId();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TRANSACTION_INT_H_*/
diff --git a/source/dnode/mnode/transaction/src/trn.c b/source/dnode/mnode/transaction/src/trn.c
deleted file mode 100644
index 0d7c1a061e..0000000000
--- a/source/dnode/mnode/transaction/src/trn.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "trnInt.h"
-
-#define SDB_TRANS_VER 1
-
-SSdbRaw *trnActionEncode(STrans *pTrans) {
-  int32_t rawDataLen = 10 * sizeof(int32_t);
-  int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs);
-  int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs);
-  int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs);
-  int32_t redoActionNum = taosArrayGetSize(pTrans->redoActions);
-  int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions);
-
-  for (int32_t index = 0; index < redoLogNum; ++index) {
-    SSdbRaw *pRaw = taosArrayGet(pTrans->redoLogs, index);
-    rawDataLen += sdbGetRawTotalSize(pRaw);
-  }
-
-  for (int32_t index = 0; index < undoLogNum; ++index) {
-    SSdbRaw *pRaw = taosArrayGet(pTrans->undoLogs, index);
-    rawDataLen += sdbGetRawTotalSize(pRaw);
-  }
-
-  for (int32_t index = 0; index < commitLogNum; ++index) {
-    SSdbRaw *pRaw = taosArrayGet(pTrans->commitLogs, index);
-    rawDataLen += sdbGetRawTotalSize(pRaw);
-  }
-
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen);
-  if (pRaw == NULL) return NULL;
-
-  int32_t dataPos = 0;
-  SDB_SET_INT32(pData, dataPos, pTrans->id)
-  SDB_SET_INT8(pData, dataPos, pTrans->stage)
-  SDB_SET_INT8(pData, dataPos, pTrans->policy)
-  SDB_SET_INT32(pData, dataPos, redoLogNum)
-  SDB_SET_INT32(pData, dataPos, undoLogNum)
-  SDB_SET_INT32(pData, dataPos, commitLogNum)
-  SDB_SET_INT32(pData, dataPos, redoActionNum)
-  SDB_SET_INT32(pData, dataPos, undoActionNum)
-  SDB_SET_DATALEN(pRaw, dataPos);
-
-  return pRaw;
-}
-
-STrans *trnActionDecode(SSdbRaw *pRaw) {
-  int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
-
-  if (sver != SDB_TRANS_VER) {
-    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
-    return NULL;
-  }
-
-  SSdbRow *pRow = sdbAllocRow(sizeof(STrans));
-  STrans  *pTrans = sdbGetRowObj(pRow);
-  if (pTrans == NULL) return NULL;
-
-  int32_t  redoLogNum = 0;
-  int32_t  undoLogNum = 0;
-  int32_t  commitLogNum = 0;
-  int32_t  redoActionNum = 0;
-  int32_t  undoActionNum = 0;
-
-  int32_t dataPos = 0;
-  SDB_GET_INT32(pRaw, pRow, dataPos, &pTrans->id)
-  SDB_GET_INT8(pRaw, pRow, dataPos, &pTrans->stage)
-  SDB_GET_INT8(pRaw, pRow, dataPos, &pTrans->policy)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &redoLogNum)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &undoLogNum)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &commitLogNum)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &redoActionNum)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &undoActionNum)
-
-  for (int32_t index = 0; index < redoLogNum; ++index) {
-    int32_t dataLen = 0;
-    SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
-
-    char *pData = malloc(dataLen);
-    SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
-    void *ret = taosArrayPush(pTrans->redoLogs, pData);
-    if (ret == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      break;
-    }
-  }
-
-  // if (code != 0) {
-  //   trnDrop(pTrans);
-  //   terrno = code;
-  //   return NULL;
-  // }
-
-  return pTrans;
-}
-
-int32_t trnActionInsert(STrans *pTrans) {
-  SArray *pArray = pTrans->redoLogs;
-  int32_t arraySize = taosArrayGetSize(pArray);
-
-  for (int32_t index = 0; index < arraySize; ++index) {
-    SSdbRaw *pRaw = taosArrayGetP(pArray, index);
-    int32_t  code = sdbWrite(pRaw);
-    if (code != 0) {
-      return code;
-    }
-  }
-
-  return 0;
-}
-
-int32_t trnActionDelete(STrans *pTrans) {
-  SArray *pArray = pTrans->redoLogs;
-  int32_t arraySize = taosArrayGetSize(pArray);
-
-  for (int32_t index = 0; index < arraySize; ++index) {
-    SSdbRaw *pRaw = taosArrayGetP(pArray, index);
-    int32_t  code = sdbWrite(pRaw);
-    if (code != 0) {
-      return code;
-    }
-  }
-
-  return 0;
-}
-
-int32_t trnActionUpdate(STrans *pSrcTrans, STrans *pDstTrans) { return 0; }
-
-int32_t trnGenerateTransId() { return 1; }
-
-STrans *trnCreate(ETrnPolicy policy) {
-  STrans *pTrans = calloc(1, sizeof(STrans));
-  if (pTrans == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  pTrans->id = trnGenerateTransId();
-  pTrans->stage = TRN_STAGE_PREPARE;
-  pTrans->policy = policy;
-  pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
-  pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
-  pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
-  pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
-  pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *));
-
-  if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
-      pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  return pTrans;
-}
-
-static void trnDropArray(SArray *pArray) {
-  for (int32_t index = 0; index < pArray->size; ++index) {
-    SSdbRaw *pRaw = taosArrayGetP(pArray, index);
-    tfree(pRaw);
-  }
-
-  taosArrayDestroy(pArray);
-}
-
-void trnDrop(STrans *pTrans) {
-  trnDropArray(pTrans->redoLogs);
-  trnDropArray(pTrans->undoLogs);
-  trnDropArray(pTrans->commitLogs);
-  trnDropArray(pTrans->redoActions);
-  trnDropArray(pTrans->undoActions);
-  tfree(pTrans);
-}
-
-void trnSetRpcHandle(STrans *pTrans, void *rpcHandle) { pTrans->rpcHandle = rpcHandle; }
-
-static int32_t trnAppendArray(SArray *pArray, SSdbRaw *pRaw) {
-  if (pArray == NULL || pRaw == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  void *ptr = taosArrayPush(pArray, &pRaw);
-  if (ptr == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t trnAppendRedoLog(STrans *pTrans, SSdbRaw *pRaw) { return trnAppendArray(pTrans->redoLogs, pRaw); }
-
-int32_t trnAppendUndoLog(STrans *pTrans, SSdbRaw *pRaw) { return trnAppendArray(pTrans->undoLogs, pRaw); }
-
-int32_t trnAppendCommitLog(STrans *pTrans, SSdbRaw *pRaw) { return trnAppendArray(pTrans->commitLogs, pRaw); }
-
-int32_t trnAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
-  return trnAppendArray(pTrans->redoActions, pMsg);
-}
-
-int32_t trnAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
-  return trnAppendArray(pTrans->undoActions, pMsg);
-}
-
-int32_t trnInit() {
-  SSdbTable table = {.sdbType = SDB_TRANS,
-                     .keyType = SDB_KEY_INT32,
-                     .encodeFp = (SdbEncodeFp)trnActionEncode,
-                     .decodeFp = (SdbDecodeFp)trnActionDecode,
-                     .insertFp = (SdbInsertFp)trnActionInsert,
-                     .updateFp = (SdbUpdateFp)trnActionUpdate,
-                     .deleteFp = (SdbDeleteFp)trnActionDelete};
-  sdbSetTable(table);
-
-  return 0;
-}
-
-void trnCleanup() {}
diff --git a/source/dnode/mnode/transaction/src/trnExec.c b/source/dnode/mnode/transaction/src/trnExec.c
deleted file mode 100644
index fc15c16225..0000000000
--- a/source/dnode/mnode/transaction/src/trnExec.c
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "trnInt.h"
-#include "trpc.h"
-
-int32_t trnPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)) {
-  if (syncfp == NULL) return -1;
-
-  SSdbRaw *pRaw = trnActionEncode(pTrans);
-  if (pRaw == NULL) {
-    mError("tranId:%d, failed to decode trans since %s", pTrans->id, terrstr());
-    return -1;
-  }
-
-  if (sdbWrite(pRaw) != 0) {
-    mError("tranId:%d, failed to write trans since %s", pTrans->id, terrstr());
-    return -1;
-  }
-
-  if ((*syncfp)(pRaw, pTrans->rpcHandle) != 0) {
-    mError("tranId:%d, failed to sync trans since %s", pTrans->id, terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void trnSendRpcRsp(void *rpcHandle, int32_t code) {
-  if (rpcHandle != NULL) {
-    SRpcMsg rspMsg = {.handle = rpcHandle, .code = terrno};
-    rpcSendResponse(&rspMsg);
-  }
-}
-
-int32_t trnApply(SSdbRaw *pRaw, void *pData, int32_t code) {
-  if (code != 0) {
-    trnSendRpcRsp(pData, terrno);
-    return 0;
-  }
-
-  if (sdbWrite(pData) != 0) {
-    code = terrno;
-    trnSendRpcRsp(pData, code);
-    terrno = code;
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t trnExecuteArray(SArray *pArray) {
-  for (int32_t index = 0; index < pArray->size; ++index) {
-    SSdbRaw *pRaw = taosArrayGetP(pArray, index);
-    if (sdbWrite(pRaw) != 0) {
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static int32_t trnExecuteRedoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->redoLogs); }
-
-static int32_t trnExecuteUndoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->undoLogs); }
-
-static int32_t trnExecuteCommitLogs(STrans *pTrans) { return trnExecuteArray(pTrans->commitLogs); }
-
-static int32_t trnExecuteRedoActions(STrans *pTrans) { return trnExecuteArray(pTrans->redoActions); }
-
-static int32_t trnExecuteUndoActions(STrans *pTrans) { return trnExecuteArray(pTrans->undoActions); }
-
-static int32_t trnPerformPrepareStage(STrans *pTrans) {
-  if (trnExecuteRedoLogs(pTrans) == 0) {
-    pTrans->stage = TRN_STAGE_EXECUTE;
-    return 0;
-  } else {
-    pTrans->stage = TRN_STAGE_ROLLBACK;
-    return -1;
-  }
-}
-
-static int32_t trnPerformExecuteStage(STrans *pTrans) {
-  int32_t code = trnExecuteRedoActions(pTrans);
-
-  if (code == 0) {
-    pTrans->stage = TRN_STAGE_COMMIT;
-    return 0;
-  } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    return -1;
-  } else {
-    if (pTrans->policy == TRN_POLICY_RETRY) {
-      pTrans->stage = TRN_STAGE_RETRY;
-    } else {
-      pTrans->stage = TRN_STAGE_ROLLBACK;
-    }
-    return 0;
-  }
-}
-
-static int32_t trnPerformCommitStage(STrans *pTrans) {
-  if (trnExecuteCommitLogs(pTrans) == 0) {
-    pTrans->stage = TRN_STAGE_EXECUTE;
-    return 0;
-  } else {
-    pTrans->stage = TRN_STAGE_ROLLBACK;
-    return -1;
-  }
-}
-
-static int32_t trnPerformRollbackStage(STrans *pTrans) {
-  if (trnExecuteCommitLogs(pTrans) == 0) {
-    pTrans->stage = TRN_STAGE_EXECUTE;
-    return 0;
-  } else {
-    pTrans->stage = TRN_STAGE_ROLLBACK;
-    return -1;
-  }
-}
-
-static int32_t trnPerformRetryStage(STrans *pTrans) {
-  if (trnExecuteCommitLogs(pTrans) == 0) {
-    pTrans->stage = TRN_STAGE_EXECUTE;
-    return 0;
-  } else {
-    pTrans->stage = TRN_STAGE_ROLLBACK;
-    return -1;
-  }
-}
-
-int32_t trnExecute(int32_t tranId) {
-  int32_t code = 0;
-
-  STrans *pTrans = sdbAcquire(SDB_TRANS, &tranId);
-  if (pTrans == NULL) {
-    return -1;
-  }
-
-  if (pTrans->stage == TRN_STAGE_PREPARE) {
-    if (trnPerformPrepareStage(pTrans) != 0) {
-      sdbRelease(pTrans);
-      return -1;
-    }
-  }
-
-  if (pTrans->stage == TRN_STAGE_EXECUTE) {
-    if (trnPerformExecuteStage(pTrans) != 0) {
-      sdbRelease(pTrans);
-      return -1;
-    }
-  }
-
-  if (pTrans->stage == TRN_STAGE_COMMIT) {
-    if (trnPerformCommitStage(pTrans) != 0) {
-      sdbRelease(pTrans);
-      return -1;
-    }
-  }
-
-  if (pTrans->stage == TRN_STAGE_ROLLBACK) {
-    if (trnPerformRollbackStage(pTrans) != 0) {
-      sdbRelease(pTrans);
-      return -1;
-    }
-  }
-
-  if (pTrans->stage == TRN_STAGE_RETRY) {
-    if (trnPerformRetryStage(pTrans) != 0) {
-      sdbRelease(pTrans);
-      return -1;
-    }
-  }
-
-  sdbRelease(pTrans);
-  return 0;
-}
\ No newline at end of file
diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c
index 71d1e8d890..81b48191bc 100644
--- a/source/dnode/vnode/tq/src/tqMetaStore.c
+++ b/source/dnode/vnode/tq/src/tqMetaStore.c
@@ -92,7 +92,7 @@ TqMetaStore* tqStoreOpen(const char* path,
   char name[pathLen+10];
 
   strcpy(name, path);
-  if(!taosDirExist(name) && !taosMkDir(name)) {
+  if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
     ASSERT(false);
   }
   strcat(name, "/" TQ_IDX_NAME);
diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h
index 1230fe17ff..5a8138b126 100644
--- a/source/libs/index/inc/index_fst.h
+++ b/source/libs/index/inc/index_fst.h
@@ -20,6 +20,7 @@
 #include "tarray.h"
 #include "index_fst_util.h"
 #include "index_fst_registry.h"
+#include "index_fst_counting_writer.h"
 
 
 typedef struct FstNode FstNode;
@@ -35,7 +36,6 @@ typedef struct FstRange {
 typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State;
 typedef enum { Included, Excluded, Unbounded} FstBound; 
 
-typedef uint32_t CheckSummer;
 
 
 /*
@@ -49,7 +49,8 @@ typedef struct FstUnFinishedNodes {
 
 #define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) 
 
-FstUnFinishedNodes *FstUnFinishedNodesCreate();  
+FstUnFinishedNodes *fstUnFinishedNodesCreate();  
+void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
 void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
 FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
 FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
@@ -58,18 +59,13 @@ void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
 void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
 void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
 uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
-uint64_t FstUnFinishedNodesFindCommPreifxAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out);
+uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out);
 
-typedef struct FstCountingWriter {
-  void*    wtr;  // wrap any writer that counts and checksum bytes written 
-  uint64_t count; 
-  CheckSummer summer;  
-} FstCountingWriter;
 
 typedef struct FstBuilder {
-  FstCountingWriter  wtr;         // The FST raw data is written directly to `wtr`.  
+  FstCountingWriter  *wrt;         // The FST raw data is written directly to `wtr`.  
   FstUnFinishedNodes *unfinished; // The stack of unfinished nodes   
-  FstRegistry        registry;    // A map of finished nodes.        
+  FstRegistry*        registry;    // A map of finished nodes.        
   SArray*            last;        // The last word added 
   CompiledAddr       lastAddr;    // The address of the last compiled node  
   uint64_t           len;         // num of keys added
@@ -127,6 +123,8 @@ typedef struct FstNode {
 #define FST_NODE_ADDR(node) node->start 
 
 FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data);
+void fstNodeDestroy(FstNode *fstNode);
+
 FstTransitions fstNodeTransitionIter(FstNode *node);
 FstTransitions* fstNodeTransitions(FstNode *node);
 bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res);
@@ -157,6 +155,9 @@ typedef struct FstIndexedValue {
   uint64_t value;
 } FstIndexedValue;
 
+FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out);
+void fstLastTransitionDestroy(FstLastTransition *trn);
+
 
 
 
diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h
new file mode 100644
index 0000000000..0eba963239
--- /dev/null
+++ b/source/libs/index/inc/index_fst_counting_writer.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __INDEX_FST_COUNTING_WRITER_H__
+#define __INDEX_FST_COUNTING_WRITER_H__
+
+typedef uint32_t CheckSummer;
+
+
+typedef struct FstCountingWriter {
+  void*    wrt;  // wrap any writer that counts and checksum bytes written 
+  uint64_t count; 
+  CheckSummer summer;  
+} FstCountingWriter;
+
+uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen); 
+
+int FstCountingWriterFlush(FstCountingWriter *write);
+
+
+FstCountingWriter *fstCountingWriterCreate(void *wtr);
+void fstCountingWriterDestroy(FstCountingWriter *w); 
+
+
+#define FST_WRITER_COUNT(writer) (writer->count)
+#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
+#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
+
+#endif
+
+
diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h
index 631c7026c5..ddd7e1f450 100644
--- a/source/libs/index/inc/index_fst_node.h
+++ b/source/libs/index/inc/index_fst_node.h
@@ -17,7 +17,11 @@
 #define __INDEX_FST_NODE_H__
 
 #include "index_fst_util.h"
+#include "index_fst_counting_writer.h"
 
+#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) 
+#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) 
+#define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
 
 typedef struct FstTransition {
   uint8_t      inp;  //The byte input associated with this transition.
@@ -37,4 +41,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
 
 void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src);
 
+bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); 
+
+void fstBuilderNodeDestroy(FstBuilderNode *node);
+
 #endif
diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h
index f19bb750c2..2504d7ccff 100644
--- a/source/libs/index/inc/index_fst_registry.h
+++ b/source/libs/index/inc/index_fst_registry.h
@@ -24,6 +24,8 @@ typedef struct FstRegistryCell {
   FstBuilderNode *node;    
 } FstRegistryCell;
 
+#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
+#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0)
 
 
 //typedef struct FstRegistryCache {
@@ -44,14 +46,17 @@ typedef struct FstRegistryEntry {
 
 // Registry relation function 
 typedef struct FstRegistry {
-   SArray *table; 
+   SArray *table;  //<FstRegistryCell> 
    uint64_t tableSize; // num of rows
    uint64_t mruSize;   // num of columns
 } FstRegistry;      
 
 // 
 FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
+void fstRegistryDestroy(FstRegistry *registry);
+
 
 FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
+void fstRegistryEntryDestroy(FstRegistryEntry *entry);
 
 #endif
diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index 91cfcb5cdf..f8f4311a4a 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -183,6 +183,7 @@ void indexMultiTermDestroy(SArray *array) {
   }
   taosArrayDestroy(array);
 }
+
 void indexInit() {
   //do nothing
 }
diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c
index 2974e7f9b5..a7ae3f2fb6 100644
--- a/source/libs/index/src/index_fst.c
+++ b/source/libs/index/src/index_fst.c
@@ -24,6 +24,18 @@ FstUnFinishedNodes *fstUnFinishedNodesCreate() {
   fstUnFinishedNodesPushEmpty(nodes, false);
   return nodes;
 }
+void unFinishedNodeDestroyElem(void* elem) {
+  FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished*)elem;
+  fstBuilderNodeDestroy(b->node); 
+  free(b->last); 
+}  
+void fstUnFinishedNodeDestroy(FstUnFinishedNodes *nodes) {
+  if (nodes == NULL) { return; } 
+
+  taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); 
+  free(nodes);
+}
+
 void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) {
   FstBuilderNode *node = malloc(sizeof(FstBuilderNode));
   node->isFinal     = isFinal;
@@ -76,11 +88,11 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
   assert(un->last == NULL);
 
   
-  FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
-  trn->inp = s->data[s->start]; 
-  trn->out = out;
   
-  un->last = trn; 
+  //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
+  //trn->inp = s->data[s->start]; 
+  //trn->out = out;
+  un->last = fstLastTransitionCreate(s->data[s->start], out); 
 
   for (uint64_t i = s->start; i <= s->end; i++) {
     FstBuilderNode *n = malloc(sizeof(FstBuilderNode));
@@ -88,9 +100,10 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
     n->finalOutput = 0;
     n->trans       = NULL;
     
-    FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
-    trn->inp = s->data[i];
-    trn->out = out; 
+    //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); 
+    //trn->inp = s->data[i];
+    //trn->out = out; 
+    FstLastTransition *trn = fstLastTransitionCreate(s->data[i], out);
 
     FstBuilderNodeUnfinished un = {.node = n, .last = trn}; 
     taosArrayPush(nodes->stack, &un); 
@@ -116,7 +129,7 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs)
   }
   return count;
 }
-uint64_t FstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) {
+uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) {
   FstSlice *s = &bs;
 
   size_t lsz = (size_t)(s->end - s->start + 1);          // data len 
@@ -199,6 +212,10 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
    } 
    return n; 
 }
+void fstNodeDestroy(FstNode *node) {
+  if (node == NULL) { return; } 
+  free(node);
+}
 FstTransitions* fstNodeTransitions(FstNode *node) {
   FstTransitions *t = malloc(sizeof(FstTransitions));
   if (NULL == t) {
@@ -275,22 +292,74 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr
 } 
 
 
-
-
 FstBuilder *fstBuilderCreate(void *w, FstType ty) {
   FstBuilder *b = malloc(sizeof(FstBuilder));  
   if (NULL == b) { return b; }
 
-  FstCountingWriter wtr = {.wtr = w, .count = 0, .summer = 0};  
-  b->wtr = wtr;
-  b->unfinished = malloc(sizeof(FstUnFinishedNodes));   
+   
+  b->wrt = fstCountingWriterCreate(w);
+  b->unfinished = fstUnFinishedNodesCreate();   
+  b->registry   = fstRegistryCreate(10000, 2) ;
+  b->last       = NULL;
+  b->lastAddr   = NONE_ADDRESS; 
+  b->len        = 0;
   return b;
-  
 }
+
+
+void fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDupe) {
+  return;
+} 
+
+CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) {
+  if (FST_BUILDER_NODE_IS_FINAL(bn) 
+      && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) 
+      && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
+    return EMPTY_ADDRESS; 
+  }
+  FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); 
+  if (entry->state == FOUND) { 
+    CompiledAddr ret = entry->addr;
+    fstRegistryEntryDestroy(entry);
+    return ret;
+  } 
+  CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt));
+
+  fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);  
+  b->lastAddr =  (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);  
+  if (entry->state == NOTFOUND) {
+    FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);    
+  }
+  fstRegistryEntryDestroy(entry);
+  
+  return b->lastAddr;  
+}
+
+
 FstSlice fstNodeAsSlice(FstNode *node) {
   FstSlice *slice = &node->data; 
   FstSlice s = fstSliceCopy(slice, slice->end, slice->dLen - 1);   
   return s; 
 }
 
+FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) {
+  FstLastTransition *trn = malloc(sizeof(FstLastTransition));
+  if (trn == NULL) { return NULL; }
+
+  trn->inp = inp;
+  trn->out = out;
+  return trn;
+}
+
+void fstLastTransitionDestroy(FstLastTransition *trn) {
+  free(trn);
+}
+void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr) {
+  return;
+}
+
+void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, CompiledAddr addr) {
+  return;
+}
+
 
diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c
new file mode 100644
index 0000000000..1486b9b203
--- /dev/null
+++ b/source/libs/index/src/index_fst_counting_writer.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "tutil.h"
+#include "index_fst_counting_writer.h"
+
+FstCountingWriter *fstCountingWriterCreate(void *wrt) {
+  FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); 
+  if (cw == NULL) { return NULL; }
+
+  cw->wrt = wrt; 
+  return cw; 
+}
+void fstCountingWriterDestroy(FstCountingWriter *cw) {
+  // free wrt object: close fd or free mem 
+  free(cw);
+}
+
+uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) {
+  if (write == NULL) { return 0; } 
+  // update checksum 
+  // write data to file/socket or mem
+  
+  write->count += bufLen;
+  return bufLen; 
+} 
+
+int FstCountingWriterFlush(FstCountingWriter *write) {
+  //write->wtr->flush
+  return 1;
+}
+
+
+
diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c
index 23af4a4a4b..5452f9cb89 100644
--- a/source/libs/index/src/index_fst_node.c
+++ b/source/libs/index/src/index_fst_node.c
@@ -21,19 +21,23 @@ FstBuilderNode *fstBuilderNodeDefault() {
   bn->trans       = NULL; 
   return bn;
 }
+void fstBuilderNodeDestroy(FstBuilderNode *node) {
+  if (node == NULL) { return; }
 
+  taosArrayDestroy(node->trans);
+  free(node);
+} 
 FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
   FstBuilderNode *node = malloc(sizeof(FstBuilderNode));  
   if (node == NULL) { return NULL; }
 
-      
+  // 
   size_t sz = taosArrayGetSize(src->trans);
   SArray *trans = taosArrayInit(sz, sizeof(FstTransition));
 
   for (size_t i = 0; i < sz; i++) {
     FstTransition *tran = taosArrayGet(src->trans, i);
-    FstTransition t = *tran;
-    taosArrayPush(trans, &t);   
+    taosArrayPush(trans, tran);   
   }
 
   node->trans = trans; 
@@ -47,9 +51,34 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
   if (dst == NULL || src == NULL) { return; } 
 
   dst->isFinal     = src->isFinal;
-  dst->finalOutput = src->finalOutput ;
-  dst->trans       = src->trans;    
+  dst->finalOutput = src->finalOutput;
 
+  // avoid mem leak
+  taosArrayDestroy(dst->trans); 
+  dst->trans       = src->trans;    
   src->trans = NULL;
 }
 
+bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
+  size_t sz = taosArrayGetSize(b->trans);  
+  assert(sz < 256);
+  if (FST_BUILDER_NODE_IS_FINAL(b) 
+      && FST_BUILDER_NODE_TRANS_ISEMPTY(b) 
+      && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) {
+    return true; 
+  } else if (sz != 1 || b->isFinal) {
+    // AnyTrans->Compile(w, addr, node);
+  } else {
+    FstTransition *tran = taosArrayGet(b->trans, 0);   
+    if (tran->addr == lastAddr && tran->out == 0) {
+       //OneTransNext::compile(w, lastAddr, tran->inp);
+       return true;
+    } else {
+      //OneTrans::Compile(w, lastAddr, *tran);
+       return true;
+    } 
+  } 
+  return true; 
+} 
+
+
diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c
index 5d6c7b1712..b25964e0e7 100644
--- a/source/libs/index/src/index_fst_registry.c
+++ b/source/libs/index/src/index_fst_registry.c
@@ -32,6 +32,7 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
     h = (h ^ (uint64_t)(trn->addr))* FNV_PRIME;
   } 
   return h %(registry->tableSize); 
+
 }
 static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
   size_t sz = taosArrayGetSize(arr);
@@ -63,8 +64,6 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
     s -= 1;
   }
 }
-#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
-#define FST_REGISTRY_CELL_INSERT(cell, addr) do {cell->addr = addr;} while(0)
 
 FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
   FstRegistry *registry = malloc(sizeof(FstRegistry));  
@@ -72,10 +71,14 @@ FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
 
   uint64_t nCells = tableSize * mruSize; 
   SArray* tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); 
+  if (NULL == tb) { 
+    free(registry); 
+    return NULL; 
+  }
+
   for (uint64_t i = 0; i < nCells; i++) {
-    FstRegistryCell *cell = taosArrayGet(tb, i);
-    cell->addr = NONE_ADDRESS; 
-    cell->node = fstBuilderNodeDefault(); 
+    FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; 
+    taosArrayPush(tb, &cell);  
   }
   
   registry->table = tb;   
@@ -84,6 +87,19 @@ FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
   return registry;   
 }
 
+void fstRegistryDestroy(FstRegistry *registry) {
+  if (registry == NULL) { return; }
+
+  SArray *tb = registry->table;
+  size_t sz = taosArrayGetSize(tb);
+  for (size_t i = 0; i < sz; i++) {
+    FstRegistryCell *cell = taosArrayGet(tb, i); 
+    fstBuilderNodeDestroy(cell->node); 
+  }
+  taosArrayDestroy(tb);
+  free(registry);
+}
+
 FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) {
   if (taosArrayGetSize(registry->table) <= 0) {
     return NULL;    
@@ -98,11 +114,9 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
     //cell->isNode && 
     if (cell->addr != NONE_ADDRESS && cell->node == bNode) {
        entry->state = FOUND;
-       entry->addr =  cell->addr ;
+       entry->addr  = cell->addr ;
        return entry; 
     } else {
-       // clone from bNode, refactor later
-       //
        fstBuilderNodeCloneFrom(cell->node, bNode);
        entry->state = NOTFOUND;
        entry->cell  = cell; // copy or not
@@ -154,5 +168,8 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
   }      
   return entry;
 }
+void fstRegistryEntryDestroy(FstRegistryEntry *entry) {
+  free(entry);  
+}
 
 
diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp
index cc0df0d42a..857060ce5e 100644
--- a/source/libs/index/test/indexTests.cpp
+++ b/source/libs/index/test/indexTests.cpp
@@ -51,6 +51,7 @@ TEST(IndexTest, index_create_test) {
     int *v = (int *)taosArrayGet(result, i);
     std::cout << "value --->" << *v  << std::endl;
   }
+  // add more test case 
   indexMultiTermQueryDestroy(multiQuery);
 
   indexOptsDestroy(opts); 
diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c
index 1aafc880be..a2041c76fc 100644
--- a/source/libs/transport/src/rpcMain.c
+++ b/source/libs/transport/src/rpcMain.c
@@ -54,10 +54,11 @@ typedef struct {
   char     secret[TSDB_KEY_LEN]; // secret for the link
   char     ckey[TSDB_KEY_LEN];   // ciphering key 
 
-  void   (*cfp)(SRpcMsg *, SEpSet *);
-  int    (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); 
+  void   (*cfp)(void *parent, SRpcMsg *, SEpSet *);
+  int    (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); 
 
   int32_t   refCount;
+  void     *parent;
   void     *idPool;   // handle to ID pool
   void     *tmrCtrl;  // handle to timer
   SHashObj *hash;     // handle returned by hash utility
@@ -260,6 +261,7 @@ void *rpcOpen(const SRpcInit *pInit) {
   pRpc->spi = pInit->spi;
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
+  pRpc->parent = pInit->parent;
   pRpc->refCount = 1;
 
   atomic_add_fetch_32(&tsRpcNum, 1);
@@ -505,14 +507,18 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) {
 }
 
 int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
+#if 0
   SRpcConn  *pConn = (SRpcConn *)thandle;
   if (pConn->user[0] == 0) return -1;
 
   pInfo->clientIp = pConn->peerIp;
   pInfo->clientPort = pConn->peerPort;
   // pInfo->serverIp = pConn->destIp;
-
+  
   tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
+#else 
+  strcpy(pInfo->user, "root");
+#endif
   return 0;
 }
 
@@ -740,7 +746,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
       if (pConn->user[0] == 0) {
         terrno = TSDB_CODE_RPC_AUTH_REQUIRED;
       } else {
-        terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey);
+        terrno = (*pRpc->afp)(pRpc->parent, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey);
       }
 
       if (terrno != 0) {
@@ -1020,8 +1026,8 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) {
    SRpcMsg *pRpcMsg = (SRpcMsg *)(param); 
    SRpcConn *pConn  = (SRpcConn *)(pRpcMsg->handle);
    SRpcInfo *pRpc   = pConn->pRpc; 
-   (*(pRpc->cfp))(pRpcMsg, NULL);
-   free(pRpcMsg);
+  (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL);
+  free(pRpcMsg);
 }
 static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
   SRpcInfo *pRpc = pConn->pRpc;
@@ -1133,9 +1139,9 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
     // for asynchronous API 
     SEpSet *pEpSet = NULL;
     if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) 
-      pEpSet = &pContext->epSet;  
+      pEpSet = &pContext->epSet;
 
-    (*pRpc->cfp)(pMsg, pEpSet);  
+    (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet);
   }
 
   // free the request message
@@ -1151,15 +1157,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
   rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen);
   rpcMsg.pCont = pHead->content;
   rpcMsg.msgType = pHead->msgType;
-  rpcMsg.code = pHead->code; 
-   
-  if ( rpcIsReq(pHead->msgType) ) {
+  rpcMsg.code = pHead->code;
+
+  if (rpcIsReq(pHead->msgType)) {
     rpcMsg.ahandle = pConn->ahandle;
     rpcMsg.handle = pConn;
     rpcAddRef(pRpc);  // add the refCount for requests
 
     // notify the server app
-    (*(pRpc->cfp))(&rpcMsg, NULL);
+    (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
   } else {
     // it's a response
     rpcMsg.handle = pContext;
diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt
index 4af8bac7f9..e5697415f1 100644
--- a/source/libs/wal/CMakeLists.txt
+++ b/source/libs/wal/CMakeLists.txt
@@ -9,4 +9,5 @@ target_include_directories(
 target_link_libraries(
     wal
     PUBLIC os
+    PUBLIC util
 )
diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h
index 3cf38a5ffc..f5f944b12b 100644
--- a/source/libs/wal/inc/walInt.h
+++ b/source/libs/wal/inc/walInt.h
@@ -16,6 +16,8 @@
 #ifndef _TD_WAL_INT_H_
 #define _TD_WAL_INT_H_
 
+#include "wal.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -24,4 +26,4 @@ extern "C" {
 }
 #endif
 
-#endif /*_TD_WAL_INT_H_*/
\ No newline at end of file
+#endif /*_TD_WAL_INT_H_*/
diff --git a/source/libs/wal/src/wal.c b/source/libs/wal/src/wal.c
index c107a94f3f..05d81e0867 100644
--- a/source/libs/wal/src/wal.c
+++ b/source/libs/wal/src/wal.c
@@ -15,40 +15,35 @@
 
 #include "wal.h"
 
-int32_t walInit() { return 0; }
-
-void walCleanUp() {}
-
-SWal *walOpen(char *path, SWalCfg *pCfg) {
-  SWal* pWal = malloc(sizeof(SWal));
-  if(pWal == NULL) {
-    return NULL;
-  }
-  return pWal;
-}
-
-int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { return 0; }
-
-void walClose(SWal *pWal) {
-  if(pWal) free(pWal);
-}
-
-void walFsync(SWal *pWal, bool force) {}
-
-int64_t walWrite(SWal *pWal, int64_t index, void *body, int32_t bodyLen) {
+int32_t walCommit(SWal *pWal, int64_t ver) {
   return 0;
 }
 
-int32_t walCommit(SWal *pWal, int64_t ver) { return 0; }
+int32_t walRollback(SWal *pWal, int64_t ver) {
+  return 0;
+}
 
-int32_t walRollback(SWal *pWal, int64_t ver) { return 0; }
-
-int32_t walPrune(SWal *pWal, int64_t ver) { return 0; }
+int32_t walPrune(SWal *pWal, int64_t ver) {
+  return 0;
+}
 
 
-int32_t walRead(SWal *, SWalHead **, int64_t ver);
-int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
+int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
+  return 0;
+}
 
-int64_t walGetFirstVer(SWal *);
-int64_t walGetSnapshotVer(SWal *);
-int64_t walGetLastVer(SWal *);
+int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {
+  return 0;
+}
+
+int64_t walGetFirstVer(SWal *pWal) {
+  return 0;
+}
+
+int64_t walGetSnapshotVer(SWal *pWal) {
+  return 0;
+}
+
+int64_t walGetLastVer(SWal *pWal) {
+  return 0;
+}
diff --git a/src/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c
similarity index 91%
rename from src/wal/src/walMgmt.c
rename to source/libs/wal/src/walMgmt.c
index dbff08d730..917723b166 100644
--- a/src/wal/src/walMgmt.c
+++ b/source/libs/wal/src/walMgmt.c
@@ -18,7 +18,6 @@
 #include "taoserror.h"
 #include "tref.h"
 #include "tfile.h"
-#include "twal.h"
 #include "walInt.h"
 
 typedef struct {
@@ -62,8 +61,8 @@ void walCleanUp() {
   wInfo("wal module is cleaned up");
 }
 
-void *walOpen(char *path, SWalCfg *pCfg) {
-  SWal *pWal = tcalloc(1, sizeof(SWal));
+SWal *walOpen(char *path, SWalCfg *pCfg) {
+  SWal *pWal = malloc(sizeof(SWal));
   if (pWal == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return NULL;
@@ -73,7 +72,7 @@ void *walOpen(char *path, SWalCfg *pCfg) {
   pWal->tfd = -1;
   pWal->fileId = -1;
   pWal->level = pCfg->walLevel;
-  pWal->keep = pCfg->keep;
+  /*pWal->keep = pCfg->keep;*/
   pWal->fsyncPeriod = pCfg->fsyncPeriod;
   tstrncpy(pWal->path, path, sizeof(pWal->path));
   pthread_mutex_init(&pWal->mutex, NULL);
@@ -86,8 +85,8 @@ void *walOpen(char *path, SWalCfg *pCfg) {
     return NULL;
   }
 
-   pWal->rid = taosAddRef(tsWal.refId, pWal);
-   if (pWal->rid < 0) {
+   pWal->rId = taosAddRef(tsWal.refId, pWal);
+   if (pWal->rId < 0) {
     walFreeObj(pWal);
     return NULL;
   }
@@ -97,9 +96,8 @@ void *walOpen(char *path, SWalCfg *pCfg) {
   return pWal;
 }
 
-int32_t walAlter(void *handle, SWalCfg *pCfg) {
-  if (handle == NULL) return TSDB_CODE_WAL_APP_ERROR;
-  SWal *pWal = handle;
+int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
+  if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR;
 
   if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
     wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level,
@@ -128,18 +126,17 @@ void walStop(void *handle) {
   wDebug("vgId:%d, stop write wal", pWal->vgId);
 }
 
-void walClose(void *handle) {
-  if (handle == NULL) return;
+void walClose(SWal *pWal) {
+  if (pWal == NULL) return;
 
-  SWal *pWal = handle;
   pthread_mutex_lock(&pWal->mutex);
   tfClose(pWal->tfd);
   pthread_mutex_unlock(&pWal->mutex);
-  taosRemoveRef(tsWal.refId, pWal->rid);
+  taosRemoveRef(tsWal.refId, pWal->rId);
 }
 
 static int32_t walInitObj(SWal *pWal) {
-  if (!taosMkDir(pWal->path)) {
+  if (taosMkDir(pWal->path) != 0) {
     wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
@@ -186,7 +183,7 @@ static void walFsyncAll() {
         wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
       }
     }
-    pWal = taosIterateRef(tsWal.refId, pWal->rid);
+    pWal = taosIterateRef(tsWal.refId, pWal->rId);
   }
 }
 
diff --git a/src/wal/src/walUtil.c b/source/libs/wal/src/walUtil.c
similarity index 99%
rename from src/wal/src/walUtil.c
rename to source/libs/wal/src/walUtil.c
index e4d9a555b3..c88cc918fe 100644
--- a/src/wal/src/walUtil.c
+++ b/source/libs/wal/src/walUtil.c
@@ -115,4 +115,4 @@ int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) {
   wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId);
 
   return 0;
-}
\ No newline at end of file
+}
diff --git a/src/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
similarity index 73%
rename from src/wal/src/walWrite.c
rename to source/libs/wal/src/walWrite.c
index cae4291eb8..023b1c4a48 100644
--- a/src/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -14,13 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
-#define TAOS_RANDOM_FILE_FAIL_TEST
+
 #include "os.h"
 #include "taoserror.h"
-#include "taosmsg.h"
 #include "tchecksum.h"
 #include "tfile.h"
-#include "twal.h"
 #include "walInt.h"
 
 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId);
@@ -43,12 +41,12 @@ int32_t walRenew(void *handle) {
     wDebug("vgId:%d, file:%s, it is closed while renew", pWal->vgId, pWal->name);
   }
 
-  if (pWal->keep == TAOS_WAL_KEEP) {
-    pWal->fileId = 0;
-  } else {
-    if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;
-    pWal->fileId++;
-  }
+  /*if (pWal->keep == TAOS_WAL_KEEP) {*/
+    /*pWal->fileId = 0;*/
+  /*} else {*/
+    /*if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;*/
+    /*pWal->fileId++;*/
+  /*}*/
 
   snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
   pWal->tfd = tfOpenCreateWrite(pWal->name);
@@ -68,7 +66,7 @@ int32_t walRenew(void *handle) {
 void walRemoveOneOldFile(void *handle) {
   SWal *pWal = handle;
   if (pWal == NULL) return;
-  if (pWal->keep == TAOS_WAL_KEEP) return;
+  /*if (pWal->keep == TAOS_WAL_KEEP) return;*/
   if (!tfValid(pWal->tfd)) return;
 
   pthread_mutex_lock(&pWal->mutex);
@@ -117,7 +115,7 @@ void walRemoveAllOldFiles(void *handle) {
 static void walUpdateChecksum(SWalHead *pHead) {
   pHead->sver = 2;
   pHead->cksum = 0;
-  pHead->cksum = taosCalcChecksum(0, (uint8_t *)pHead, sizeof(*pHead) + pHead->len);
+  pHead->cksum = taosCalcChecksum(0, (uint8_t *)pHead, sizeof(SWalHead) + pHead->len);
 }
 
 static int walValidateChecksum(SWalHead *pHead) {
@@ -134,10 +132,14 @@ static int walValidateChecksum(SWalHead *pHead) {
 
 #endif
 
-int32_t walWrite(void *handle, SWalHead *pHead) {
-  if (handle == NULL) return -1;
+int64_t walWrite(SWal *pWal, int64_t index, void *body, int32_t bodyLen) {
+  if (pWal == NULL) return -1;
 
-  SWal *  pWal = handle;
+  SWalHead *pHead = malloc(sizeof(SWalHead) + bodyLen);
+  if(pHead == NULL) {
+    return -1;
+  }
+  pHead->version = index;
   int32_t code = 0;
 
   // no wal
@@ -146,6 +148,9 @@ int32_t walWrite(void *handle, SWalHead *pHead) {
   if (pHead->version <= pWal->version) return 0;
 
   pHead->signature = WAL_SIGNATURE;
+  pHead->len = bodyLen;
+  memcpy(pHead->cont, body, bodyLen);
+
 #if defined(WAL_CHECKSUM_WHOLE)
   walUpdateChecksum(pHead);
 #else
@@ -173,8 +178,7 @@ int32_t walWrite(void *handle, SWalHead *pHead) {
   return code;
 }
 
-void walFsync(void *handle, bool forceFsync) {
-  SWal *pWal = handle;
+void walFsync(SWal *pWal, bool forceFsync) {
   if (pWal == NULL || !tfValid(pWal->tfd)) return;
 
   if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) {
@@ -211,7 +215,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
     count++;
   }
 
-  if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;
+  /*if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;*/
 
   if (count == 0) {
     wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId);
@@ -307,119 +311,10 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd,
 
   return TSDB_CODE_WAL_FILE_CORRUPTED;
 }
-// Add SMemRowType ahead of SDataRow
-static void expandSubmitBlk(SSubmitBlk *pDest, SSubmitBlk *pSrc, int32_t *lenExpand) {
-  // copy the header firstly
-  memcpy(pDest, pSrc, sizeof(SSubmitBlk));
-
-  int32_t nRows = htons(pDest->numOfRows);
-  int32_t dataLen = htonl(pDest->dataLen);
-
-  if ((nRows <= 0) || (dataLen <= 0)) {
-    return;
-  }
-
-  char *pDestData = pDest->data;
-  char *pSrcData = pSrc->data;
-  for (int32_t i = 0; i < nRows; ++i) {
-    memRowSetType(pDestData, SMEM_ROW_DATA);
-    memcpy(memRowDataBody(pDestData), pSrcData, dataRowLen(pSrcData));
-    pDestData = POINTER_SHIFT(pDestData, memRowTLen(pDestData));
-    pSrcData = POINTER_SHIFT(pSrcData, dataRowLen(pSrcData));
-    ++(*lenExpand);
-  }
-  pDest->dataLen = htonl(dataLen + nRows * sizeof(uint8_t));
-}
-
-// Check SDataRow by comparing the SDataRow len and SSubmitBlk dataLen
-static bool walIsSDataRow(void *pBlkData, int nRows, int32_t dataLen) {
-  if ((nRows <= 0) || (dataLen <= 0)) {
-    return true;
-  }
-  int32_t len = 0, kvLen = 0;
-  for (int i = 0; i < nRows; ++i) {
-    len += dataRowLen(pBlkData);
-    if (len > dataLen) {
-      return false;
-    }
-
-    /**
-     * For SDataRow between version [2.1.5.0 and 2.1.6.X], it would never conflict.
-     * For SKVRow between version [2.1.5.0 and 2.1.6.X], it may conflict in below scenario
-     *   -  with 1st type byte 0x01 and sversion  0x0101(257), thus do further check
-     */
-    if (dataRowLen(pBlkData) == 257) {
-      SMemRow  memRow = pBlkData;
-      SKVRow   kvRow = memRowKvBody(memRow);
-      int      nCols = kvRowNCols(kvRow);
-      uint16_t calcTsOffset = (uint16_t)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nCols);
-      uint16_t realTsOffset = (kvRowColIdx(kvRow))->offset;
-      if (calcTsOffset == realTsOffset) {
-        kvLen += memRowKvTLen(memRow);
-      }
-    }
-    pBlkData = POINTER_SHIFT(pBlkData, dataRowLen(pBlkData));
-  }
-  if (len != dataLen) {
-    return false;
-  }
-  if (kvLen == dataLen) {
-    return false;
-  }
-  return true;
-}
-// for WAL SMemRow/SDataRow compatibility
-static int walSMemRowCheck(SWalHead *pHead) {
-  if ((pHead->sver < 2) && (pHead->msgType == TSDB_MSG_TYPE_SUBMIT)) {
-    SSubmitMsg *pMsg = (SSubmitMsg *)pHead->cont;
-    int32_t     numOfBlocks = htonl(pMsg->numOfBlocks);
-    if (numOfBlocks <= 0) {
-      return 0;
-    }
-
-    int32_t     nTotalRows = 0;
-    SSubmitBlk *pBlk = (SSubmitBlk *)pMsg->blocks;
-    for (int32_t i = 0; i < numOfBlocks; ++i) {
-      int32_t dataLen = htonl(pBlk->dataLen);
-      int32_t nRows = htons(pBlk->numOfRows);
-      nTotalRows += nRows;
-      if (!walIsSDataRow(pBlk->data, nRows, dataLen)) {
-        return 0;
-      }
-      pBlk = (SSubmitBlk *)POINTER_SHIFT(pBlk, sizeof(SSubmitBlk) + dataLen);
-    }
-    ASSERT(nTotalRows >= 0);
-    SWalHead *pWalHead = (SWalHead *)calloc(sizeof(SWalHead) + pHead->len + nTotalRows * sizeof(uint8_t), 1);
-    if (pWalHead == NULL) {
-      return -1;
-    }
-
-    memcpy(pWalHead, pHead, sizeof(SWalHead) + sizeof(SSubmitMsg));
-
-    SSubmitMsg *pDestMsg = (SSubmitMsg *)pWalHead->cont;
-    SSubmitBlk *pDestBlks = (SSubmitBlk *)pDestMsg->blocks;
-    SSubmitBlk *pSrcBlks = (SSubmitBlk *)pMsg->blocks;
-    int32_t     lenExpand = 0;
-    for (int32_t i = 0; i < numOfBlocks; ++i) {
-      expandSubmitBlk(pDestBlks, pSrcBlks, &lenExpand);
-      pDestBlks = POINTER_SHIFT(pDestBlks, htonl(pDestBlks->dataLen) + sizeof(SSubmitBlk));
-      pSrcBlks = POINTER_SHIFT(pSrcBlks, htonl(pSrcBlks->dataLen) + sizeof(SSubmitBlk));
-    }
-    if (lenExpand > 0) {
-      pDestMsg->header.contLen = htonl(pDestMsg->length) + lenExpand;
-      pDestMsg->length = htonl(pDestMsg->header.contLen);
-      pWalHead->len = pWalHead->len + lenExpand;
-    }
-
-    memcpy(pHead, pWalHead, sizeof(SWalHead) + pWalHead->len);
-    tfree(pWalHead);
-  }
-  return 0;
-}
 
 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) {
   int32_t size = WAL_MAX_SIZE;
-  void *  buffer = tmalloc(size);
+  void *  buffer = malloc(size);
   if (buffer == NULL) {
     wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
@@ -541,14 +436,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
     pWal->version = pHead->version;
 
     // wInfo("writeFp: %ld", offset);
-    if (0 != walSMemRowCheck(pHead)) {
-      wError("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64,
-             pWal->vgId, fileId, pHead->version, pWal->version, pHead->len, offset);
-      tfClose(tfd);
-      tfree(buffer);
-      return TAOS_SYSTEM_ERROR(errno);
-    }
-    (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL);
+    (*writeFp)(pVnode, pHead, NULL);
   }
 
   tfClose(tfd);
@@ -558,9 +446,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
   return code;
 }
 
-uint64_t walGetVersion(twalh param) {
-  SWal *pWal = param;
-  if (pWal == 0) return 0;
+uint64_t walGetVersion(SWal *pWal) {
+  if (pWal == NULL) return 0;
 
   return pWal->version;
 }
@@ -570,10 +457,9 @@ uint64_t walGetVersion(twalh param) {
 // Some new wal record cannot be written to the wal file in dnode1 for wal version not reset, then fversion and the record in wal file may inconsistent, 
 // At this time, if dnode2 down, dnode1 switched to master. After dnode2 start and restore data from dnode1, data loss will occur
 
-void walResetVersion(twalh param, uint64_t newVer) {
-  SWal *pWal = param;
-  if (pWal == 0) return;
+void walResetVersion(SWal *pWal, uint64_t newVer) {
+  if (pWal == NULL) return;
   wInfo("vgId:%d, version reset from %" PRIu64 " to %" PRIu64, pWal->vgId, pWal->version, newVer);
 
   pWal->version = newVer;
-}
\ No newline at end of file
+}
diff --git a/source/libs/wal/test/walTests.cpp b/source/libs/wal/test/walTests.cpp
index e69de29bb2..505728fbe4 100644
--- a/source/libs/wal/test/walTests.cpp
+++ b/source/libs/wal/test/walTests.cpp
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "twal.h"
+#include "tfile.h"
+
+int64_t  ver = 0;
+void    *pWal = NULL;
+
+int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
+  // do nothing
+  SWalHead *pHead = data;
+
+  if (pHead->version > ver)
+    ver = pHead->version;
+
+  walWrite(pWal, pHead);
+
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  char path[128] = "/tmp/wal";
+  int  level = 2;
+  int  total = 5;
+  int  rows = 10000;
+  int  size = 128;
+  int  keep = 0;
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      tstrncpy(path, argv[++i], sizeof(path));
+    } else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
+      level = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
+      rows = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
+      keep = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      total = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      size = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
+      ver = atoll(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      dDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p path]: wal file path default is:%s\n", path);
+      printf("  [-l level]: log level, default is:%d\n", level);
+      printf("  [-t total]: total wal files, default is:%d\n", total);
+      printf("  [-r rows]: rows of records per wal file, default is:%d\n", rows);
+      printf("  [-k keep]: keep the wal after closing, default is:%d\n", keep);
+      printf("  [-v version]: initial version, default is:%" PRId64 "\n", ver);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", dDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+
+  taosInitLog("wal.log", 100000, 10);
+  tfInit();
+  walInit();
+
+  SWalCfg walCfg = {0};
+  walCfg.walLevel = level;
+  walCfg.keep = keep;
+
+  pWal = walOpen(path, &walCfg);
+  if (pWal == NULL) {
+    printf("failed to open wal\n");
+    exit(-1);
+  }
+
+  int ret = walRestore(pWal, NULL, writeToQueue);
+  if (ret <0) {
+    printf("failed to restore wal\n");
+    exit(-1);
+  }
+
+  printf("version starts from:%" PRId64 "\n", ver);
+  
+  int contLen = sizeof(SWalHead) + size;
+  SWalHead *pHead = (SWalHead *) malloc(contLen);
+
+  for (int i=0; i<total; ++i) {
+    for (int k=0; k<rows; ++k) {
+      pHead->version = ++ver;
+      pHead->len = size;
+      walWrite(pWal, pHead);
+    }
+       
+    printf("renew a wal, i:%d\n", i);
+    walRenew(pWal);
+  }
+
+  printf("%d wal files are written\n", total);
+
+  int64_t index = 0;
+  char    name[256];
+
+  while (1) {
+    int code = walGetWalFile(pWal, name, &index);
+    if (code == -1) {
+      printf("failed to get wal file, index:%" PRId64 "\n", index);
+      break;
+    }
+
+    printf("index:%" PRId64 " wal:%s\n", index, name);
+    if (code == 0) break;
+  }
+
+  getchar();
+
+  walClose(pWal);
+  walCleanUp();
+  tfCleanup();
+
+  return 0;
+}
diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c
index bf9446c578..cfa0028925 100644
--- a/source/os/src/osDir.c
+++ b/source/os/src/osDir.c
@@ -58,15 +58,15 @@ void taosRemoveDir(const char *dirname) {
   printf("dir:%s is removed\n", dirname);
 }
 
-bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; }
+int32_t taosDirExist(char *dirname) { return access(dirname, F_OK); }
 
-bool taosMkDir(const char *dirname) {
+int32_t taosMkDir(const char *dirname) {
   int32_t code = mkdir(dirname, 0755);
   if (code < 0 && errno == EEXIST) {
-    return true;
+    return 0;
   }
 
-  return code == 0;
+  return code;
 }
 
 void taosRemoveOldFiles(char *dirname, int32_t keepDays) {
@@ -112,12 +112,12 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) {
   rmdir(dirname);
 }
 
-bool taosExpandDir(char *dirname, char *outname, int32_t maxlen) {
+int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) {
   wordexp_t full_path;
   if (0 != wordexp(dirname, &full_path, 0)) {
     printf("failed to expand path:%s since %s", dirname, strerror(errno));
     wordfree(&full_path);
-    return false;
+    return -1;
   }
 
   if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
@@ -126,16 +126,16 @@ bool taosExpandDir(char *dirname, char *outname, int32_t maxlen) {
 
   wordfree(&full_path);
 
-  return true;
+  return 0;
 }
 
-bool taosRealPath(char *dirname, int32_t maxlen) {
+int32_t taosRealPath(char *dirname, int32_t maxlen) {
   char tmp[PATH_MAX] = {0};
   if (realpath(dirname, tmp) != NULL) {
     strncpy(dirname, tmp, maxlen);
   }
 
-  return true;
+  return 0;
 }
 
 #endif
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index 6b3f08a446..8a6f389366 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -154,7 +154,7 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
       taosExpandDir(input_value, option, cfg->ptrLength);
       taosRealPath(option, cfg->ptrLength);
 
-      if (!taosMkDir(option)) {
+      if (taosMkDir(option) != 0) {
         uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value,
                strerror(errno));
         return false;
@@ -335,7 +335,7 @@ void taosReadGlobalLogCfg() {
   fclose(fp);
 }
 
-bool taosReadGlobalCfg() {
+int32_t taosReadGlobalCfg() {
   char * line, *option, *value, *value2, *value3;
   int    olen, vlen, vlen2, vlen3;
   char   fileName[PATH_MAX] = {0};
@@ -345,7 +345,7 @@ bool taosReadGlobalCfg() {
   if (fp == NULL) {
     fp = fopen(configDir, "r");
     if (fp == NULL) {
-      return false;
+      return -1;
     }
   }
 
@@ -393,7 +393,7 @@ bool taosReadGlobalCfg() {
   //   taosSetAllDebugFlag();
   // }
 
-  return true;
+  return 0;
 }
 
 void taosPrintGlobalCfg() {
@@ -402,7 +402,7 @@ void taosPrintGlobalCfg() {
 
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
-    if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
+    if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
     if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
     
     int optionLen = (int)strlen(cfg->option);
@@ -487,7 +487,7 @@ void taosDumpGlobalCfg() {
   printf("==================================\n");
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
-    if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
+    if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
     if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
     if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue;
 
@@ -499,7 +499,7 @@ void taosDumpGlobalCfg() {
 
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
-    if (tscEmbedded == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
+    if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
     if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
     if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue;
 
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 0b10274c00..6d4f4eb6a7 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -78,6 +78,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,             "Memory corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,               "Data file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR,               "Checksum error")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG,                  "Invalid config message")
+TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED,            "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     "too many Ref Objs")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED,               "Ref ID is removed")
@@ -235,20 +236,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS,  "Invalid topic partito
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST,      "Topic already exists")
 
 // dnode
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        "Message not processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY,            "Dnode out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE, "Mnode Id not match Dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS,       "Action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING,                  "Dnode is exiting")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN,          "Invalid message length")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR,    "Read dnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR,   "Write dnode.json error")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED,   "Mnode already deployed")
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED,       "Mnode not deployed")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_READ_MNODE_FILE_ERROR,    "Read mnode.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR,   "Write mnode.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS,          "No permission for disk files in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN,          "Invalid message length")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS,       "Action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES,          "Too many vnode directories")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING,                  "Dnode is exiting")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR,   "Parse vnodes.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR,   "Parse dnodes.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_INVALID,         "Mnode Id invalid")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_FOUND,       "Mnode Id not found")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR,    "Read mnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR,   "Write mnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES,    "Too many vnode directories")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR,    "Read vnodes.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR,   "Write vnodes.json error")
 
 // vnode
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS,       "Action in progress")
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 2fb84656b6..e6cc3a53af 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -68,6 +68,8 @@ typedef struct {
   pthread_mutex_t logMutex;
 } SLogObj;
 
+int8_t  tscEmbeddedInUtil = 0;
+
 int32_t tsLogKeepDays = 0;
 int8_t  tsAsyncLog = 1;
 float   tsTotalLogDirGB = 0;
diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c
index 50b1e0002d..5606ab248d 100644
--- a/source/util/src/tnote.c
+++ b/source/util/src/tnote.c
@@ -40,7 +40,7 @@ static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote,
   taosNotePrint(pNote, "==================================================");
 }
 
-void taosInitNotes() {
+int32_t taosInitNotes() {
   char name[TSDB_FILENAME_LEN * 2] = {0};
 
 #if 0
@@ -58,7 +58,8 @@ void taosInitNotes() {
     snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir);
     taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name);
   }
-#endif  
+#endif
+  return 0;
 }
 
 static bool taosLockNote(int32_t fd, SNoteObj *pNote) {
diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c
index a0af98a9d2..6464224c5e 100644
--- a/src/client/src/tscSub.c
+++ b/src/client/src/tscSub.c
@@ -371,7 +371,7 @@ void tscSaveSubscriptionProgress(void* sub) {
 
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
-  if (!taosMkDir(path)) {
+  if (taosMkDir(path) != 0) {
     tscError("failed to create subscribe dir: %s", path);
   }
 
diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c
index 0ca8e49fa7..badc97ba35 100644
--- a/src/client/src/tscSystem.c
+++ b/src/client/src/tscSystem.c
@@ -148,7 +148,7 @@ void taos_init_imp(void) {
     }
 
     taosReadGlobalCfg();
-    if (taosCheckGlobalCfg()) {
+    if (taosCheckGlobalCfg() != 0) {
       tscInitRes = -1;
       return;
     }
diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c
index 5c9dc0995d..29713686ea 100644
--- a/src/kit/shell/src/shellMain.c
+++ b/src/kit/shell/src/shellMain.c
@@ -106,7 +106,7 @@ int main(int argc, char* argv[]) {
     taosInitGlobalCfg();
     taosReadGlobalLogCfg();
 
-    if (!taosReadGlobalCfg()) {
+    if (taosReadGlobalCfg() ! =0) {
       printf("TDengine read global config failed");
       exit(EXIT_FAILURE);
     }
diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c
index b8e8972d93..547f862c20 100644
--- a/src/tfs/src/tfs.c
+++ b/src/tfs/src/tfs.c
@@ -252,7 +252,7 @@ int tfsMkdirAt(const char *rname, int level, int id) {
   char   aname[TMPNAME_LEN];
 
   snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
-  if (!taosMkDir(aname)) {
+  if (taosMkDir(aname) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h
deleted file mode 100644
index 890b404ce9..0000000000
--- a/src/wal/inc/walInt.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_WAL_INT_H
-#define TDENGINE_WAL_INT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "tlog.h"
-
-extern int32_t wDebugFlag;
-
-#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
-#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
-#define wWarn(...)  { if (wDebugFlag & DEBUG_WARN)  { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
-#define wInfo(...)  { if (wDebugFlag & DEBUG_INFO)  { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
-#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-
-#define WAL_PREFIX     "wal"
-#define WAL_PREFIX_LEN 3
-#define WAL_REFRESH_MS 1000
-#define WAL_MAX_SIZE   (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
-#define WAL_SIGNATURE  ((uint32_t)(0xFAFBFDFE))
-#define WAL_PATH_LEN   (TSDB_FILENAME_LEN + 12)
-#define WAL_FILE_LEN   (WAL_PATH_LEN + 32)
-#define WAL_FILE_NUM   1 // 3
-
-typedef struct {
-  uint64_t version;
-  int64_t  fileId;
-  int64_t  rid;
-  int64_t  tfd;
-  int32_t  vgId;
-  int32_t  keep;
-  int32_t  level;
-  int32_t  fsyncPeriod;
-  int32_t  fsyncSeq;
-  int8_t   stop;
-  int8_t   reserved[3];
-  char     path[WAL_PATH_LEN];
-  char     name[WAL_FILE_LEN];
-  pthread_mutex_t mutex;
-} SWal;
-
-int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId);
-int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId);
-int32_t walGetNewFile(SWal *pWal, int64_t *newFileId);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c
deleted file mode 100644
index 505728fbe4..0000000000
--- a/src/wal/test/waltest.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-//#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tutil.h"
-#include "tglobal.h"
-#include "tlog.h"
-#include "twal.h"
-#include "tfile.h"
-
-int64_t  ver = 0;
-void    *pWal = NULL;
-
-int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
-  // do nothing
-  SWalHead *pHead = data;
-
-  if (pHead->version > ver)
-    ver = pHead->version;
-
-  walWrite(pWal, pHead);
-
-  return 0;
-}
-
-int main(int argc, char *argv[]) {
-  char path[128] = "/tmp/wal";
-  int  level = 2;
-  int  total = 5;
-  int  rows = 10000;
-  int  size = 128;
-  int  keep = 0;
-
-  for (int i=1; i<argc; ++i) {
-    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
-      tstrncpy(path, argv[++i], sizeof(path));
-    } else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
-      level = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
-      rows = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
-      keep = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
-      total = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
-      size = atoi(argv[++i]);
-    } else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
-      ver = atoll(argv[++i]);
-    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
-      dDebugFlag = atoi(argv[++i]);
-    } else {
-      printf("\nusage: %s [options] \n", argv[0]);
-      printf("  [-p path]: wal file path default is:%s\n", path);
-      printf("  [-l level]: log level, default is:%d\n", level);
-      printf("  [-t total]: total wal files, default is:%d\n", total);
-      printf("  [-r rows]: rows of records per wal file, default is:%d\n", rows);
-      printf("  [-k keep]: keep the wal after closing, default is:%d\n", keep);
-      printf("  [-v version]: initial version, default is:%" PRId64 "\n", ver);
-      printf("  [-d debugFlag]: debug flag, default:%d\n", dDebugFlag);
-      printf("  [-h help]: print out this help\n\n");
-      exit(0);
-    }
-  }
-
-  taosInitLog("wal.log", 100000, 10);
-  tfInit();
-  walInit();
-
-  SWalCfg walCfg = {0};
-  walCfg.walLevel = level;
-  walCfg.keep = keep;
-
-  pWal = walOpen(path, &walCfg);
-  if (pWal == NULL) {
-    printf("failed to open wal\n");
-    exit(-1);
-  }
-
-  int ret = walRestore(pWal, NULL, writeToQueue);
-  if (ret <0) {
-    printf("failed to restore wal\n");
-    exit(-1);
-  }
-
-  printf("version starts from:%" PRId64 "\n", ver);
-  
-  int contLen = sizeof(SWalHead) + size;
-  SWalHead *pHead = (SWalHead *) malloc(contLen);
-
-  for (int i=0; i<total; ++i) {
-    for (int k=0; k<rows; ++k) {
-      pHead->version = ++ver;
-      pHead->len = size;
-      walWrite(pWal, pHead);
-    }
-       
-    printf("renew a wal, i:%d\n", i);
-    walRenew(pWal);
-  }
-
-  printf("%d wal files are written\n", total);
-
-  int64_t index = 0;
-  char    name[256];
-
-  while (1) {
-    int code = walGetWalFile(pWal, name, &index);
-    if (code == -1) {
-      printf("failed to get wal file, index:%" PRId64 "\n", index);
-      break;
-    }
-
-    printf("index:%" PRId64 " wal:%s\n", index, name);
-    if (code == 0) break;
-  }
-
-  getchar();
-
-  walClose(pWal);
-  walCleanUp();
-  tfCleanup();
-
-  return 0;
-}