Merge pull request #10847 from taosdata/feature/shm

Refactor the dnode framework code to start taosd as a subprocess

include/common/tglobal.h

@@ -51,6 +51,7 @@ extern int32_t tsCompatibleModel;
 extern bool    tsEnableSlaveQuery;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerDay[3];
+extern int32_t tsMultiProcess;
 
 // monitor
 extern bool     tsEnableMonitor;

include/common/tmsg.h

@@ -697,6 +697,7 @@ typedef struct {
 
 int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
 int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
+void    tFreeSStatusRsp(SStatusRsp* pRsp);
 
 typedef struct {
   int32_t reserved;
@@ -2318,6 +2319,14 @@ typedef struct {
 
 #pragma pack(pop)
 
+struct SRpcMsg;
+struct SEpSet;
+struct SMgmtWrapper;
+typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq);
+typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* rpcMsg);
+typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* rpcMsg);
+typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* rpcMsg);
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/bnode/bnode.h

@@ -21,24 +21,19 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SBnode       SBnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
 } SBnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SBnodeOpt;
 
 /* ------------------------ SBnode ------------------------ */
@@ -76,13 +71,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad);
  */
 int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs);
 
-/**
- * @brief Drop a bnode.
- *
- * @param path Path of the bnode.
- */
-void bndDestroy(const char *path);
-
 #ifdef __cplusplus
 }
 #endif

include/dnode/mgmt/dnode.h

@@ -33,8 +33,7 @@ typedef struct SDnode SDnode;
 int32_t dndInit();
 
 /**
- * @brief clear the environment
+ * @brief Clear the environment
- *
  */
 void dndCleanup();
 
@@ -42,22 +41,24 @@ void dndCleanup();
 typedef struct {
   int32_t   numOfSupportVnodes;
   uint16_t  serverPort;
-  char      dataDir[TSDB_FILENAME_LEN];
+  char      dataDir[PATH_MAX];
   char      localEp[TSDB_EP_LEN];
   char      localFqdn[TSDB_FQDN_LEN];
   char      firstEp[TSDB_EP_LEN];
   char      secondEp[TSDB_EP_LEN];
   SDiskCfg *pDisks;
   int32_t   numOfDisks;
-} SDnodeObjCfg;
+} SDnodeOpt;
+
+typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent;
 
 /**
  * @brief Initialize and start the dnode.
  *
- * @param pCfg Config of the dnode.
+ * @param pOption Option of the dnode.
  * @return SDnode* The dnode object.
  */
-SDnode *dndCreate(SDnodeObjCfg *pCfg);
+SDnode *dndCreate(const SDnodeOpt *pOption);
 
 /**
  * @brief Stop and cleanup the dnode.
@@ -66,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg);
  */
 void dndClose(SDnode *pDnode);
 
+/**
+ * @brief Run dnode until specific event is receive.
+ *
+ * @param pDnode The dnode object to run.
+ */
+int32_t dndRun(SDnode *pDnode);
+
+/**
+ * @brief Handle event in the dnode.
+ *
+ * @param pDnode The dnode object to close.
+ * @param event The event to handle.
+ */
+void dndHandleEvent(SDnode *pDnode, EDndEvent event);
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/mnode/mnode.h

@@ -23,14 +23,8 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode    SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SMnode       SMnode;
-typedef struct SMnodeMsg SMnodeMsg;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
 
 typedef struct {
   int32_t        dnodeId;
@@ -38,12 +32,12 @@ typedef struct {
   int8_t         replica;
   int8_t         selfIndex;
   SReplica       replicas[TSDB_MAX_REPLICA];
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  PutReqToMWriteQFp putReqToMWriteQFp;
+  PutToQueueFp   putToWriteQFp;
-  PutReqToMReadQFp  putReqToMReadQFp;
+  PutToQueueFp   putToReadQFp;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SMnodeOpt;
 
 /* ------------------------ SMnode ------------------------ */
@@ -73,11 +67,11 @@ void mndClose(SMnode *pMnode);
 int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption);
 
 /**
- * @brief Drop a mnode.
+ * @brief Start mnode
  *
- * @param path Path of the mnode.
+ * @param pMnode The mnode object.
  */
-void mndDestroy(const char *path);
+int32_t mndStart(SMnode *pMnode);
 
 /**
  * @brief Get mnode monitor info.
@@ -104,37 +98,13 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
  */
 int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
 
-/**
- * @brief Initialize mnode msg.
- *
- * @param pMnode The mnode object.
- * @param pMsg The request rpc msg.
- * @return int32_t The created mnode msg.
- */
-SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- */
-void mndCleanupMsg(SMnodeMsg *pMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- * @param code The error code.
- */
-void mndSendRsp(SMnodeMsg *pMsg, int32_t code);
-
 /**
  * @brief Process the read, write, sync request.
  *
  * @param pMsg The request msg.
  * @return int32_t 0 for success, -1 for failure.
  */
-void mndProcessMsg(SMnodeMsg *pMsg);
+int32_t mndProcessMsg(SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

include/dnode/qnode/qnode.h

@@ -21,11 +21,8 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SQnode       SQnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
   int64_t numOfStartTask;
@@ -39,13 +36,12 @@ typedef struct {
 } SQnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SQnodeOpt;
 
 /* ------------------------ SQnode ------------------------ */

include/dnode/snode/snode.h

@@ -25,24 +25,19 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SSnode       SSnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
 } SSnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SSnodeOpt;
 
 /* ------------------------ SSnode ------------------------ */
@@ -77,20 +72,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
  * @param pSnode The snode object.
  * @param pMsg The request message
  * @param pRsp The response message
- * @return int32_t 0 for success, -1 for failure
  */
-// int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
+void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
+void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
-int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
-int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
-/**
- * @brief Drop a snode.
- *
- * @param path Path of the snode.
- */
-void sndDestroy(const char *path);
 
 #ifdef __cplusplus
 }

include/libs/qworker/qworker.h

@@ -49,10 +49,10 @@ typedef struct {
 } SQWorkerStat;
 
 typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
-typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *);
+typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *);
 
 int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
-                    putReqToQueryQFp fp1, sendReqToDnodeFp fp2);
+                    putReqToQueryQFp fp1, sendReqFp fp2);
 
 int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 

include/libs/transport/trpc.h

@@ -48,6 +48,14 @@ typedef struct SRpcMsg {
 
 } SRpcMsg;
 
+typedef struct {
+  char    user[TSDB_USER_LEN];
+  SRpcMsg rpcMsg;
+  int32_t rspLen;
+  void   *pRsp;
+  void   *pNode;
+} SNodeMsg;
+
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
   char *   label;         // for debug purpose

include/os/osEnv.h

@@ -45,6 +45,7 @@ extern SDiskSpace tsTempSpace;
 
 void osInit();
 void osUpdate();
+void osCleanup();
 bool osLogSpaceAvailable();
 void osSetTimezone(const char *timezone);
 

include/util/taoserror.h

@@ -75,6 +75,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x010B)
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
 #define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
+#define TSDB_CODE_OUT_OF_SHM_MEM                TAOS_DEF_ERROR_CODE(0, 0x010E)
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
 #define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0112)
@@ -277,34 +278,14 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)
 #define TSDB_CODE_DND_OFFLINE                   TAOS_DEF_ERROR_CODE(0, 0x0401)
 #define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0402)
-#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0410)
+#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x0403)
-#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0411)
+#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x0404)
-#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0420)
+#define TSDB_CODE_NODE_PARSE_FILE_ERROR         TAOS_DEF_ERROR_CODE(0, 0x0405)
-#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0421)
+#define TSDB_CODE_NODE_INVALID_OPTION           TAOS_DEF_ERROR_CODE(0, 0x0406)
-#define TSDB_CODE_DND_MNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0422)
+#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0410)
-#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0423)
+#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0411)
-#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0424)
+#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0412)
-#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0430)
+#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0413)
-#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0431)
-#define TSDB_CODE_DND_QNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0432)
-#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0433)
-#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0434)
-#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0440)
-#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0441)
-#define TSDB_CODE_DND_SNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0442)
-#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0443)
-#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0444)
-#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0450)
-#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0451)
-#define TSDB_CODE_DND_BNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0452)
-#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0453)
-#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0454)
-#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0460)
-#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0461)
-#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0462)
-#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0463)
-#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0464)
-#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0465)
 
 // vnode
 #define TSDB_CODE_VND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0500)

include/util/tprocess.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_PROCESS_H_
+#define _TD_UTIL_PROCESS_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SProcQueue SProcQueue;
+typedef struct SProcObj   SProcObj;
+typedef void *(*ProcMallocFp)(int32_t contLen);
+typedef void *(*ProcFreeFp)(void *pCont);
+typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+
+typedef struct {
+  int32_t       childQueueSize;
+  ProcConsumeFp childConsumeFp;
+  ProcMallocFp  childMallocHeadFp;
+  ProcFreeFp    childFreeHeadFp;
+  ProcMallocFp  childMallocBodyFp;
+  ProcFreeFp    childFreeBodyFp;
+  int32_t       parentQueueSize;
+  ProcConsumeFp parentConsumeFp;
+  ProcMallocFp  parentdMallocHeadFp;
+  ProcFreeFp    parentFreeHeadFp;
+  ProcMallocFp  parentMallocBodyFp;
+  ProcFreeFp    parentFreeBodyFp;
+  bool          testFlag;
+  void         *pParent;
+  const char   *name;
+} SProcCfg;
+
+SProcObj *taosProcInit(const SProcCfg *pCfg);
+void      taosProcCleanup(SProcObj *pProc);
+int32_t   taosProcRun(SProcObj *pProc);
+void      taosProcStop(SProcObj *pProc);
+bool      taosProcIsChild(SProcObj *pProc);
+
+int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_PROCESS_H_*/

include/util/tthread.h

@@ -26,6 +26,8 @@ TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param);
 bool       taosDestoryThread(TdThread* pthread);
 bool       taosThreadRunning(TdThread* pthread);
 
+typedef void *(*ThreadFp)(void *param);
+
 #ifdef __cplusplus
 }
 #endif

source/common/src/tglobal.c

@@ -45,6 +45,7 @@ float   tsRatioOfQueryCores = 1.0f;
 int32_t tsMaxBinaryDisplayWidth = 30;
 bool    tsEnableSlaveQuery = 1;
 bool    tsPrintAuth = 0;
+int32_t tsMultiProcess = 0;
 
 // monitor
 bool     tsEnableMonitor = 1;
@@ -309,7 +310,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1;
-  if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
 
   if (cfgAddString(pCfg, "version", version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1;
@@ -340,6 +340,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1;
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
@@ -403,7 +404,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
     return -1;
   }
 
-  tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
+  tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
   tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
   tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
   tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
@@ -457,6 +458,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
   tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
   tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
+  tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32;
 
   tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
   tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;

source/common/src/tmsg.c

@@ -757,6 +757,8 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
   return 0;
 }
 
+void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); }
+
 int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);

source/dnode/bnode/src/bnode.c

@@ -25,5 +25,3 @@ void bndClose(SBnode *pBnode) { free(pBnode); }
 int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; }
 
 int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; }
-
-void bndDestroy(const char *path) {}

source/dnode/mgmt/CMakeLists.txt

@@ -1,2 +1,30 @@
-add_subdirectory(daemon)
+aux_source_directory(src DNODE_SRC)
-add_subdirectory(impl)
+aux_source_directory(dnode/src DNODE_SRC)
+aux_source_directory(qnode/src DNODE_SRC)
+aux_source_directory(bnode/src DNODE_SRC)
+aux_source_directory(snode/src DNODE_SRC)
+aux_source_directory(vnode/src DNODE_SRC)
+aux_source_directory(mnode/src DNODE_SRC)
+aux_source_directory(container/src DNODE_SRC)
+
+add_library(dnode STATIC ${DNODE_SRC})
+target_link_libraries(
+    dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
+)
+target_include_directories(
+    dnode
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc"
+)
+
+add_subdirectory(main)
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/dnode/mgmt/bnode/inc/bm.h

@@ -16,17 +16,13 @@
 #ifndef _TD_DND_BNODE_H_
 #define _TD_DND_BNODE_H_
 
+#include "dnd.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitBnode(SDnode *pDnode);
+void bmGetMgmtFp(SMgmtWrapper *pWrapper);
-void    dndCleanupBnode(SDnode *pDnode);
-
-void    dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/bnode/inc/bmInt.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_BNODE_INT_H_
+#define _TD_DND_BNODE_INT_H_
+
+#include "bm.h"
+#include "bnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBnodeMgmt {
+  SBnode       *pBnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  writeWorker;
+} SBnodeMgmt;
+
+// bmInt.c
+int32_t bmOpen(SMgmtWrapper *pWrapper);
+int32_t bmDrop(SMgmtWrapper *pWrapper);
+
+// bmMsg.c
+void    bmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// bmWorker.c
+int32_t bmStartWorker(SBnodeMgmt *pMgmt);
+void    bmStopWorker(SBnodeMgmt *pMgmt);
+int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_BNODE_INT_H_*/

source/dnode/mgmt/bnode/src/bmInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t bmOpenImp(SBnodeMgmt *pMgmt) {
+  SBnodeOpt option = {0};
+  bmInitOption(pMgmt, &option);
+
+  pMgmt->pBnode = bndOpen(pMgmt->path, &option);
+  if (pMgmt->pBnode == NULL) {
+    dError("failed to open bnode since %s", terrstr());
+    return -1;
+  }
+
+  if (bmStartWorker(pMgmt) != 0) {
+    dError("failed to start bnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write bnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void bmCloseImp(SBnodeMgmt *pMgmt) {
+  if (pMgmt->pBnode != NULL) {
+    bmStopWorker(pMgmt);
+    bndClose(pMgmt->pBnode);
+    pMgmt->pBnode = NULL;
+  }
+}
+
+int32_t bmDrop(SMgmtWrapper *pWrapper) {
+  SBnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("bnode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop bnode since %s", terrstr());
+    return -1;
+  }
+
+  bmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("bnode-mgmt is dropped");
+  return 0;
+}
+
+static void bmClose(SMgmtWrapper *pWrapper) {
+  SBnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("bnode-mgmt start to cleanup");
+  bmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("bnode-mgmt is cleaned up");
+}
+
+int32_t bmOpen(SMgmtWrapper *pWrapper) {
+  dInfo("bnode-mgmt start to init");
+  SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = bmOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init bnode-mgmt since %s", terrstr());
+    bmClose(pWrapper);
+  } else {
+    dInfo("bnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void bmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = bmOpen;
+  mgmtFp.closeFp = bmClose;
+  mgmtFp.createMsgFp = bmProcessCreateReq;
+  mgmtFp.dropMsgFp = bmProcessDropReq;
+  mgmtFp.requiredFp = bmRequire;
+
+  bmInitMsgHandles(pWrapper);
+  pWrapper->name = "bnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/bnode/src/bmMsg.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateBnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return bmOpen(pWrapper);
+  }
+}
+
+int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropBnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop bnode since %s", terrstr());
+    return -1;
+  } else {
+    return bmDrop(pWrapper);
+  }
+}
+
+void bmInitMsgHandles(SMgmtWrapper *pWrapper) {}

source/dnode/mgmt/bnode/src/bmWorker.c

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+  SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
+  dndSendRsp(pWrapper, &rpcRsp);
+
+  dTrace("msg:%p, is freed", pMsg);
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t numOfMsgs, int32_t code) {
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+    bmSendErrorRsp(pWrapper, pMsg, code);
+  }
+}
+
+static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) {
+  SMgmtWrapper *pWrapper = pMgmt->pWrapper;
+
+  SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
+  if (pArray == NULL) {
+    bmSendErrorRsps(pWrapper, qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
+    return;
+  }
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+    dTrace("msg:%p, will be processed in bnode queue", pMsg);
+    if (taosArrayPush(pArray, &pMsg) == NULL) {
+      bmSendErrorRsp(pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
+
+  bndProcessWMsgs(pMgmt->pBnode, pArray);
+
+  for (size_t i = 0; i < numOfMsgs; i++) {
+    SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
+  }
+  taosArrayDestroy(pArray);
+}
+
+int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->writeWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, bmProcessQueue) != 0) {
+    dError("failed to start bnode write worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void bmStopWorker(SBnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); }

source/dnode/mgmt/container/inc/dnd.h

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_H_
+#define _TD_DND_H_
+
+#include "os.h"
+
+#include "cJSON.h"
+#include "monitor.h"
+#include "tcache.h"
+#include "tcrc32c.h"
+#include "tdatablock.h"
+#include "tglobal.h"
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+#include "tprocess.h"
+#include "tqueue.h"
+#include "trpc.h"
+#include "tthread.h"
+#include "ttime.h"
+#include "tworker.h"
+
+#include "dnode.h"
+#include "tfs.h"
+#include "wal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
+#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
+
+typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType;
+typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus;
+typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus;
+typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
+typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType;
+
+typedef struct SMgmtFp      SMgmtFp;
+typedef struct SMgmtWrapper SMgmtWrapper;
+typedef struct SMsgHandle   SMsgHandle;
+typedef struct SDnodeMgmt   SDnodeMgmt;
+typedef struct SVnodesMgmt  SVnodesMgmt;
+typedef struct SMnodeMgmt   SMnodeMgmt;
+typedef struct SQnodeMgmt   SQnodeMgmt;
+typedef struct SSnodeMgmt   SSnodeMgmt;
+typedef struct SBnodeMgmt   SBnodeMgmt;
+
+typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg);
+typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper);
+typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper);
+typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper);
+typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required);
+
+typedef struct {
+  EWorkerType type;
+  const char *name;
+  int32_t     minNum;
+  int32_t     maxNum;
+  void       *queueFp;
+  void       *param;
+  STaosQueue *queue;
+  union {
+    SQWorkerPool pool;
+    SWWorkerPool mpool;
+  };
+} SDnodeWorker;
+
+typedef struct SMsgHandle {
+  NodeMsgFp     msgFp;
+  SMgmtWrapper *pWrapper;
+} SMsgHandle;
+
+typedef struct SMgmtFp {
+  OpenNodeFp    openFp;
+  CloseNodeFp   closeFp;
+  StartNodeFp   startFp;
+  CreateNodeFp  createMsgFp;
+  DropNodeFp    dropMsgFp;
+  RequireNodeFp requiredFp;
+} SMgmtFp;
+
+typedef struct SMgmtWrapper {
+  const char *name;
+  char       *path;
+  int32_t     refCount;
+  SRWLatch    latch;
+  bool        deployed;
+  bool        required;
+  EProcType   procType;
+  SProcObj   *pProc;
+  void       *pMgmt;
+  SDnode     *pDnode;
+  NodeMsgFp   msgFps[TDMT_MAX];
+  SMgmtFp     fp;
+} SMgmtWrapper;
+
+typedef struct {
+  void      *serverRpc;
+  void      *clientRpc;
+  SMsgHandle msgHandles[TDMT_MAX];
+} STransMgmt;
+
+typedef struct SDnode {
+  int64_t      clusterId;
+  int32_t      dnodeId;
+  int32_t      numOfSupportVnodes;
+  int64_t      rebootTime;
+  char        *localEp;
+  char        *localFqdn;
+  char        *firstEp;
+  char        *secondEp;
+  char        *dataDir;
+  SDiskCfg    *pDisks;
+  int32_t      numOfDisks;
+  uint16_t     serverPort;
+  bool         dropped;
+  EDndStatus   status;
+  EDndEvent    event;
+  EProcType    procType;
+  SStartupReq  startup;
+  TdFilePtr    pLockFile;
+  STransMgmt   trans;
+  SMgmtWrapper wrappers[NODE_MAX];
+} SDnode;
+
+EDndStatus    dndGetStatus(SDnode *pDnode);
+void          dndSetStatus(SDnode *pDnode, EDndStatus stat);
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
+void          dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp);
+void          dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
+void          dndSendMonitorReport(SDnode *pDnode);
+
+int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
+void    dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
+
+int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
+                      int32_t maxNum, void *queueFp);
+void    dndCleanupWorker(SDnodeWorker *pWorker);
+int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg);
+
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
+
+int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
+int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_H_*/

source/dnode/mgmt/container/inc/dndInt.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_INT_H_
+#define _TD_DND_INT_H_
+
+#include "dnd.h"
+
+#include "bm.h"
+#include "dm.h"
+#include "mm.h"
+#include "qm.h"
+#include "sm.h"
+#include "vm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// dndInt.c
+int32_t     dndInit();
+void        dndCleanup();
+const char *dndStatStr(EDndStatus stat);
+void        dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
+TdFilePtr   dndCheckRunning(char *dataDir);
+void        dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
+
+// dndMsg.c
+void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet);
+
+// dndExec.c
+int32_t dndOpenNode(SMgmtWrapper *pWrapper);
+void    dndCloseNode(SMgmtWrapper *pWrapper);
+int32_t dndRun(SDnode *pDnode);
+
+// dndObj.c
+SDnode *dndCreate(const SDnodeOpt *pOption);
+void    dndClose(SDnode *pDnode);
+void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
+
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
+int32_t       dndMarkWrapper(SMgmtWrapper *pWrapper);
+void          dndReleaseWrapper(SMgmtWrapper *pWrapper);
+
+// dndTransport.c
+int32_t dndInitServer(SDnode *pDnode);
+void    dndCleanupServer(SDnode *pDnode);
+int32_t dndInitClient(SDnode *pDnode);
+void    dndCleanupClient(SDnode *pDnode);
+int32_t dndInitMsgHandle(SDnode *pDnode);
+void    dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_INT_H_*/

source/dnode/mgmt/container/src/dndExec.c

@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static void dndResetLog(SMgmtWrapper *pMgmt) {
+  char logname[24] = {0};
+  snprintf(logname, sizeof(logname), "%slog", pMgmt->name);
+
+  dInfo("node:%s, reset log to %s", pMgmt->name, logname);
+  taosCloseLog();
+  taosInitLog(logname, 1);
+}
+
+static bool dndRequireNode(SMgmtWrapper *pWrapper) {
+  bool required = false;
+  int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required);
+  if (!required) {
+    dDebug("node:%s, no need to start", pWrapper->name);
+  } else {
+    dDebug("node:%s, need to start", pWrapper->name);
+  }
+  return required;
+}
+
+int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
+  int32_t code = (*pWrapper->fp.openFp)(pWrapper);
+  if (code != 0) {
+    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
+    return -1;
+  } else {
+    dDebug("node:%s, has been opened", pWrapper->name);
+  }
+
+  pWrapper->deployed = true;
+  return 0;
+}
+
+void dndCloseNode(SMgmtWrapper *pWrapper) {
+  taosWLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    (*pWrapper->fp.closeFp)(pWrapper);
+    pWrapper->deployed = false;
+  }
+  if (pWrapper->pProc) {
+    taosProcCleanup(pWrapper->pProc);
+    pWrapper->pProc = NULL;
+  }
+  taosWUnLockLatch(&pWrapper->latch);
+}
+
+static int32_t dndRunInSingleProcess(SDnode *pDnode) {
+  dInfo("dnode run in single process mode");
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    if (taosMkDir(pWrapper->path) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+      return -1;
+    }
+
+    dInfo("node:%s, will start in single process", pWrapper->name);
+    pWrapper->procType = PROC_SINGLE;
+    if (dndOpenNode(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+    if (pWrapper->fp.startFp == NULL) continue;
+    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
+  dndCleanupServer(pDnode);
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    if (except == n) continue;
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    dndCloseNode(pWrapper);
+  }
+}
+
+static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
+  dTrace("msg:%p, get from child queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  pRpc->pCont = pCont;
+
+  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
+  int32_t   code = (*msgFp)(pWrapper, pMsg);
+
+  if (code != 0) {
+    if (pRpc->msgType & 1U) {
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
+      dndSendRsp(pWrapper, &rsp);
+    }
+
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pCont);
+  }
+}
+
+static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) {
+  dTrace("msg:%p, get from parent queue", pRsp);
+  pRsp->pCont = pCont;
+  dndSendRpcRsp(pWrapper, pRsp);
+  free(pRsp);
+}
+
+static int32_t dndRunInMultiProcess(SDnode *pDnode) {
+  dInfo("dnode run in multi process mode");
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    if (taosMkDir(pWrapper->path) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+      return -1;
+    }
+
+    if (n == DNODE) {
+      dInfo("node:%s, will start in parent process", pWrapper->name);
+      pWrapper->procType = PROC_SINGLE;
+      if (dndOpenNode(pWrapper) != 0) {
+        dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+        return -1;
+      }
+      continue;
+    }
+
+    SProcCfg  cfg = {.childQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
+                     .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
+                     .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
+                     .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
+                     .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                     .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                     .parentQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
+                     .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
+                     .parentdMallocHeadFp = (ProcMallocFp)malloc,
+                     .parentFreeHeadFp = (ProcFreeFp)free,
+                     .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                     .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                     .testFlag = 0,
+                     .pParent = pWrapper,
+                     .name = pWrapper->name};
+    SProcObj *pProc = taosProcInit(&cfg);
+    if (pProc == NULL) {
+      dError("node:%s, failed to fork since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+
+    pWrapper->pProc = pProc;
+
+    if (taosProcIsChild(pProc)) {
+      dInfo("node:%s, will start in child process", pWrapper->name);
+      pWrapper->procType = PROC_CHILD;
+      dndResetLog(pWrapper);
+
+      dInfo("node:%s, clean up resources inherited from parent", pWrapper->name);
+      dndClearNodesExecpt(pDnode, n);
+
+      dInfo("node:%s, will be initialized in child process", pWrapper->name);
+      dndOpenNode(pWrapper);
+    } else {
+      dInfo("node:%s, will not start in parent process", pWrapper->name);
+      pWrapper->procType = PROC_PARENT;
+    }
+
+    if (taosProcRun(pProc) != 0) {
+      dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+#if 0
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+  if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) {
+    dndReleaseWrapper(pWrapper);
+    dError("failed to start dnode worker since %s", terrstr());
+    return -1;
+  }
+
+  dndReleaseWrapper(pWrapper);
+#endif
+  return 0;
+}
+
+int32_t dndRun(SDnode *pDnode) {
+  if (tsMultiProcess == 0) {
+    if (dndRunInSingleProcess(pDnode) != 0) {
+      dError("failed to run dnode in single process mode since %s", terrstr());
+      return -1;
+    }
+  } else {
+    if (dndRunInMultiProcess(pDnode) != 0) {
+      dError("failed to run dnode in multi process mode since %s", terrstr());
+      return -1;
+    }
+  }
+
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      break;
+    }
+    taosMsleep(100);
+  }
+
+  return 0;
+}

source/dnode/mgmt/container/src/dndFile.c

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 1024;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto _OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto _OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto _OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_Number) {
+    dError("failed to read %s since deployed not found", file);
+    goto _OVER;
+  }
+  *pDeployed = deployed->valueint != 0;
+
+  code = 0;
+  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+
+_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 1024;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d\n", deployed);
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s, deployed:%d", realfile, deployed);
+  return 0;
+}

source/dnode/mgmt/container/src/dndInt.c

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int8_t once = DND_ENV_INIT;
+
+int32_t dndInit() {
+  dDebug("start to init dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
+    terrno = TSDB_CODE_REPEAT_INIT;
+    dError("failed to init dnode env since %s", terrstr());
+    return -1;
+  }
+
+  taosIgnSIGPIPE();
+  taosBlockSIGPIPE();
+  taosResolveCRC();
+
+  if (rpcInit() != 0) {
+    dError("failed to init rpc since %s", terrstr());
+    dndCleanup();
+    return -1;
+  }
+
+  SMonCfg monCfg = {0};
+  monCfg.maxLogs = tsMonitorMaxLogs;
+  monCfg.port = tsMonitorPort;
+  monCfg.server = tsMonitorFqdn;
+  monCfg.comp = tsMonitorComp;
+  if (monInit(&monCfg) != 0) {
+    dError("failed to init monitor since %s", terrstr());
+    dndCleanup();
+    return -1;
+  }
+
+  dInfo("dnode env is initialized");
+  return 0;
+}
+
+void dndCleanup() {
+  dDebug("start to cleanup dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
+    dError("dnode env is already cleaned up");
+    return;
+  }
+
+  monCleanup();
+  rpcCleanup();
+  walCleanUp();
+  taosStopCacheRefreshWorker();
+  dInfo("dnode env is cleaned up");
+}
+
+void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) {
+  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
+}
+
+EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
+
+void dndSetStatus(SDnode *pDnode, EDndStatus status) {
+  if (pDnode->status != status) {
+    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
+    pDnode->status = status;
+  }
+}
+
+const char *dndStatStr(EDndStatus status) {
+  switch (status) {
+    case DND_STAT_INIT:
+      return "init";
+    case DND_STAT_RUNNING:
+      return "running";
+    case DND_STAT_STOPPED:
+      return "stopped";
+    default:
+      return "unknown";
+  }
+}
+
+void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
+  SStartupReq *pStartup = &pDnode->startup;
+  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
+  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
+  pStartup->finished = 0;
+}
+
+void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
+  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
+  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
+}
+
+TdFilePtr dndCheckRunning(char *dataDir) {
+  char filepath[PATH_MAX] = {0};
+  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
+
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    return NULL;
+  }
+
+  int32_t ret = taosLockFile(pFile);
+  if (ret != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    taosCloseFile(&pFile);
+    return NULL;
+  }
+
+  dDebug("file:%s is locked", filepath);
+  return pFile;
+}
+
+void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
+  dDebug("startup req is received");
+
+  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
+  dndGetStartup(pDnode, pStartup);
+
+  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
+
+  SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)};
+  rpcSendResponse(&rpcRsp);
+}

source/dnode/mgmt/container/src/dndMonitor.c

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
+  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
+  pInfo->logdir.size = tsLogSpace.size;
+  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
+  pInfo->tempdir.size = tsTempSpace.size;
+
+  return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo);
+}
+
+static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
+  pInfo->protocol = 1;
+  pInfo->dnode_id = pDnode->dnodeId;
+  pInfo->cluster_id = pDnode->clusterId;
+  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+}
+
+static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
+  pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
+  taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
+  taosGetCpuCores(&pInfo->cpu_cores);
+  taosGetProcMemory(&pInfo->mem_engine);
+  taosGetSysMemory(&pInfo->mem_system);
+  pInfo->mem_total = tsTotalMemoryKB;
+  pInfo->disk_engine = 0;
+  pInfo->disk_used = tsDataSpace.size.used;
+  pInfo->disk_total = tsDataSpace.size.total;
+  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
+  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
+
+  vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo);
+  pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required);
+}
+
+void dndSendMonitorReport(SDnode *pDnode) {
+  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+  SMonInfo *pMonitor = monCreateMonitorInfo();
+  if (pMonitor == NULL) return;
+
+  SMonBasicInfo basicInfo = {0};
+  dndGetMonitorBasicInfo(pDnode, &basicInfo);
+  monSetBasicInfo(pMonitor, &basicInfo);
+
+  SMonClusterInfo clusterInfo = {0};
+  SMonVgroupInfo  vgroupInfo = {0};
+  SMonGrantInfo   grantInfo = {0};
+  if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
+    monSetClusterInfo(pMonitor, &clusterInfo);
+    monSetVgroupInfo(pMonitor, &vgroupInfo);
+    monSetGrantInfo(pMonitor, &grantInfo);
+  }
+
+  SMonDnodeInfo dnodeInfo = {0};
+  dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
+  monSetDnodeInfo(pMonitor, &dnodeInfo);
+
+  SMonDiskInfo diskInfo = {0};
+  if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
+    monSetDiskInfo(pMonitor, &diskInfo);
+  }
+
+  taosArrayDestroy(clusterInfo.dnodes);
+  taosArrayDestroy(clusterInfo.mnodes);
+  taosArrayDestroy(vgroupInfo.vgroups);
+  taosArrayDestroy(diskInfo.datadirs);
+
+  monSendReport(pMonitor);
+  monCleanupMonitorInfo(pMonitor);
+}

source/dnode/mgmt/container/src/dndMsg.c

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+  if (pWrapper != NULL) {
+    dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet);
+  }
+  dndReleaseWrapper(pWrapper);
+}
+
+static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
+  if (msgFp == NULL) {
+    terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+  }
+
+  return msgFp;
+}
+
+static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
+  SRpcConnInfo connInfo = {0};
+  if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) {
+    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+    dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle);
+    return -1;
+  }
+
+  memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
+  memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
+
+  return 0;
+}
+
+void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
+  if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) {
+    dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet);
+  }
+
+  int32_t   code = -1;
+  SNodeMsg *pMsg = NULL;
+  NodeMsgFp msgFp = NULL;
+
+  if (dndMarkWrapper(pWrapper) != 0) goto _OVER;
+  if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
+  if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
+  if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER;
+
+  dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
+  if (pWrapper->procType == PROC_SINGLE) {
+    code = (*msgFp)(pWrapper->pMgmt, pMsg);
+  } else if (pWrapper->procType == PROC_PARENT) {
+    code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen);
+  } else {
+  }
+
+_OVER:
+  if (code == 0) {
+    if (pWrapper->procType == PROC_PARENT) {
+      dTrace("msg:%p, is freed", pMsg);
+      taosFreeQitem(pMsg);
+      rpcFreeCont(pRpc->pCont);
+    }
+  } else {
+    dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr());
+    if (pRpc->msgType & 1U) {
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
+      dndSendRsp(pWrapper, &rsp);
+    }
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pRpc->pCont);
+  }
+
+  dndReleaseWrapper(pWrapper);
+}
+
+static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper != NULL) {
+    dndReleaseWrapper(pWrapper);
+    terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+    dError("failed to create node since %s", terrstr());
+    return -1;
+  }
+
+  pWrapper = &pDnode->wrappers[ntype];
+
+  if (taosMkDir(pWrapper->path) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+    return -1;
+  }
+
+  int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
+  } else {
+    dDebug("node:%s, has been opened", pWrapper->name);
+    pWrapper->deployed = true;
+  }
+
+  return code;
+}
+
+static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper == NULL) {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    dError("failed to drop node since %s", terrstr());
+    return -1;
+  }
+
+  taosWLockLatch(&pWrapper->latch);
+  pWrapper->deployed = false;
+
+  int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    pWrapper->deployed = true;
+    dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
+  } else {
+    pWrapper->deployed = false;
+    dDebug("node:%s, has been dropped", pWrapper->name);
+  }
+
+  taosWUnLockLatch(&pWrapper->latch);
+  dndReleaseWrapper(pWrapper);
+  return code;
+}
+
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) {
+  switch (pMsg->rpcMsg.msgType) {
+    case TDMT_DND_CREATE_MNODE:
+      return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_DROP_MNODE:
+      return dndProcessDropNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_CREATE_QNODE:
+      return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_DROP_QNODE:
+      return dndProcessDropNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_CREATE_SNODE:
+      return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_DROP_SNODE:
+      return dndProcessDropNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_CREATE_BNODE:
+      return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg);
+    case TDMT_DND_DROP_BNODE:
+      return dndProcessDropNodeMsg(pDnode, BNODE, pMsg);
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      return -1;
+  }
+}

source/dnode/mgmt/container/src/dndObj.c

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) {
+  pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
+  pDnode->serverPort = pOption->serverPort;
+  pDnode->dataDir = strdup(pOption->dataDir);
+  pDnode->localEp = strdup(pOption->localEp);
+  pDnode->localFqdn = strdup(pOption->localFqdn);
+  pDnode->firstEp = strdup(pOption->firstEp);
+  pDnode->secondEp = strdup(pOption->secondEp);
+  pDnode->pDisks = pOption->pDisks;
+  pDnode->numOfDisks = pOption->numOfDisks;
+  pDnode->rebootTime = taosGetTimestampMs();
+
+  if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
+      pDnode->secondEp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndClearMemory(SDnode *pDnode) {
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
+    tfree(pMgmt->path);
+  }
+  if (pDnode->pLockFile != NULL) {
+    taosUnLockFile(pDnode->pLockFile);
+    taosCloseFile(&pDnode->pLockFile);
+    pDnode->pLockFile = NULL;
+  }
+  tfree(pDnode->localEp);
+  tfree(pDnode->localFqdn);
+  tfree(pDnode->firstEp);
+  tfree(pDnode->secondEp);
+  tfree(pDnode->dataDir);
+  free(pDnode);
+  dDebug("dnode object memory is cleared, data:%p", pDnode);
+}
+
+SDnode *dndCreate(const SDnodeOpt *pOption) {
+  dInfo("start to create dnode object");
+  int32_t code = -1;
+  char    path[PATH_MAX];
+  SDnode *pDnode = NULL;
+
+  pDnode = calloc(1, sizeof(SDnode));
+  if (pDnode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _OVER;
+  }
+
+  if (dndInitMemory(pDnode, pOption) != 0) {
+    goto _OVER;
+  }
+
+  dndSetStatus(pDnode, DND_STAT_INIT);
+  pDnode->pLockFile = dndCheckRunning(pDnode->dataDir);
+  if (pDnode->pLockFile == NULL) {
+    goto _OVER;
+  }
+
+  if (dndInitServer(pDnode) != 0) {
+    dError("failed to init trans server since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (dndInitClient(pDnode) != 0) {
+    dError("failed to init trans client since %s", terrstr());
+    goto _OVER;
+  }
+
+  dmGetMgmtFp(&pDnode->wrappers[DNODE]);
+  mmGetMgmtFp(&pDnode->wrappers[MNODE]);
+  vmGetMgmtFp(&pDnode->wrappers[VNODES]);
+  qmGetMgmtFp(&pDnode->wrappers[QNODE]);
+  smGetMgmtFp(&pDnode->wrappers[SNODE]);
+  bmGetMgmtFp(&pDnode->wrappers[BNODE]);
+
+  if (dndInitMsgHandle(pDnode) != 0) {
+    goto _OVER;
+  }
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
+    pWrapper->path = strdup(path);
+    pWrapper->pDnode = pDnode;
+    if (pWrapper->path == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _OVER;
+    }
+
+    pWrapper->procType = PROC_SINGLE;
+    taosInitRWLatch(&pWrapper->latch);
+  }
+
+  code = 0;
+
+_OVER:
+  if (code != 0 && pDnode) {
+    dndClearMemory(pDnode);
+    dError("failed to create dnode object since %s", terrstr());
+  } else {
+    dInfo("dnode object is created, data:%p", pDnode);
+  }
+
+  return pDnode;
+}
+
+void dndClose(SDnode *pDnode) {
+  if (pDnode == NULL) return;
+
+  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
+    dError("dnode is shutting down, data:%p", pDnode);
+    return;
+  }
+
+  dInfo("start to close dnode, data:%p", pDnode);
+  dndSetStatus(pDnode, DND_STAT_STOPPED);
+
+  dndCleanupServer(pDnode);
+  dndCleanupClient(pDnode);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    dndCloseNode(pWrapper);
+  }
+
+  dndClearMemory(pDnode);
+  dInfo("dnode object is closed, data:%p", pDnode);
+}
+
+void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
+  dInfo("dnode object receive event %d, data:%p", event, pDnode);
+  pDnode->event = event;
+}
+
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
+  SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+  SMgmtWrapper *pRetWrapper = pWrapper;
+
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    pRetWrapper = NULL;
+  }
+  taosRUnLockLatch(&pWrapper->latch);
+
+  return pRetWrapper;
+}
+
+int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
+  int32_t code = 0;
+
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    code = -1;
+  }
+  taosRUnLockLatch(&pWrapper->latch);
+
+  return code;
+}
+
+void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
+  if (pWrapper == NULL) return;
+
+  taosRLockLatch(&pWrapper->latch);
+  int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
+  taosRUnLockLatch(&pWrapper->latch);
+  dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
+}

source/dnode/mgmt/container/src/dndTransport.c

@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+#define INTERNAL_USER   "_dnd"
+#define INTERNAL_CKEY   "_key"
+#define INTERNAL_SECRET "_pwd"
+
+static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
+  SDnode     *pDnode = parent;
+  STransMgmt *pMgmt = &pDnode->trans;
+  tmsg_t      msgType = pRsp->msgType;
+
+  if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+    // if (pRsp == NULL || pRsp->pCont == NULL) return;
+    dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
+    rpcFreeCont(pRsp->pCont);
+    return;
+  }
+
+  SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
+  if (pHandle->msgFp != NULL) {
+    dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType),
+           pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code));
+    dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet);
+  } else {
+    dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
+    rpcFreeCont(pRsp->pCont);
+  }
+}
+
+int32_t dndInitClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.label = "DND";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp = dndProcessResponse;
+  rpcInit.sessions = 1024;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.user = INTERNAL_USER;
+  rpcInit.ckey = INTERNAL_CKEY;
+  rpcInit.spi = 1;
+  rpcInit.parent = pDnode;
+
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
+  rpcInit.secret = pass;
+
+  pMgmt->clientRpc = rpcOpen(&rpcInit);
+  if (pMgmt->clientRpc == NULL) {
+    dError("failed to init dnode rpc client");
+    return -1;
+  }
+
+  dDebug("dnode rpc client is initialized");
+  return 0;
+}
+
+void dndCleanupClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+  if (pMgmt->clientRpc) {
+    rpcClose(pMgmt->clientRpc);
+    pMgmt->clientRpc = NULL;
+    dDebug("dnode rpc client is closed");
+  }
+}
+
+static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
+  SDnode     *pDnode = param;
+  STransMgmt *pMgmt = &pDnode->trans;
+  tmsg_t      msgType = pReq->msgType;
+
+  if (msgType == TDMT_DND_NETWORK_TEST) {
+    dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle);
+    dndProcessStartupReq(pDnode, pReq);
+    return;
+  }
+
+  if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+    dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pReq->pCont);
+    return;
+  }
+
+  if (pReq->pCont == NULL) {
+    dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    return;
+  }
+
+  SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
+  if (pHandle->msgFp != NULL) {
+    dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name,
+           pReq->handle, pReq->ahandle);
+    dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet);
+  } else {
+    dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pReq->pCont);
+  }
+}
+
+static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  SEpSet epSet = {0};
+  dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
+  rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
+}
+
+static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  int32_t code = 0;
+  char    pass[TSDB_PASSWORD_LEN + 1] = {0};
+
+  if (strcmp(user, INTERNAL_USER) == 0) {
+    taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
+  } else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
+    taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
+  } else {
+    code = -1;
+  }
+
+  if (code == 0) {
+    memcpy(secret, pass, TSDB_PASSWORD_LEN);
+    *spi = 1;
+    *encrypt = 0;
+    *ckey = 0;
+  }
+
+  return code;
+}
+
+static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SDnode *pDnode = parent;
+
+  if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) {
+    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
+    return 0;
+  }
+
+  if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) {
+    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
+    return 0;
+  }
+
+  if (terrno != TSDB_CODE_APP_NOT_READY) {
+    dTrace("failed to get user auth from mnode since %s", terrstr());
+    return -1;
+  }
+
+  SAuthReq authReq = {0};
+  tstrncpy(authReq.user, user, TSDB_USER_LEN);
+  int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSAuthReq(pReq, contLen, &authReq);
+
+  SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
+  SRpcMsg rpcRsp = {0};
+  dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
+  dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
+
+  if (rpcRsp.code != 0) {
+    terrno = rpcRsp.code;
+    dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
+  } else {
+    SAuthRsp authRsp = {0};
+    tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
+    memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
+    memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
+    *spi = authRsp.spi;
+    *encrypt = authRsp.encrypt;
+    dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
+           authRsp.encrypt);
+  }
+
+  rpcFreeCont(rpcRsp.pCont);
+  return rpcRsp.code;
+}
+
+int32_t dndInitServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
+  if (numOfThreads < 1) {
+    numOfThreads = 1;
+  }
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort = pDnode->serverPort;
+  rpcInit.label = "DND";
+  rpcInit.numOfThreads = numOfThreads;
+  rpcInit.cfp = dndProcessRequest;
+  rpcInit.sessions = tsMaxShellConns;
+  rpcInit.connType = TAOS_CONN_SERVER;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.afp = dndRetrieveUserAuthInfo;
+  rpcInit.parent = pDnode;
+
+  pMgmt->serverRpc = rpcOpen(&rpcInit);
+  if (pMgmt->serverRpc == NULL) {
+    dError("failed to init dnode rpc server");
+    return -1;
+  }
+
+  dDebug("dnode rpc server is initialized");
+  return 0;
+}
+
+void dndCleanupServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+  if (pMgmt->serverRpc) {
+    rpcClose(pMgmt->serverRpc);
+    pMgmt->serverRpc = NULL;
+    dDebug("dnode rpc server is closed");
+  }
+}
+
+int32_t dndInitMsgHandle(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+
+    for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) {
+      NodeMsgFp msgFp = pWrapper->msgFps[msgIndex];
+      if (msgFp == NULL) continue;
+
+      SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex];
+      if (pHandle->msgFp != NULL) {
+        dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex],
+               pHandle->pWrapper->name, pWrapper->name);
+        return -1;
+      } else {
+        dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name);
+        pHandle->msgFp = msgFp;
+        pHandle->pWrapper = pWrapper;
+      }
+    }
+  }
+
+  return 0;
+}
+
+static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
+  if (pMgmt->clientRpc == NULL) {
+    terrno = TSDB_CODE_DND_OFFLINE;
+    return -1;
+  }
+
+  rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL);
+  return 0;
+}
+
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) {
+  if (pWrapper->procType == PROC_CHILD) {
+  } else {
+    SDnode *pDnode = pWrapper->pDnode;
+    if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+      terrno = TSDB_CODE_DND_OFFLINE;
+      dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle);
+      return -1;
+    }
+    return dndSendRpcReq(&pDnode->trans, pEpSet, pReq);
+  }
+}
+
+int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
+  if (pWrapper->procType == PROC_CHILD) {
+  } else {
+    SDnode     *pDnode = pWrapper->pDnode;
+    STransMgmt *pTrans = &pDnode->trans;
+    SEpSet      epSet = {0};
+    dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
+    return dndSendRpcReq(pTrans, &epSet, pReq);
+  }
+}
+
+void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+  if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
+    SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
+    dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp);
+  } else {
+    rpcSendResponse(pRsp);
+  }
+}
+
+void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+  if (pWrapper->procType == PROC_CHILD) {
+    int32_t code = -1;
+    do {
+      code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen);
+      if (code != 0) {
+        taosMsleep(10);
+      }
+    } while (code != 0);
+  } else {
+    dndSendRpcRsp(pWrapper, pRsp);
+  }
+}

source/dnode/mgmt/container/src/dndWorker.c

@@ -14,11 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dndWorker.h"
+#include "dndInt.h"
 
-int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
+int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
                       int32_t maxNum, void *queueFp) {
-  if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
+  if (pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return -1;
   }
@@ -28,7 +28,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
   pWorker->minNum = minNum;
   pWorker->maxNum = maxNum;
   pWorker->queueFp = queueFp;
-  pWorker->pDnode = pDnode;
+  pWorker->param = param;
 
   if (pWorker->type == DND_WORKER_SINGLE) {
     SQWorkerPool *pPool = &pWorker->pool;
@@ -39,7 +39,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp);
+    pWorker->queue = tQWorkerAllocQueue(pPool, param, (FItem)queueFp);
     if (pWorker->queue == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -52,7 +52,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp);
+    pWorker->queue = tWWorkerAllocQueue(pPool, param, (FItems)queueFp);
     if (pWorker->queue == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -65,6 +65,8 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
 }
 
 void dndCleanupWorker(SDnodeWorker *pWorker) {
+  if (pWorker->queue == NULL) return;
+
   while (!taosQueueEmpty(pWorker->queue)) {
     taosMsleep(10);
   }
@@ -79,31 +81,13 @@ void dndCleanupWorker(SDnodeWorker *pWorker) {
   }
 }
 
-int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) {
+int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg) {
   if (pWorker == NULL || pWorker->queue == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return -1;
   }
 
-  void *pMsg = NULL;
-  if (contLen != 0) {
-    pMsg = taosAllocateQitem(contLen);
-    if (pMsg != NULL) {
-      memcpy(pMsg, pCont, contLen);
-    }
-  } else {
-    pMsg = pCont;
-  }
-
-  if (pMsg == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
   if (taosWriteQitem(pWorker->queue, pMsg) != 0) {
-    if (contLen != 0) {
-      taosFreeQitem(pMsg);
-    }
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }

source/dnode/mgmt/daemon/src/dmnCfg.c

@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dmnInt.h"
-#include "tconfig.h"
-
-SDnodeObjCfg dmnGetObjCfg() {
-  SConfig *pCfg = taosGetCfg();
-  SDnodeObjCfg objCfg = {0};
-
-  objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
-  tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir));
-  tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp));
-  tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp));
-  objCfg.serverPort = tsServerPort;
-  tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn));
-  snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort);
-  objCfg.pDisks = tsDiskCfg;
-  objCfg.numOfDisks = tsDiskCfgNum;
-  return objCfg;
-}
-
-void dmnDumpCfg() {
-  SConfig *pCfg = taosGetCfg();
-  cfgDumpCfg(pCfg, 0, 1);
-}

source/dnode/mgmt/daemon/src/dmnMain.c

@@ -1,136 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dmnInt.h"
-
-static struct {
-  bool stop;
-  bool dumpConfig;
-  bool generateGrant;
-  bool printAuth;
-  bool printVersion;
-  char envFile[PATH_MAX];
-  char apolloUrl[PATH_MAX];
-} dmn = {0};
-
-static void dmnSigintHandle(int signum, void *info, void *ctx) {
-  uInfo("signal:%d is received", signum);
-  dmn.stop = true;
-}
-
-static void dmnSetSignalHandle() {
-  taosSetSignal(SIGTERM, dmnSigintHandle);
-  taosSetSignal(SIGHUP, dmnSigintHandle);
-  taosSetSignal(SIGINT, dmnSigintHandle);
-  taosSetSignal(SIGABRT, dmnSigintHandle);
-  taosSetSignal(SIGBREAK, dmnSigintHandle);
-}
-
-static void dmnWaitSignal() {
-  dmnSetSignalHandle();
-  while (!dmn.stop) {
-    taosMsleep(100);
-  }
-}
-
-static int32_t dmnParseOption(int32_t argc, char const *argv[]) {
-  for (int32_t i = 1; i < argc; ++i) {
-    if (strcmp(argv[i], "-c") == 0) {
-      if (i < argc - 1) {
-        if (strlen(argv[++i]) >= PATH_MAX) {
-          printf("config file path overflow");
-          return -1;
-        }
-        tstrncpy(configDir, argv[i], PATH_MAX);
-      } else {
-        printf("'-c' requires a parameter, default is %s\n", configDir);
-        return -1;
-      }
-    } else if (strcmp(argv[i], "-C") == 0) {
-      dmn.dumpConfig = true;
-    } else if (strcmp(argv[i], "-k") == 0) {
-      dmn.generateGrant = true;
-    } else if (strcmp(argv[i], "-V") == 0) {
-      dmn.printVersion = true;
-    } else {
-    }
-  }
-
-  return 0;
-}
-
-int32_t dmnRunDnode() {
-  if (dndInit() != 0) {
-    uInfo("Failed to start TDengine, please check the log");
-    return -1;
-  }
-
-  SDnodeObjCfg objCfg = dmnGetObjCfg();
-  SDnode      *pDnode = dndCreate(&objCfg);
-  if (pDnode == NULL) {
-    uInfo("Failed to start TDengine, please check the log");
-    return -1;
-  }
-
-  uInfo("Started TDengine service successfully.");
-  dmnWaitSignal();
-  uInfo("TDengine is shut down!");
-
-  dndClose(pDnode);
-  dndCleanup();
-  taosCloseLog();
-  taosCleanupCfg();
-  return 0;
-}
-
-int main(int argc, char const *argv[]) {
-  if (!taosCheckSystemIsSmallEnd()) {
-    uError("TDengine does not run on non-small-end machines.");
-    return -1;
-  }
-
-  if (dmnParseOption(argc, argv) != 0) {
-    return -1;
-  }
-
-  if (dmn.generateGrant) {
-    dmnGenerateGrant();
-    return 0;
-  }
-
-  if (dmn.printVersion) {
-    dmnPrintVersion();
-    return 0;
-  }
-
-  if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) {
-    uInfo("Failed to start TDengine since read config error");
-    return -1;
-  }
-
-  if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) {
-    uInfo("Failed to start TDengine since read config error");
-    return -1;
-  }
-
-  if (dmn.dumpConfig) {
-    dmnDumpCfg();
-    taosCleanupCfg();
-    return 0;
-  }
-
-  return dmnRunDnode();
-}

source/dnode/mgmt/dnode/inc/dm.h

@@ -13,21 +13,26 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_WORKER_H_
+#ifndef _TD_DND_DNODE_H_
-#define _TD_DND_WORKER_H_
+#define _TD_DND_DNODE_H_
+
+#include "dnd.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
+typedef struct SDnodeMgmt SDnodeMgmt;
-                      int32_t maxNum, void *queueFp);
+
-void    dndCleanupWorker(SDnodeWorker *pWorker);
+void dmGetMgmtFp(SMgmtWrapper *pWrapper);
-int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen);
+void dmInitMsgHandles(SMgmtWrapper *pWrapper);
+
+void    dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void    dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void    dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_WORKER_H_*/
+#endif /*_TD_DND_DNODE_H_*/

source/dnode/mgmt/dnode/inc/dmInt.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_DNODE_INT_H_
+#define _TD_DND_DNODE_INT_H_
+
+#include "dm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SDnodeMgmt {
+  int64_t       dver;
+  int64_t       updateTime;
+  int8_t        statusSent;
+  SEpSet        mnodeEpSet;
+  SHashObj     *dnodeHash;
+  SArray       *dnodeEps;
+  TdThread    *threadId;
+  SRWLatch      latch;
+  SDnodeWorker  mgmtWorker;
+  SDnodeWorker  statusWorker;
+  const char   *path;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+} SDnodeMgmt;
+
+// dmFile.c
+int32_t dmReadFile(SDnodeMgmt *pMgmt);
+int32_t dmWriteFile(SDnodeMgmt *pMgmt);
+void    dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps);
+
+// dmMsg.c
+void    dmSendStatusReq(SDnodeMgmt *pMgmt);
+int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+// dmWorker.c
+int32_t dmStartWorker(SDnodeMgmt *pMgmt);
+void    dmStopWorker(SDnodeMgmt *pMgmt);
+int32_t dmStartThread(SDnodeMgmt *pMgmt);
+int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_DNODE_INT_H_*/

source/dnode/mgmt/dnode/src/dmFile.c

@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+
+static void dmPrintDnodes(SDnodeMgmt *pMgmt);
+static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep);
+static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps);
+
+int32_t dmReadFile(SDnodeMgmt *pMgmt) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 256 * 1024;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+  SDnode   *pDnode = pMgmt->pDnode;
+
+  pMgmt->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp));
+  if (pMgmt->dnodeEps == NULL) {
+    dError("failed to calloc dnodeEp array since %s", strerror(errno));
+    goto PRASE_DNODE_OVER;
+  }
+
+  snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_DNODE_OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_Number) {
+    dError("failed to read %s since dnodeId not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dnodeId = dnodeId->valueint;
+
+  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_String) {
+    dError("failed to read %s since clusterId not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->clusterId = atoll(clusterId->valuestring);
+
+  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
+  if (!dropped || dropped->type != cJSON_Number) {
+    dError("failed to read %s since dropped not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dropped = dropped->valueint;
+
+  cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes");
+  if (!dnodes || dnodes->type != cJSON_Array) {
+    dError("failed to read %s since dnodes not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  int32_t numOfDnodes = cJSON_GetArraySize(dnodes);
+  if (numOfDnodes <= 0) {
+    dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes);
+    goto PRASE_DNODE_OVER;
+  }
+
+  for (int32_t i = 0; i < numOfDnodes; ++i) {
+    cJSON *node = cJSON_GetArrayItem(dnodes, i);
+    if (node == NULL) break;
+
+    SDnodeEp dnodeEp = {0};
+
+    cJSON *did = cJSON_GetObjectItem(node, "id");
+    if (!did || did->type != cJSON_Number) {
+      dError("failed to read %s since dnodeId not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+
+    dnodeEp.id = dnodeId->valueint;
+
+    cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
+    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+      dError("failed to read %s since dnodeFqdn not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+    tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
+    if (!dnodePort || dnodePort->type != cJSON_Number) {
+      dError("failed to read %s since dnodePort not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+
+    dnodeEp.ep.port = dnodePort->valueint;
+
+    cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
+    if (!isMnode || isMnode->type != cJSON_Number) {
+      dError("failed to read %s since isMnode not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+    dnodeEp.isMnode = isMnode->valueint;
+
+    taosArrayPush(pMgmt->dnodeEps, &dnodeEp);
+  }
+
+  code = 0;
+  dInfo("succcessed to read file %s", file);
+  dmPrintDnodes(pMgmt);
+
+PRASE_DNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  if (dmIsEpChanged(pMgmt, pDnode->dnodeId, pDnode->localEp)) {
+    dError("localEp %s different with %s and need reconfigured", pDnode->localEp, file);
+    return -1;
+  }
+
+  if (taosArrayGetSize(pMgmt->dnodeEps) == 0) {
+    SDnodeEp dnodeEp = {0};
+    dnodeEp.isMnode = 1;
+    taosGetFqdnPortFromEp(pDnode->firstEp, &dnodeEp.ep);
+    taosArrayPush(pMgmt->dnodeEps, &dnodeEp);
+  }
+
+  dmResetDnodes(pMgmt, pMgmt->dnodeEps);
+
+  terrno = code;
+  return code;
+}
+
+int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    dError("failed to write %s since %s", file, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 256 * 1024;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", pDnode->dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pDnode->dropped);
+  len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
+
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  for (int32_t i = 0; i < numOfEps; ++i) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->dnodeEps, i);
+    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pDnodeEp->id);
+    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
+    len += snprintf(content + len, maxLen - len, "    \"port\": %u,\n", pDnodeEp->ep.port);
+    len += snprintf(content + len, maxLen - len, "    \"isMnode\": %d\n", pDnodeEp->isMnode);
+    if (i < numOfEps - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  pMgmt->updateTime = taosGetTimestampMs();
+  dDebug("successed to write %s", file);
+  return 0;
+}
+
+void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) {
+  int32_t numOfEps = taosArrayGetSize(dnodeEps);
+  if (numOfEps <= 0) return;
+
+  taosWLockLatch(&pMgmt->latch);
+
+  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  if (numOfEps != numOfEpsOld) {
+    dmResetDnodes(pMgmt, dnodeEps);
+    dmWriteFile(pMgmt);
+  } else {
+    int32_t size = numOfEps * sizeof(SDnodeEp);
+    if (memcmp(pMgmt->dnodeEps->pData, dnodeEps->pData, size) != 0) {
+      dmResetDnodes(pMgmt, dnodeEps);
+      dmWriteFile(pMgmt);
+    }
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+}
+
+static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps) {
+  if (pMgmt->dnodeEps != dnodeEps) {
+    SArray *tmp = pMgmt->dnodeEps;
+    pMgmt->dnodeEps = taosArrayDup(dnodeEps);
+    taosArrayDestroy(tmp);
+  }
+
+  pMgmt->mnodeEpSet.inUse = 0;
+  pMgmt->mnodeEpSet.numOfEps = 0;
+
+  int32_t mIndex = 0;
+  int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps);
+
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i);
+    if (!pDnodeEp->isMnode) continue;
+    if (mIndex >= TSDB_MAX_REPLICA) continue;
+    pMgmt->mnodeEpSet.numOfEps++;
+
+    pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep;
+    mIndex++;
+  }
+
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i);
+    taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
+  }
+
+  dmPrintDnodes(pMgmt);
+}
+
+static void dmPrintDnodes(SDnodeMgmt *pMgmt) {
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  dDebug("print dnode ep list, num:%d", numOfEps);
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pEp = taosArrayGet(pMgmt->dnodeEps, i);
+    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
+  }
+}
+
+static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) {
+  bool changed = false;
+  taosRLockLatch(&pMgmt->latch);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    char epstr[TSDB_EP_LEN + 1];
+    snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
+    changed = strcmp(ep, epstr) != 0;
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+  return changed;
+}

source/dnode/mgmt/dnode/src/dmInt.c

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+
+void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) {
+  taosRLockLatch(&pMgmt->latch);
+  *pEpSet = pMgmt->mnodeEpSet;
+  taosRUnLockLatch(&pMgmt->latch);
+}
+
+void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) {
+  dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
+
+  taosWLockLatch(&pMgmt->latch);
+  pMgmt->mnodeEpSet = *pEpSet;
+  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
+    dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+}
+
+void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
+  SDnodeMgmt *pMgmt = pWrapper->pMgmt;
+  taosRLockLatch(&pMgmt->latch);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    if (pPort != NULL) {
+      *pPort = pDnodeEp->ep.port;
+    }
+    if (pFqdn != NULL) {
+      tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
+    }
+    if (pEp != NULL) {
+      snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
+    }
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+}
+
+void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  SEpSet epSet = {0};
+  dmGetMnodeEpSet(pMgmt, &epSet);
+
+  dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse);
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+    if (strcmp(epSet.eps[i].fqdn, pDnode->localFqdn) == 0 && epSet.eps[i].port == pDnode->serverPort) {
+      epSet.inUse = (i + 1) % epSet.numOfEps;
+    }
+
+    epSet.eps[i].port = htons(epSet.eps[i].port);
+  }
+
+  rpcSendRedirectRsp(pReq->handle, &epSet);
+}
+
+static int32_t dmStart(SMgmtWrapper *pWrapper) {
+  dDebug("dnode mgmt start to run");
+  return dmStartThread(pWrapper->pMgmt);
+}
+
+int32_t dmInit(SMgmtWrapper *pWrapper) {
+  SDnode     *pDnode = pWrapper->pDnode;
+  SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt));
+  dInfo("dnode-mgmt is initialized");
+
+  pDnode->dnodeId = 0;
+  pDnode->dropped = 0;
+  pDnode->clusterId = 0;
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pDnode;
+  pMgmt->pWrapper = pWrapper;
+  taosInitRWLatch(&pMgmt->latch);
+
+  pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pMgmt->dnodeHash == NULL) {
+    dError("failed to init dnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dmReadFile(pMgmt) != 0) {
+    dError("failed to read file since %s", terrstr());
+    return -1;
+  }
+
+  if (pDnode->dropped) {
+    dError("dnode will not start since its already dropped");
+    return -1;
+  }
+
+  if (dmStartWorker(pMgmt) != 0) {
+    return -1;
+  }
+
+  pWrapper->pMgmt = pMgmt;
+  dInfo("dnode-mgmt is initialized");
+  return 0;
+}
+
+void dmCleanup(SMgmtWrapper *pWrapper) {
+  SDnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("dnode-mgmt start to clean up");
+  dmStopWorker(pMgmt);
+
+  taosWLockLatch(&pMgmt->latch);
+
+  if (pMgmt->dnodeEps != NULL) {
+    taosArrayDestroy(pMgmt->dnodeEps);
+    pMgmt->dnodeEps = NULL;
+  }
+
+  if (pMgmt->dnodeHash != NULL) {
+    taosHashCleanup(pMgmt->dnodeHash);
+    pMgmt->dnodeHash = NULL;
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+
+  free(pMgmt);
+  pWrapper->pMgmt = NULL;
+  dInfo("dnode-mgmt is cleaned up");
+}
+
+int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) {
+  *required = true;
+  return 0;
+}
+
+void dmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = dmInit;
+  mgmtFp.closeFp = dmCleanup;
+  mgmtFp.startFp = dmStart;
+  mgmtFp.requiredFp = dmRequire;
+
+  dmInitMsgHandles(pWrapper);
+  pWrapper->name = "dnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/dnode/src/dmMsg.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+#include "vm.h"
+
+void dmSendStatusReq(SDnodeMgmt *pMgmt) {
+  SDnode    *pDnode = pMgmt->pDnode;
+  SStatusReq req = {0};
+
+  taosRLockLatch(&pMgmt->latch);
+  req.sver = tsVersion;
+  req.dver = pMgmt->dver;
+  req.dnodeId = pDnode->dnodeId;
+  req.clusterId = pDnode->clusterId;
+  req.rebootTime = pDnode->rebootTime;
+  req.updateTime = pMgmt->updateTime;
+  req.numOfCores = tsNumOfCores;
+  req.numOfSupportVnodes = pDnode->numOfSupportVnodes;
+  tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN);
+
+  req.clusterCfg.statusInterval = tsStatusInterval;
+  req.clusterCfg.checkTime = 0;
+  char timestr[32] = "1970-01-01 00:00:00.00";
+  (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN);
+  memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN);
+  memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
+  taosRUnLockLatch(&pMgmt->latch);
+
+  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
+  vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads);
+
+  int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
+  void   *pHead = rpcMallocCont(contLen);
+  tSerializeSStatusReq(pHead, contLen, &req);
+  taosArrayDestroy(req.pVloads);
+
+  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
+  pMgmt->statusSent = 1;
+
+  dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);
+  dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg);
+}
+
+static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  if (pDnode->dnodeId == 0) {
+    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
+    taosWLockLatch(&pMgmt->latch);
+    pDnode->dnodeId = pCfg->dnodeId;
+    pDnode->clusterId = pCfg->clusterId;
+    dmWriteFile(pMgmt);
+    taosWUnLockLatch(&pMgmt->latch);
+  }
+}
+
+int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pMgmt->pDnode;
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+
+  if (pRsp->code != TSDB_CODE_SUCCESS) {
+    if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) {
+      dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId);
+      pDnode->dropped = 1;
+      dmWriteFile(pMgmt);
+    }
+  } else {
+    SStatusRsp statusRsp = {0};
+    if (pRsp->pCont != NULL && pRsp->contLen != 0 &&
+        tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
+      pMgmt->dver = statusRsp.dver;
+      dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
+      dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps);
+    }
+    tFreeSStatusRsp(&statusRsp);
+  }
+
+  pMgmt->statusSent = 0;
+}
+
+int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+  dError("auth rsp is received, but not supported yet");
+  return 0;
+}
+
+int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+  dError("grant rsp is received, but not supported yet");
+  return 0;
+}
+
+int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg       *pReq = &pMsg->rpcMsg;
+  SDCfgDnodeReq *pCfg = pReq->pCont;
+  dError("config req is received, but not supported yet");
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+}
+
+void dmInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by DNODE
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg);
+
+  // Requests handled by MNODE
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+}

source/dnode/mgmt/dnode/src/dmWorker.c

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bm.h"
+#include "dmInt.h"
+#include "mm.h"
+#include "qm.h"
+#include "sm.h"
+#include "vm.h"
+
+static void *dmThreadRoutine(void *param) {
+  SDnodeMgmt *pMgmt = param;
+  SDnode     *pDnode = pMgmt->pDnode;
+  int64_t     lastStatusTime = taosGetTimestampMs();
+  int64_t     lastMonitorTime = lastStatusTime;
+
+  setThreadName("dnode-hb");
+
+  while (true) {
+    taosThreadTestCancel();
+    taosMsleep(200);
+    if (dndGetStatus(pDnode) != DND_STAT_RUNNING || pDnode->dropped) {
+      continue;
+    }
+
+    int64_t curTime = taosGetTimestampMs();
+
+    float statusInterval = (curTime - lastStatusTime) / 1000.0f;
+    if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
+      dmSendStatusReq(pMgmt);
+      lastStatusTime = curTime;
+    }
+
+    float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
+    if (monitorInterval >= tsMonitorInterval) {
+      dndSendMonitorReport(pDnode);
+      lastMonitorTime = curTime;
+    }
+  }
+}
+
+static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode *pDnode = pMgmt->pDnode;
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t code = -1;
+  dTrace("msg:%p, will be processed in dnode queue", pMsg);
+
+  switch (pRpc->msgType) {
+    case TDMT_DND_CREATE_MNODE:
+    case TDMT_DND_CREATE_QNODE:
+    case TDMT_DND_CREATE_SNODE:
+    case TDMT_DND_CREATE_BNODE:
+    case TDMT_DND_DROP_MNODE:
+    case TDMT_DND_DROP_QNODE:
+    case TDMT_DND_DROP_SNODE:
+    case TDMT_DND_DROP_BNODE:
+      code = dndProcessNodeMsg(pMgmt->pDnode, pMsg);
+      break;
+    case TDMT_DND_CONFIG_DNODE:
+      code = dmProcessConfigReq(pMgmt, pMsg);
+      break;
+    case TDMT_MND_STATUS_RSP:
+      code = dmProcessStatusRsp(pMgmt, pMsg);
+      break;
+    case TDMT_MND_AUTH_RSP:
+      code = dmProcessAuthRsp(pMgmt, pMsg);
+      break;
+    case TDMT_MND_GRANT_RSP:
+      code = dmProcessGrantRsp(pMgmt, pMsg);
+      break;
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType));
+  }
+
+  if (pRpc->msgType & 1u) {
+    if (code != 0) code = terrno;
+    SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dmProcessQueue) != 0) {
+    dError("failed to start dnode mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dmProcessQueue) != 0) {
+    dError("failed to start dnode mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t dmStartThread(SDnodeMgmt *pMgmt) {
+  pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt);
+  if (pMgmt->threadId == NULL) {
+    dError("failed to init dnode thread");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+void dmStopWorker(SDnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->mgmtWorker);
+  dndCleanupWorker(&pMgmt->statusWorker);
+
+  if (pMgmt->threadId != NULL) {
+    taosDestoryThread(pMgmt->threadId);
+    pMgmt->threadId = NULL;
+  }
+}
+
+int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->mgmtWorker;
+  if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) {
+    pWorker = &pMgmt->statusWorker;
+  }
+
+  dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}

source/dnode/mgmt/impl/CMakeLists.txt

@@ -1,14 +0,0 @@
-aux_source_directory(src DNODE_SRC)
-add_library(dnode STATIC ${DNODE_SRC})
-target_link_libraries(
-    dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
-)
-target_include_directories(
-    dnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-if(${BUILD_TEST})
-    add_subdirectory(test)
-endif(${BUILD_TEST})

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_ENV_H_
-#define _TD_DND_ENV_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "dndInt.h"
-
-typedef struct {
-  EWorkerType type;
-  const char *name;
-  int32_t     minNum;
-  int32_t     maxNum;
-  void       *queueFp;
-  SDnode     *pDnode;
-  STaosQueue *queue;
-  union {
-    SQWorkerPool pool;
-    SWWorkerPool mpool;
-  };
-} SDnodeWorker;
-
-typedef struct {
-  char *dnode;
-  char *mnode;
-  char *snode;
-  char *bnode;
-  char *vnodes;
-} SDnodeDir;
-
-typedef struct {
-  int32_t      dnodeId;
-  int32_t      dropped;
-  int64_t      clusterId;
-  int64_t      dver;
-  int64_t      rebootTime;
-  int64_t      updateTime;
-  int8_t       statusSent;
-  SEpSet       mnodeEpSet;
-  char        *file;
-  SHashObj    *dnodeHash;
-  SArray      *pDnodeEps;
-  TdThread   *threadId;
-  SRWLatch     latch;
-  SDnodeWorker mgmtWorker;
-  SDnodeWorker statusWorker;
-} SDnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SMnode      *pMnode;
-  SRWLatch     latch;
-  SDnodeWorker readWorker;
-  SDnodeWorker writeWorker;
-  SDnodeWorker syncWorker;
-  int8_t       replica;
-  int8_t       selfIndex;
-  SReplica     replicas[TSDB_MAX_REPLICA];
-} SMnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SQnode      *pQnode;
-  SRWLatch     latch;
-  SDnodeWorker queryWorker;
-  SDnodeWorker fetchWorker;
-} SQnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  int8_t       uniqueWorkerInUse;
-  SSnode      *pSnode;
-  SRWLatch     latch;
-  SArray      *uniqueWorkers;  // SArray<SDnodeWorker*>
-  SDnodeWorker sharedWorker;
-} SSnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SBnode      *pBnode;
-  SRWLatch     latch;
-  SDnodeWorker writeWorker;
-} SBnodeMgmt;
-
-typedef struct {
-  int32_t openVnodes;
-  int32_t totalVnodes;
-  int32_t masterNum;
-  int64_t numOfSelectReqs;
-  int64_t numOfInsertReqs;
-  int64_t numOfInsertSuccessReqs;
-  int64_t numOfBatchInsertReqs;
-  int64_t numOfBatchInsertSuccessReqs;
-} SVnodesStat;
-
-typedef struct {
-  SVnodesStat  stat;
-  SHashObj    *hash;
-  SRWLatch     latch;
-  SQWorkerPool queryPool;
-  SFWorkerPool fetchPool;
-  SWWorkerPool syncPool;
-  SWWorkerPool writePool;
-} SVnodesMgmt;
-
-typedef struct {
-  void    *serverRpc;
-  void    *clientRpc;
-  DndMsgFp msgFp[TDMT_MAX];
-} STransMgmt;
-
-typedef struct SDnode {
-  EStat        stat;
-  SDnodeObjCfg cfg;
-  SDnodeDir    dir;
-  TdFilePtr    pLockFile;
-  SDnodeMgmt   dmgmt;
-  SMnodeMgmt   mmgmt;
-  SQnodeMgmt   qmgmt;
-  SSnodeMgmt   smgmt;
-  SBnodeMgmt   bmgmt;
-  SVnodesMgmt  vmgmt;
-  STransMgmt   tmgmt;
-  STfs        *pTfs;
-  SStartupReq  startup;
-} SDnode;
-
-int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_ENV_H_*/

source/dnode/mgmt/impl/inc/dndMgmt.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_DNODE_H_
-#define _TD_DND_DNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitMgmt(SDnode *pDnode);
-void    dndStopMgmt(SDnode *pDnode);
-void    dndCleanupMgmt(SDnode *pDnode);
-
-int32_t dndGetDnodeId(SDnode *pDnode);
-int64_t dndGetClusterId(SDnode *pDnode);
-void    dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
-void    dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet);
-
-void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg);
-void dndSendStatusReq(SDnode *pDnode);
-void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet);
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_DNODE_H_*/

source/dnode/mgmt/impl/inc/dndMnode.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_MNODE_H_
-#define _TD_DND_MNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitMnode(SDnode *pDnode);
-void    dndCleanupMnode(SDnode *pDnode);
-
-int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
-void    dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-
-int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                               SMonGrantInfo *pGrantInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_MNODE_H_*/

source/dnode/mgmt/impl/inc/dndSnode.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_SNODE_H_
-#define _TD_DND_SNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitSnode(SDnode *pDnode);
-void    dndCleanupSnode(SDnode *pDnode);
-
-// void    dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-
-void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_SNODE_H_*/

source/dnode/mgmt/impl/inc/dndVnodes.h

@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_VNODES_H_
-#define _TD_DND_VNODES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitVnodes(SDnode *pDnode);
-void    dndCleanupVnodes(SDnode *pDnode);
-void    dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads);
-void    dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-
-int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-
-int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_VNODES_H_*/

source/dnode/mgmt/impl/src/dndBnode.c

@@ -1,392 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndBnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs);
-
-static SBnode *dndAcquireBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  SBnode     *pBnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pBnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pBnode = pMgmt->pBnode;
-  } else {
-    terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pBnode != NULL) {
-    dTrace("acquire bnode, refCount:%d", refCount);
-  }
-  return pBnode;
-}
-
-static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) {
-  if (pBnode == NULL) return;
-
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release bnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadBnodeFile(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  int32_t     code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_BNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_BNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_BNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_BNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_BNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_BNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteBnodeFile(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartBnodeWorker(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) {
-    dError("failed to start bnode write worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopBnodeWorker(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->writeWorker);
-}
-
-static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  SBnode     *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode != NULL) {
-    dndReleaseBnode(pDnode, pBnode);
-    terrno = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED;
-    dError("failed to create bnode since %s", terrstr());
-    return -1;
-  }
-
-  SBnodeOpt option = {0};
-  dndBuildBnodeOption(pDnode, &option);
-
-  pBnode = bndOpen(pDnode->dir.bnode, &option);
-  if (pBnode == NULL) {
-    dError("failed to open bnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartBnodeWorker(pDnode) != 0) {
-    dError("failed to start bnode worker since %s", terrstr());
-    bndClose(pBnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteBnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write bnode file since %s", terrstr());
-    dndStopBnodeWorker(pDnode);
-    bndClose(pBnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pBnode = pBnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("bnode open successfully");
-  return 0;
-}
-
-static int32_t dndDropBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode == NULL) {
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteBnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseBnode(pDnode, pBnode);
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseBnode(pDnode, pBnode);
-  dndStopBnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteBnodeFile(pDnode);
-  bndClose(pBnode);
-  pMgmt->pBnode = NULL;
-  bndDestroy(pDnode->dir.bnode);
-
-  return 0;
-}
-
-int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateBnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION;
-    dError("failed to create bnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenBnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropBnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION;
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropBnode(pDnode);
-  }
-}
-
-static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) {
-  SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-  rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) {
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    SRpcMsg *pMsg = NULL;
-    taosGetQitem(qall, (void **)&pMsg);
-    dndSendBnodeErrorRsp(pMsg, code);
-  }
-}
-
-static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode == NULL) {
-    dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
-    return;
-  }
-
-  SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
-  if (pArray == NULL) {
-    dndReleaseBnode(pDnode, pBnode);
-    dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
-    return;
-  }
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    SRpcMsg *pMsg = NULL;
-    taosGetQitem(qall, (void **)&pMsg);
-    void *ptr = taosArrayPush(pArray, &pMsg);
-    if (ptr == NULL) {
-      dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
-    }
-  }
-
-  bndProcessWMsgs(pBnode, pArray);
-
-  for (size_t i = 0; i < numOfMsgs; i++) {
-    SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-  taosArrayDestroy(pArray);
-  dndReleaseBnode(pDnode, pBnode);
-}
-
-static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
-
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseBnode(pDnode, pBnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg);
-}
-
-int32_t dndInitBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadBnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("bnode has been deployed and needs to be deleted");
-    bndDestroy(pDnode->dir.bnode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenBnode(pDnode);
-}
-
-void dndCleanupBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  if (pMgmt->pBnode) {
-    dndStopBnodeWorker(pDnode);
-    bndClose(pMgmt->pBnode);
-    pMgmt->pBnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndEnv.c

@@ -1,334 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndBnode.h"
-#include "dndMgmt.h"
-#include "dndMnode.h"
-#include "dndQnode.h"
-#include "dndSnode.h"
-#include "dndTransport.h"
-#include "dndVnodes.h"
-#include "monitor.h"
-#include "sync.h"
-#include "tfs.h"
-#include "wal.h"
-
-static int8_t once = DND_ENV_INIT;
-
-EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
-
-void dndSetStat(SDnode *pDnode, EStat stat) {
-  dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat));
-  pDnode->stat = stat;
-}
-
-const char *dndStatStr(EStat stat) {
-  switch (stat) {
-    case DND_STAT_INIT:
-      return "init";
-    case DND_STAT_RUNNING:
-      return "running";
-    case DND_STAT_STOPPED:
-      return "stopped";
-    default:
-      return "unknown";
-  }
-}
-
-void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
-  SStartupReq *pStartup = &pDnode->startup;
-  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
-  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
-  pStartup->finished = 0;
-}
-
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
-  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
-  pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
-}
-
-static TdFilePtr dndCheckRunning(char *dataDir) {
-  char filepath[PATH_MAX] = {0};
-  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
-
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    return NULL;
-  }
-
-  int32_t ret = taosLockFile(pFile);
-  if (ret != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    taosCloseFile(&pFile);
-    return NULL;
-  }
-
-  return pFile;
-}
-
-static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) {
-  pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
-  if (pDnode->pLockFile == NULL) {
-    return -1;
-  }
-
-  char path[PATH_MAX + 100];
-  snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.mnode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.vnodes = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.dnode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.snode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.bnode = tstrdup(path);
-
-  if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL ||
-      pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) {
-    dError("failed to malloc dir object");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.dnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.mnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.vnodes) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.snode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.bnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg));
-  return 0;
-}
-
-static void dndCloseDir(SDnode *pDnode) {
-  tfree(pDnode->dir.mnode);
-  tfree(pDnode->dir.vnodes);
-  tfree(pDnode->dir.dnode);
-  tfree(pDnode->dir.snode);
-  tfree(pDnode->dir.bnode);
-
-  if (pDnode->pLockFile != NULL) {
-    taosUnLockFile(pDnode->pLockFile);
-    taosCloseFile(&pDnode->pLockFile);
-    pDnode->pLockFile = NULL;
-  }
-}
-
-SDnode *dndCreate(SDnodeObjCfg *pCfg) {
-  dInfo("start to create dnode object");
-
-  SDnode *pDnode = calloc(1, sizeof(SDnode));
-  if (pDnode == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    dError("failed to create dnode object since %s", terrstr());
-    return NULL;
-  }
-
-  dndSetStat(pDnode, DND_STAT_INIT);
-
-  if (dndInitDir(pDnode, pCfg) != 0) {
-    dError("failed to init dnode dir since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  SDiskCfg dCfg = {0};
-  tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN);
-  dCfg.level = 0;
-  dCfg.primary = 1;
-  SDiskCfg *pDisks = pDnode->cfg.pDisks;
-  int32_t   numOfDisks = pDnode->cfg.numOfDisks;
-  if (numOfDisks <= 0 || pDisks == NULL) {
-    pDisks = &dCfg;
-    numOfDisks = 1;
-  }
-
-  pDnode->pTfs = tfsOpen(pDisks, numOfDisks);
-  if (pDnode->pTfs == NULL) {
-    dError("failed to init tfs since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitMgmt(pDnode) != 0) {
-    dError("failed to init mgmt since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitVnodes(pDnode) != 0) {
-    dError("failed to init vnodes since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitQnode(pDnode) != 0) {
-    dError("failed to init qnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitSnode(pDnode) != 0) {
-    dError("failed to init snode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitBnode(pDnode) != 0) {
-    dError("failed to init bnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitMnode(pDnode) != 0) {
-    dError("failed to init mnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitTrans(pDnode) != 0) {
-    dError("failed to init transport since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  dndSetStat(pDnode, DND_STAT_RUNNING);
-  dndSendStatusReq(pDnode);
-  dndReportStartup(pDnode, "TDengine", "initialized successfully");
-  dInfo("dnode object is created, data:%p", pDnode);
-
-  return pDnode;
-}
-
-void dndClose(SDnode *pDnode) {
-  if (pDnode == NULL) return;
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    dError("dnode is shutting down, data:%p", pDnode);
-    return;
-  }
-
-  dInfo("start to close dnode, data:%p", pDnode);
-  dndSetStat(pDnode, DND_STAT_STOPPED);
-  dndCleanupTrans(pDnode);
-  dndStopMgmt(pDnode);
-  dndCleanupMnode(pDnode);
-  dndCleanupBnode(pDnode);
-  dndCleanupSnode(pDnode);
-  dndCleanupQnode(pDnode);
-  dndCleanupVnodes(pDnode);
-  dndCleanupMgmt(pDnode);
-  tfsClose(pDnode->pTfs);
-
-  dndCloseDir(pDnode);
-  free(pDnode);
-  dInfo("dnode object is closed, data:%p", pDnode);
-}
-
-int32_t dndInit() {
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
-    terrno = TSDB_CODE_REPEAT_INIT;
-    dError("failed to init dnode env since %s", terrstr());
-    return -1;
-  }
-
-  taosIgnSIGPIPE();
-  taosBlockSIGPIPE();
-  taosResolveCRC();
-
-  if (rpcInit() != 0) {
-    dError("failed to init rpc since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  if (walInit() != 0) {
-    dError("failed to init wal since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  SVnodeOpt vnodeOpt = {
-      .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode};
-
-  if (vnodeInit(&vnodeOpt) != 0) {
-    dError("failed to init vnode since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp};
-  if (monInit(&monCfg) != 0) {
-    dError("failed to init monitor since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  dInfo("dnode env is initialized");
-  return 0;
-}
-
-void dndCleanup() {
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
-    dError("dnode env is already cleaned up");
-    return;
-  }
-
-  walCleanUp();
-  vnodeCleanup();
-  rpcCleanup();
-  monCleanup();
-
-  taosStopCacheRefreshWorker();
-  dInfo("dnode env is cleaned up");
-}
-
-int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
-  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
-  pInfo->logdir.size = tsLogSpace.size;
-  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
-  pInfo->tempdir.size = tsTempSpace.size;
-
-  return tfsGetMonitorInfo(pDnode->pTfs, pInfo);
-}

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -1,764 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "dndMgmt.h"
-#include "dndBnode.h"
-#include "dndMnode.h"
-#include "dndQnode.h"
-#include "dndSnode.h"
-#include "dndTransport.h"
-#include "dndVnodes.h"
-#include "dndWorker.h"
-#include "monitor.h"
-
-static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static int32_t dndReadDnodes(SDnode *pDnode);
-static int32_t dndWriteDnodes(SDnode *pDnode);
-static void   *dnodeThreadRoutine(void *param);
-
-static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-static void    dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp);
-static void    dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp);
-static void    dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp);
-
-int32_t dndGetDnodeId(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t dnodeId = pMgmt->dnodeId;
-  taosRUnLockLatch(&pMgmt->latch);
-  return dnodeId;
-}
-
-int64_t dndGetClusterId(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int64_t clusterId = pMgmt->clusterId;
-  taosRUnLockLatch(&pMgmt->latch);
-  return clusterId;
-}
-
-void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-
-  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pDnodeEp != NULL) {
-    if (pPort != NULL) {
-      *pPort = pDnodeEp->ep.port;
-    }
-    if (pFqdn != NULL) {
-      tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
-    }
-    if (pEp != NULL) {
-      snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    }
-  }
-
-  taosRUnLockLatch(&pMgmt->latch);
-}
-
-void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  *pEpSet = pMgmt->mnodeEpSet;
-  taosRUnLockLatch(&pMgmt->latch);
-}
-
-void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) {
-  tmsg_t msgType = pReq->msgType;
-
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-
-  dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse);
-  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
-    if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) {
-      epSet.inUse = (i + 1) % epSet.numOfEps;
-    }
-
-    epSet.eps[i].port = htons(epSet.eps[i].port);
-  }
-
-  rpcSendRedirectRsp(pReq->handle, &epSet);
-}
-
-static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
-  dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  pMgmt->mnodeEpSet = *pEpSet;
-  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
-    dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-}
-
-static void dndPrintDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  dDebug("print dnode ep list, num:%d", numOfEps);
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i);
-    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
-  }
-}
-
-static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pMgmt->pDnodeEps != pDnodeEps) {
-    SArray *tmp = pMgmt->pDnodeEps;
-    pMgmt->pDnodeEps = taosArrayDup(pDnodeEps);
-    taosArrayDestroy(tmp);
-  }
-
-  pMgmt->mnodeEpSet.inUse = 0;
-  pMgmt->mnodeEpSet.numOfEps = 0;
-
-  int32_t mIndex = 0;
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps);
-
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
-    if (!pDnodeEp->isMnode) continue;
-    if (mIndex >= TSDB_MAX_REPLICA) continue;
-    pMgmt->mnodeEpSet.numOfEps++;
-
-    pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep;
-    mIndex++;
-  }
-
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
-    taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
-  }
-
-  dndPrintDnodes(pDnode);
-}
-
-static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) {
-  bool changed = false;
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-
-  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pDnodeEp != NULL) {
-    char epstr[TSDB_EP_LEN + 1];
-    snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    changed = strcmp(pEp, epstr) != 0;
-  }
-
-  taosRUnLockLatch(&pMgmt->latch);
-  return changed;
-}
-
-static int32_t dndReadDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp));
-  if (pMgmt->pDnodeEps == NULL) {
-    dError("failed to calloc dnodeEp array since %s", strerror(errno));
-    goto PRASE_DNODE_OVER;
-  }
-
-  int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR;
-  int32_t len = 0;
-  int32_t maxLen = 256 * 1024;
-  char   *content = calloc(1, maxLen + 1);
-  cJSON  *root = NULL;
-
-  // fp = fopen(pMgmt->file, "r");
-  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", pMgmt->file);
-    code = 0;
-    goto PRASE_DNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read %s since dnodeId not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->dnodeId = dnodeId->valueint;
-
-  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read %s since clusterId not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->clusterId = atoll(clusterId->valuestring);
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes");
-  if (!dnodes || dnodes->type != cJSON_Array) {
-    dError("failed to read %s since dnodes not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  int32_t numOfDnodes = cJSON_GetArraySize(dnodes);
-  if (numOfDnodes <= 0) {
-    dError("failed to read %s since numOfDnodes:%d invalid", pMgmt->file, numOfDnodes);
-    goto PRASE_DNODE_OVER;
-  }
-
-  for (int32_t i = 0; i < numOfDnodes; ++i) {
-    cJSON *node = cJSON_GetArrayItem(dnodes, i);
-    if (node == NULL) break;
-
-    SDnodeEp dnodeEp = {0};
-
-    cJSON *did = cJSON_GetObjectItem(node, "id");
-    if (!did || did->type != cJSON_Number) {
-      dError("failed to read %s since dnodeId not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-
-    dnodeEp.id = dnodeId->valueint;
-
-    cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
-    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
-      dError("failed to read %s since dnodeFqdn not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-    tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
-
-    cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
-    if (!dnodePort || dnodePort->type != cJSON_Number) {
-      dError("failed to read %s since dnodePort not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-
-    dnodeEp.ep.port = dnodePort->valueint;
-
-    cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
-    if (!isMnode || isMnode->type != cJSON_Number) {
-      dError("failed to read %s since isMnode not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-    dnodeEp.isMnode = isMnode->valueint;
-
-    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
-  }
-
-  code = 0;
-  dInfo("succcessed to read file %s", pMgmt->file);
-  dndPrintDnodes(pDnode);
-
-PRASE_DNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
-    dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
-    return -1;
-  }
-
-  if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
-    SDnodeEp dnodeEp = {0};
-    dnodeEp.isMnode = 1;
-    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
-    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
-  }
-
-  dndResetDnodes(pDnode, pMgmt->pDnodeEps);
-
-  terrno = 0;
-  return 0;
-}
-
-static int32_t dndWriteDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  // FILE *fp = fopen(pMgmt->file, "w");
-  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    dError("failed to write %s since %s", pMgmt->file, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 256 * 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", pMgmt->dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
-
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  for (int32_t i = 0; i < numOfEps; ++i) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i);
-    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pDnodeEp->id);
-    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
-    len += snprintf(content + len, maxLen - len, "    \"port\": %u,\n", pDnodeEp->ep.port);
-    len += snprintf(content + len, maxLen - len, "    \"isMnode\": %d\n", pDnodeEp->isMnode);
-    if (i < numOfEps - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-  terrno = 0;
-
-  pMgmt->updateTime = taosGetTimestampMs();
-  dDebug("successed to write %s", pMgmt->file);
-  return 0;
-}
-
-void dndSendStatusReq(SDnode *pDnode) {
-  SStatusReq req = {0};
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  req.sver = tsVersion;
-  req.dver = pMgmt->dver;
-  req.dnodeId = pMgmt->dnodeId;
-  req.clusterId = pMgmt->clusterId;
-  req.rebootTime = pMgmt->rebootTime;
-  req.updateTime = pMgmt->updateTime;
-  req.numOfCores = tsNumOfCores;
-  req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes;
-  memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN);
-
-  req.clusterCfg.statusInterval = tsStatusInterval;
-  req.clusterCfg.checkTime = 0;
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN);
-  memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN);
-  memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
-  taosRUnLockLatch(&pMgmt->latch);
-
-  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
-  dndGetVnodeLoads(pDnode, req.pVloads);
-
-  int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
-  void   *pHead = rpcMallocCont(contLen);
-  tSerializeSStatusReq(pHead, contLen, &req);
-  taosArrayDestroy(req.pVloads);
-
-  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
-  pMgmt->statusSent = 1;
-
-  dTrace("pDnode:%p, send status req to mnode", pDnode);
-  dndSendReqToMnode(pDnode, &rpcMsg);
-}
-
-static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  if (pMgmt->dnodeId == 0) {
-    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
-    taosWLockLatch(&pMgmt->latch);
-    pMgmt->dnodeId = pCfg->dnodeId;
-    pMgmt->clusterId = pCfg->clusterId;
-    dndWriteDnodes(pDnode);
-    taosWUnLockLatch(&pMgmt->latch);
-  }
-}
-
-static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) {
-  int32_t numOfEps = taosArrayGetSize(pDnodeEps);
-  if (numOfEps <= 0) return;
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  if (numOfEps != numOfEpsOld) {
-    dndResetDnodes(pDnode, pDnodeEps);
-    dndWriteDnodes(pDnode);
-  } else {
-    int32_t size = numOfEps * sizeof(SDnodeEp);
-    if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) {
-      dndResetDnodes(pDnode, pDnodeEps);
-      dndWriteDnodes(pDnode);
-    }
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-}
-
-static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pRsp->code != TSDB_CODE_SUCCESS) {
-    if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) {
-      dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId);
-      pMgmt->dropped = 1;
-      dndWriteDnodes(pDnode);
-    }
-  } else {
-    SStatusRsp statusRsp = {0};
-    if (pRsp->pCont != NULL && pRsp->contLen != 0 &&
-        tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
-      pMgmt->dver = statusRsp.dver;
-      dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg);
-      dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps);
-    }
-    taosArrayDestroy(statusRsp.pDnodeEps);
-  }
-
-  pMgmt->statusSent = 0;
-}
-
-static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); }
-
-static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) {
-  dError("grant rsp is received, but not supported yet");
-}
-
-static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dError("config req is received, but not supported yet");
-  SDCfgDnodeReq *pCfg = pReq->pCont;
-  return TSDB_CODE_OPS_NOT_SUPPORT;
-}
-
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dDebug("startup req is received");
-
-  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
-  dndGetStartup(pDnode, pStartup);
-
-  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-
-  SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)};
-  rpcSendResponse(&rpcRsp);
-}
-
-static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
-  pInfo->dnode_id = dndGetDnodeId(pDnode);
-  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
-  pInfo->cluster_id = dndGetClusterId(pDnode);
-  pInfo->protocol = 1;
-}
-
-static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
-  pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f);
-  taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
-  pInfo->cpu_cores = tsNumOfCores;
-  taosGetProcMemory(&pInfo->mem_engine);
-  taosGetSysMemory(&pInfo->mem_system);
-  pInfo->mem_total = tsTotalMemoryKB;
-  pInfo->disk_engine = 0;
-  pInfo->disk_used = tsDataSpace.size.used;
-  pInfo->disk_total = tsDataSpace.size.total;
-  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
-  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
-
-  SVnodesStat *pStat = &pDnode->vmgmt.stat;
-  pInfo->req_select = pStat->numOfSelectReqs;
-  pInfo->req_insert = pStat->numOfInsertReqs;
-  pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
-  pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
-  pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
-  pInfo->errors = tsNumOfErrorLogs;
-  pInfo->vnodes_num = pStat->totalVnodes;
-  pInfo->masters = pStat->masterNum;
-  pInfo->has_mnode = pDnode->mmgmt.deployed;
-}
-
-static void dndSendMonitorReport(SDnode *pDnode) {
-  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
-  dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort);
-
-  SMonInfo *pMonitor = monCreateMonitorInfo();
-  if (pMonitor == NULL) return;
-
-  SMonBasicInfo basicInfo = {0};
-  dndGetMonitorBasicInfo(pDnode, &basicInfo);
-  monSetBasicInfo(pMonitor, &basicInfo);
-
-  SMonClusterInfo clusterInfo = {0};
-  SMonVgroupInfo  vgroupInfo = {0};
-  SMonGrantInfo   grantInfo = {0};
-  if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
-    monSetClusterInfo(pMonitor, &clusterInfo);
-    monSetVgroupInfo(pMonitor, &vgroupInfo);
-    monSetGrantInfo(pMonitor, &grantInfo);
-  }
-
-  SMonDnodeInfo dnodeInfo = {0};
-  dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
-  monSetDnodeInfo(pMonitor, &dnodeInfo);
-
-  SMonDiskInfo diskInfo = {0};
-  if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
-    monSetDiskInfo(pMonitor, &diskInfo);
-  }
-
-  taosArrayDestroy(clusterInfo.dnodes);
-  taosArrayDestroy(clusterInfo.mnodes);
-  taosArrayDestroy(vgroupInfo.vgroups);
-  taosArrayDestroy(diskInfo.datadirs);
-
-  monSendReport(pMonitor);
-  monCleanupMonitorInfo(pMonitor);
-}
-
-static void *dnodeThreadRoutine(void *param) {
-  SDnode     *pDnode = param;
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  int64_t     lastStatusTime = taosGetTimestampMs();
-  int64_t     lastMonitorTime = lastStatusTime;
-
-  setThreadName("dnode-hb");
-
-  while (true) {
-    taosThreadTestCancel();
-    taosMsleep(200);
-    if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) {
-      continue;
-    }
-
-    int64_t curTime = taosGetTimestampMs();
-
-    float statusInterval = (curTime - lastStatusTime) / 1000.0f;
-    if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
-      dndSendStatusReq(pDnode);
-      lastStatusTime = curTime;
-    }
-
-    float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
-    if (monitorInterval >= tsMonitorInterval) {
-      dndSendMonitorReport(pDnode);
-      lastMonitorTime = curTime;
-    }
-  }
-}
-
-int32_t dndInitMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  pMgmt->dnodeId = 0;
-  pMgmt->rebootTime = taosGetTimestampMs();
-  pMgmt->dropped = 0;
-  pMgmt->clusterId = 0;
-  taosInitRWLatch(&pMgmt->latch);
-
-  char path[PATH_MAX];
-  snprintf(path, PATH_MAX, "%s/dnode.json", pDnode->dir.dnode);
-  pMgmt->file = strdup(path);
-  if (pMgmt->file == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  if (pMgmt->dnodeHash == NULL) {
-    dError("failed to init dnode hash");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (dndReadDnodes(pDnode) != 0) {
-    dError("failed to read file:%s since %s", pMgmt->file, terrstr());
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dError("dnode not start since its already dropped");
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) {
-    dError("failed to start dnode mgmt worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) {
-    dError("failed to start dnode mgmt worker since %s", terrstr());
-    return -1;
-  }
-
-  pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode);
-  if (pMgmt->threadId == NULL) {
-    dError("failed to init dnode thread");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  dInfo("dnode-mgmt is initialized");
-  return 0;
-}
-
-void dndStopMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  dndCleanupWorker(&pMgmt->mgmtWorker);
-  dndCleanupWorker(&pMgmt->statusWorker);
-
-  if (pMgmt->threadId != NULL) {
-    taosDestoryThread(pMgmt->threadId);
-    pMgmt->threadId = NULL;
-  }
-}
-
-void dndCleanupMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  if (pMgmt->pDnodeEps != NULL) {
-    taosArrayDestroy(pMgmt->pDnodeEps);
-    pMgmt->pDnodeEps = NULL;
-  }
-
-  if (pMgmt->dnodeHash != NULL) {
-    taosHashCleanup(pMgmt->dnodeHash);
-    pMgmt->dnodeHash = NULL;
-  }
-
-  if (pMgmt->file != NULL) {
-    free(pMgmt->file);
-    pMgmt->file = NULL;
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-  dInfo("dnode-mgmt is cleaned up");
-}
-
-void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) {
-    dndUpdateMnodeEpSet(pDnode, pEpSet);
-  }
-
-  SDnodeWorker *pWorker = &pMgmt->mgmtWorker;
-  if (pMsg->msgType == TDMT_MND_STATUS_RSP) {
-    pWorker = &pMgmt->statusWorker;
-  }
-
-  if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-}
-
-static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = 0;
-
-  switch (pMsg->msgType) {
-    case TDMT_DND_CREATE_MNODE:
-      code = dndProcessCreateMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_ALTER_MNODE:
-      code = dndProcessAlterMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_MNODE:
-      code = dndProcessDropMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_QNODE:
-      code = dndProcessCreateQnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_QNODE:
-      code = dndProcessDropQnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_SNODE:
-      code = dndProcessCreateSnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_SNODE:
-      code = dndProcessDropSnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_BNODE:
-      code = dndProcessCreateBnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_BNODE:
-      code = dndProcessDropBnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CONFIG_DNODE:
-      code = dndProcessConfigDnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_MND_STATUS_RSP:
-      dndProcessStatusRsp(pDnode, pMsg);
-      break;
-    case TDMT_MND_AUTH_RSP:
-      dndProcessAuthRsp(pDnode, pMsg);
-      break;
-    case TDMT_MND_GRANT_RSP:
-      dndProcessGrantRsp(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_VNODE:
-      code = dndProcessCreateVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_ALTER_VNODE:
-      code = dndProcessAlterVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_VNODE:
-      code = dndProcessDropVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_SYNC_VNODE:
-      code = dndProcessSyncVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_COMPACT_VNODE:
-      code = dndProcessCompactVnodeReq(pDnode, pMsg);
-      break;
-    default:
-      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-      code = -1;
-      dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType));
-      break;
-  }
-
-  if (pMsg->msgType & 1u) {
-    if (code != 0) code = terrno;
-    SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle};
-    rpcSendResponse(&rsp);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-  pMsg->pCont = NULL;
-  taosFreeQitem(pMsg);
-}

source/dnode/mgmt/impl/src/dndMnode.c

@@ -1,642 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndMnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg);
-static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg);
-
-static SMnode *dndAcquireMnode(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  SMnode     *pMnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pMnode = pMgmt->pMnode;
-  } else {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pMnode != NULL) {
-    dTrace("acquire mnode, refCount:%d", refCount);
-  }
-  return pMnode;
-}
-
-static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) {
-  if (pMnode == NULL) return;
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release mnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadMnodeFile(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  int32_t     code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 4096;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_MNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
-  if (!mnodes || mnodes->type != cJSON_Array) {
-    dError("failed to read %s since nodes not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  pMgmt->replica = cJSON_GetArraySize(mnodes);
-  if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
-    dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
-    goto PRASE_MNODE_OVER;
-  }
-
-  for (int32_t i = 0; i < pMgmt->replica; ++i) {
-    cJSON *node = cJSON_GetArrayItem(mnodes, i);
-    if (node == NULL) break;
-
-    SReplica *pReplica = &pMgmt->replicas[i];
-
-    cJSON *id = cJSON_GetObjectItem(node, "id");
-    if (!id || id->type != cJSON_Number) {
-      dError("failed to read %s since id not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    pReplica->id = id->valueint;
-
-    cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
-    if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
-      dError("failed to read %s since fqdn not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
-
-    cJSON *port = cJSON_GetObjectItem(node, "port");
-    if (!port || port->type != cJSON_Number) {
-      dError("failed to read %s since port not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    pReplica->port = port->valueint;
-  }
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_MNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteMnodeFile(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 4096;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"mnodes\": [{\n");
-  for (int32_t i = 0; i < pMgmt->replica; ++i) {
-    SReplica *pReplica = &pMgmt->replicas[i];
-    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pReplica->id);
-    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pReplica->fqdn);
-    len += snprintf(content + len, maxLen - len, "    \"port\": %u\n", pReplica->port);
-    if (i < pMgmt->replica - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartMnodeWorker(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode read worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode write worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode sync worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopMnodeWorker(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 1) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->readWorker);
-  dndCleanupWorker(&pMgmt->writeWorker);
-  dndCleanupWorker(&pMgmt->syncWorker);
-}
-
-static bool dndNeedDeployMnode(SDnode *pDnode) {
-  if (dndGetDnodeId(pDnode) > 0) {
-    return false;
-  }
-
-  if (dndGetClusterId(pDnode) > 0) {
-    return false;
-  }
-
-  if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) {
-    return false;
-  }
-
-  return true;
-}
-
-static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg);
-  return 0;
-}
-
-static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg);
-  return 0;
-}
-
-static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ;
-  pOption->putReqToMReadQFp = dndPutMsgToMReadQ;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-}
-
-static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  dndInitMnodeOption(pDnode, pOption);
-  pOption->replica = 1;
-  pOption->selfIndex = 0;
-  SReplica *pReplica = &pOption->replicas[0];
-  pReplica->id = 1;
-  pReplica->port = pDnode->cfg.serverPort;
-  memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN);
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pMgmt->selfIndex = pOption->selfIndex;
-  pMgmt->replica = pOption->replica;
-  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-}
-
-static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  dndInitMnodeOption(pDnode, pOption);
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pOption->selfIndex = pMgmt->selfIndex;
-  pOption->replica = pMgmt->replica;
-  memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-}
-
-static int32_t dndBuildMnodeOptionFromReq(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
-  dndInitMnodeOption(pDnode, pOption);
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-
-  pOption->replica = pCreate->replica;
-  pOption->selfIndex = -1;
-  for (int32_t i = 0; i < pCreate->replica; ++i) {
-    SReplica *pReplica = &pOption->replicas[i];
-    pReplica->id = pCreate->replicas[i].id;
-    pReplica->port = pCreate->replicas[i].port;
-    memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-    if (pReplica->id == pOption->dnodeId) {
-      pOption->selfIndex = i;
-    }
-  }
-
-  if (pOption->selfIndex == -1) {
-    dError("failed to build mnode options since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pMgmt->selfIndex = pOption->selfIndex;
-  pMgmt->replica = pOption->replica;
-  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-  return 0;
-}
-
-static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = mndOpen(pDnode->dir.mnode, pOption);
-  if (pMnode == NULL) {
-    dError("failed to open mnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartMnodeWorker(pDnode) != 0) {
-    dError("failed to start mnode worker since %s", terrstr());
-    mndClose(pMnode);
-    mndDestroy(pDnode->dir.mnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteMnodeFile(pDnode) != 0) {
-    dError("failed to write mnode file since %s", terrstr());
-    pMgmt->deployed = 0;
-    dndStopMnodeWorker(pDnode);
-    mndClose(pMnode);
-    mndDestroy(pDnode->dir.mnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pMnode = pMnode;
-  pMgmt->deployed = 1;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("mnode open successfully");
-  return 0;
-}
-
-static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  if (mndAlter(pMnode, pOption) != 0) {
-    dError("failed to alter mnode since %s", terrstr());
-    dndReleaseMnode(pDnode, pMnode);
-    return -1;
-  }
-
-  dndReleaseMnode(pDnode, pMnode);
-  return 0;
-}
-
-static int32_t dndDropMnode(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteMnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseMnode(pDnode, pMnode);
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseMnode(pDnode, pMnode);
-  dndStopMnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteMnodeFile(pDnode);
-  mndClose(pMnode);
-  pMgmt->pMnode = NULL;
-  mndDestroy(pDnode->dir.mnode);
-
-  return 0;
-}
-
-
-int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateMnodeReq createReq = {0};
-  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeOpt option = {0};
-  if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    dndReleaseMnode(pDnode, pMnode);
-    terrno = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to create mnode");
-  return dndOpenMnode(pDnode, &option);
-}
-
-int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDAlterMnodeReq alterReq = {0};
-  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (alterReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeOpt option = {0};
-  if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to alter mnode");
-  int32_t code = dndAlterMnode(pDnode, &option);
-  dndReleaseMnode(pDnode, pMnode);
-
-  return code;
-}
-
-int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropMnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to drop mnode");
-  int32_t code = dndDropMnode(pDnode);
-  dndReleaseMnode(pDnode, pMnode);
-
-  return code;
-}
-
-static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    mndProcessMsg(pMsg);
-    dndReleaseMnode(pDnode, pMnode);
-  } else {
-    mndSendRsp(pMsg, terrno);
-  }
-
-  mndCleanupMsg(pMsg);
-}
-
-static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) {
-  int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
-    if (pMsg == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-    } else {
-      code = dndWriteMsgToWorker(pWorker, pMsg, 0);
-      if (code != 0) code = terrno;
-    }
-
-    if (code != 0) {
-      mndCleanupMsg(pMsg);
-    }
-  }
-  dndReleaseMnode(pDnode, pMnode);
-
-  if (code != 0) {
-    if (pRpcMsg->msgType & 1u) {
-      if (code == TSDB_CODE_DND_MNODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) {
-        dndSendRedirectRsp(pDnode, pRpcMsg);
-      } else {
-        SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code};
-        rpcSendResponse(&rsp);
-      }
-    }
-    rpcFreeCont(pRpcMsg->pCont);
-  }
-}
-
-void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg);
-}
-
-void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg);
-}
-
-void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg);
-}
-
-int32_t dndInitMnode(SDnode *pDnode) {
-  dInfo("dnode-mnode start to init");
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadMnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("mnode has been deployed and needs to be deleted");
-    mndDestroy(pDnode->dir.mnode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) {
-    bool needDeploy = dndNeedDeployMnode(pDnode);
-    if (!needDeploy) {
-      dDebug("mnode does not need to be deployed");
-      return 0;
-    }
-
-    dInfo("start to deploy mnode");
-    SMnodeOpt option = {0};
-    dndBuildMnodeDeployOption(pDnode, &option);
-    return dndOpenMnode(pDnode, &option);
-  } else {
-    dInfo("start to open mnode");
-    SMnodeOpt option = {0};
-    dndBuildMnodeOpenOption(pDnode, &option);
-    return dndOpenMnode(pDnode, &option);
-  }
-}
-
-void dndCleanupMnode(SDnode *pDnode) {
-  dInfo("dnode-mnode start to clean up");
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  if (pMgmt->pMnode) {
-    dndStopMnodeWorker(pDnode);
-    mndClose(pMgmt->pMnode);
-    pMgmt->pMnode = NULL;
-  }
-  dInfo("dnode-mnode is cleaned up");
-}
-
-int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_APP_NOT_READY;
-    dTrace("failed to get user auth since %s", terrstr());
-    return -1;
-  }
-
-  int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
-  dndReleaseMnode(pDnode, pMnode);
-
-  dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
-  return code;
-}
-
-int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                               SMonGrantInfo *pGrantInfo) {
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) return -1;
-
-  int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
-  dndReleaseMnode(pDnode, pMnode);
-  return code;
-}

source/dnode/mgmt/impl/src/dndQnode.c

@@ -1,375 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndQnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static SQnode *dndAcquireQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  SQnode     *pQnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pQnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pQnode = pMgmt->pQnode;
-  } else {
-    terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pQnode != NULL) {
-    dTrace("acquire qnode, refCount:%d", refCount);
-  }
-  return pQnode;
-}
-
-static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) {
-  if (pQnode == NULL) return;
-
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release qnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadQnodeFile(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  int32_t     code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_QNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_QNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_QNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_QNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_QNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_QNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteQnodeFile(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartQnodeWorker(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) {
-    dError("failed to start qnode query worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) {
-    dError("failed to start qnode fetch worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopQnodeWorker(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->queryWorker);
-  dndCleanupWorker(&pMgmt->fetchWorker);
-}
-
-static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    dndReleaseQnode(pDnode, pQnode);
-    terrno = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED;
-    dError("failed to create qnode since %s", terrstr());
-    return -1;
-  }
-
-  SQnodeOpt option = {0};
-  dndBuildQnodeOption(pDnode, &option);
-
-  pQnode = qndOpen(&option);
-  if (pQnode == NULL) {
-    dError("failed to open qnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartQnodeWorker(pDnode) != 0) {
-    dError("failed to start qnode worker since %s", terrstr());
-    qndClose(pQnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteQnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write qnode file since %s", terrstr());
-    dndStopQnodeWorker(pDnode);
-    qndClose(pQnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pQnode = pQnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("qnode open successfully");
-  return 0;
-}
-
-static int32_t dndDropQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode == NULL) {
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteQnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseQnode(pDnode, pQnode);
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseQnode(pDnode, pQnode);
-  dndStopQnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteQnodeFile(pDnode);
-  qndClose(pQnode);
-  pMgmt->pQnode = NULL;
-
-  return 0;
-}
-
-int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateQnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION;
-    dError("failed to create qnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenQnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropQnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION;
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropQnode(pDnode);
-  }
-}
-
-static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  SRpcMsg    *pRsp = NULL;
-  int32_t     code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    code = qndProcessMsg(pQnode, pMsg, &pRsp);
-  }
-  dndReleaseQnode(pDnode, pQnode);
-
-  if (pMsg->msgType & 1u) {
-    if (pRsp != NULL) {
-      pRsp->ahandle = pMsg->ahandle;
-      rpcSendResponse(pRsp);
-      free(pRsp);
-    } else {
-      if (code != 0) code = terrno;
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-    }
-  }
-
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseQnode(pDnode, pQnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
-}
-
-void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
-}
-
-int32_t dndInitQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadQnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) return 0;
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenQnode(pDnode);
-}
-
-void dndCleanupQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  if (pMgmt->pQnode) {
-    dndStopQnodeWorker(pDnode);
-    qndClose(pMgmt->pQnode);
-    pMgmt->pQnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndSnode.c

@@ -1,505 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndSnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-typedef struct {
-  int32_t     vgId;
-  int32_t     refCount;
-  int32_t     snVersion;
-  int8_t      dropped;
-  char       *path;
-  SSnode     *pImpl;
-  STaosQueue *pSharedQ;
-  STaosQueue *pUniqueQ;
-} SSnodeObj;
-
-static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs);
-
-static SSnode *dndAcquireSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  SSnode     *pSnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pSnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pSnode = pMgmt->pSnode;
-  } else {
-    terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pSnode != NULL) {
-    dTrace("acquire snode, refCount:%d", refCount);
-  }
-  return pSnode;
-}
-
-static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) {
-  if (pSnode == NULL) return;
-
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release snode, refCount:%d", refCount);
-}
-
-static int32_t dndReadSnodeFile(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  int32_t     code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_SNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_SNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_SNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_SNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_SNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_SNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteSnodeFile(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartSnodeWorker(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *));
-  for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
-    SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker));
-    if (pUniqueWorker == NULL) {
-      return -1;
-    }
-    if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) {
-      dError("failed to start snode unique worker since %s", terrstr());
-      return -1;
-    }
-    taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker);
-  }
-  if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM,
-                    SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) {
-    dError("failed to start snode shared worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopSnodeWorker(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
-    SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i);
-    dndCleanupWorker(worker);
-  }
-  taosArrayDestroy(pMgmt->uniqueWorkers);
-}
-
-static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  SSnode     *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    dndReleaseSnode(pDnode, pSnode);
-    terrno = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED;
-    dError("failed to create snode since %s", terrstr());
-    return -1;
-  }
-
-  SSnodeOpt option = {0};
-  dndBuildSnodeOption(pDnode, &option);
-
-  pSnode = sndOpen(pDnode->dir.snode, &option);
-  if (pSnode == NULL) {
-    dError("failed to open snode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartSnodeWorker(pDnode) != 0) {
-    dError("failed to start snode worker since %s", terrstr());
-    sndClose(pSnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteSnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write snode file since %s", terrstr());
-    dndStopSnodeWorker(pDnode);
-    sndClose(pSnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pSnode = pSnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("snode open successfully");
-  return 0;
-}
-
-static int32_t dndDropSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode == NULL) {
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteSnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseSnode(pDnode, pSnode);
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseSnode(pDnode, pSnode);
-  dndStopSnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteSnodeFile(pDnode);
-  sndClose(pSnode);
-  pMgmt->pSnode = NULL;
-  sndDestroy(pDnode->dir.snode);
-
-  return 0;
-}
-
-int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateSnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION;
-    dError("failed to create snode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenSnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropSnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION;
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropSnode(pDnode);
-  }
-}
-
-static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
-  /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    for (int32_t i = 0; i < numOfMsgs; i++) {
-      SRpcMsg *pMsg = NULL;
-      taosGetQitem(qall, (void **)&pMsg);
-
-      sndProcessUMsg(pSnode, pMsg);
-
-      rpcFreeCont(pMsg->pCont);
-      taosFreeQitem(pMsg);
-    }
-    dndReleaseSnode(pDnode, pSnode);
-  } else {
-    for (int32_t i = 0; i < numOfMsgs; i++) {
-      SRpcMsg *pMsg = NULL;
-      taosGetQitem(qall, (void **)&pMsg);
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-
-      rpcFreeCont(pMsg->pCont);
-      taosFreeQitem(pMsg);
-    }
-  }
-}
-
-static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    sndProcessSMsg(pSnode, pMsg);
-    dndReleaseSnode(pDnode, pSnode);
-  } else {
-    SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-    rpcSendResponse(&rpcRsp);
-  }
-
-#if 0
-  if (pMsg->msgType & 1u) {
-    if (pRsp != NULL) {
-      pRsp->ahandle = pMsg->ahandle;
-      rpcSendResponse(pRsp);
-      free(pRsp);
-    } else {
-      if (code != 0) code = terrno;
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-    }
-  }
-#endif
-
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static FORCE_INLINE int32_t dndGetSWTypeFromMsg(SRpcMsg *pMsg) {
-  SStreamExecMsgHead *pHead = pMsg->pCont;
-  pHead->workerType = htonl(pHead->workerType);
-  return pHead->workerType;
-}
-
-static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) {
-  SMsgHead *pHead = pMsg->pCont;
-  pHead->streamTaskId = htonl(pHead->streamTaskId);
-  return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM;
-}
-
-static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    int32_t       index = dndGetSWIdFromMsg(pMsg);
-    SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index);
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0);
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg);
-}
-
-void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    int32_t workerType = dndGetSWTypeFromMsg(pMsg);
-    if (workerType == SND_WORKER_TYPE__SHARED) {
-      dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
-    } else {
-      dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
-    }
-  }
-}
-
-void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
-}
-
-void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
-}
-
-int32_t dndInitSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadSnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("snode has been deployed and needs to be deleted");
-    sndDestroy(pDnode->dir.snode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenSnode(pDnode);
-}
-
-void dndCleanupSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  if (pMgmt->pSnode) {
-    dndStopSnodeWorker(pDnode);
-    sndClose(pMgmt->pSnode);
-    pMgmt->pSnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndTransport.c

@@ -1,422 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-/* this file is mainly responsible for the communication between DNODEs. Each
- * dnode works as both server and client. Dnode may send status, grant, config
- * messages to mnode, mnode may send create/alter/drop table/vnode messages
- * to dnode. All theses messages are handled from here
- */
-
-#define _DEFAULT_SOURCE
-#include "dndTransport.h"
-#include "dndMgmt.h"
-#include "dndMnode.h"
-#include "dndSnode.h"
-#include "dndVnodes.h"
-
-#define INTERNAL_USER   "_dnd"
-#define INTERNAL_CKEY   "_key"
-#define INTERNAL_SECRET "_pwd"
-
-static void dndInitMsgFp(STransMgmt *pMgmt) {
-  // Requests handled by DNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessMgmtMsg;
-
-  // Requests handled by MNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg;
-
-  // Requests handled by VNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg;
-
-  // Requests handled by SNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_DEPLOY)] = dndProcessSnodeMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_EXEC)] = dndProcessSnodeExecMsg;
-}
-
-static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
-  SDnode     *pDnode = parent;
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  tmsg_t msgType = pRsp->msgType;
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    if (pRsp == NULL || pRsp->pCont == NULL) return;
-    dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
-    rpcFreeCont(pRsp->pCont);
-    return;
-  }
-
-  DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
-  if (fp != NULL) {
-    dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF,
-           pRsp->ahandle);
-    (*fp)(pDnode, pRsp, pEpSet);
-  } else {
-    dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
-    rpcFreeCont(pRsp->pCont);
-  }
-}
-
-static int32_t dndInitClient(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.label = "D-C";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp = dndProcessResponse;
-  rpcInit.sessions = 1024;
-  rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.user = INTERNAL_USER;
-  rpcInit.ckey = INTERNAL_CKEY;
-  rpcInit.spi = 1;
-  rpcInit.parent = pDnode;
-
-  char pass[TSDB_PASSWORD_LEN + 1] = {0};
-  taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
-  rpcInit.secret = pass;
-
-  pMgmt->clientRpc = rpcOpen(&rpcInit);
-  if (pMgmt->clientRpc == NULL) {
-    dError("failed to init dnode rpc client");
-    return -1;
-  }
-
-  dDebug("dnode rpc client is initialized");
-  return 0;
-}
-
-static void dndCleanupClient(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->clientRpc) {
-    rpcClose(pMgmt->clientRpc);
-    pMgmt->clientRpc = NULL;
-    dDebug("dnode rpc client is closed");
-  }
-}
-
-static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
-  SDnode     *pDnode = param;
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  tmsg_t msgType = pReq->msgType;
-  if (msgType == TDMT_DND_NETWORK_TEST) {
-    dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle);
-    dndProcessStartupReq(pDnode, pReq);
-    return;
-  }
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-    return;
-  } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) {
-    dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-    return;
-  }
-
-  if (pReq->pCont == NULL) {
-    dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    return;
-  }
-
-  DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
-  if (fp != NULL) {
-    dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    (*fp)(pDnode, pReq, pEpSet);
-  } else {
-    dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-  }
-}
-
-static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-  rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
-}
-
-static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  if (strcmp(user, INTERNAL_USER) == 0) {
-    char pass[TSDB_PASSWORD_LEN + 1] = {0};
-    taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
-    memcpy(secret, pass, TSDB_PASSWORD_LEN);
-    *spi = 1;
-    *encrypt = 0;
-    *ckey = 0;
-    return 0;
-  } else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
-    char pass[TSDB_PASSWORD_LEN + 1] = {0};
-    taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
-    memcpy(secret, pass, TSDB_PASSWORD_LEN);
-    *spi = 1;
-    *encrypt = 0;
-    *ckey = 0;
-    return 0;
-  } else {
-    return -1;
-  }
-}
-
-static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  SDnode *pDnode = parent;
-
-  if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) {
-    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
-    return 0;
-  }
-
-  if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) {
-    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
-    return 0;
-  }
-
-  if (terrno != TSDB_CODE_APP_NOT_READY) {
-    dTrace("failed to get user auth from mnode since %s", terrstr());
-    return -1;
-  }
-
-  SAuthReq authReq = {0};
-  tstrncpy(authReq.user, user, TSDB_USER_LEN);
-  int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void   *pReq = rpcMallocCont(contLen);
-  tSerializeSAuthReq(pReq, contLen, &authReq);
-
-  SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
-  SRpcMsg rpcRsp = {0};
-  dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
-  dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
-
-  if (rpcRsp.code != 0) {
-    terrno = rpcRsp.code;
-    dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
-  } else {
-    SAuthRsp authRsp = {0};
-    tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
-    memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
-    memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
-    *spi = authRsp.spi;
-    *encrypt = authRsp.encrypt;
-    dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
-           authRsp.encrypt);
-  }
-
-  rpcFreeCont(rpcRsp.pCont);
-  return rpcRsp.code;
-}
-
-static int32_t dndInitServer(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  dndInitMsgFp(pMgmt);
-
-  int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
-  if (numOfThreads < 1) {
-    numOfThreads = 1;
-  }
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.localPort = pDnode->cfg.serverPort;
-  rpcInit.label = "D-S";
-  rpcInit.numOfThreads = numOfThreads;
-  rpcInit.cfp = dndProcessRequest;
-  rpcInit.sessions = tsMaxShellConns;
-  rpcInit.connType = TAOS_CONN_SERVER;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.afp = dndRetrieveUserAuthInfo;
-  rpcInit.parent = pDnode;
-
-  pMgmt->serverRpc = rpcOpen(&rpcInit);
-  if (pMgmt->serverRpc == NULL) {
-    dError("failed to init dnode rpc server");
-    return -1;
-  }
-
-  dDebug("dnode rpc server is initialized");
-  return 0;
-}
-
-static void dndCleanupServer(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->serverRpc) {
-    rpcClose(pMgmt->serverRpc);
-    pMgmt->serverRpc = NULL;
-    dDebug("dnode rpc server is closed");
-  }
-}
-
-int32_t dndInitTrans(SDnode *pDnode) {
-  if (dndInitClient(pDnode) != 0) {
-    return -1;
-  }
-
-  if (dndInitServer(pDnode) != 0) {
-    return -1;
-  }
-
-  dInfo("dnode-transport is initialized");
-  return 0;
-}
-
-void dndCleanupTrans(SDnode *pDnode) {
-  dInfo("dnode-transport start to clean up");
-  dndCleanupServer(pDnode);
-  dndCleanupClient(pDnode);
-  dInfo("dnode-transport is cleaned up");
-}
-
-int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->clientRpc == NULL) {
-    terrno = TSDB_CODE_DND_OFFLINE;
-    return -1;
-  }
-
-  rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL);
-  return 0;
-}
-
-int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pReq) {
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-  return dndSendReqToDnode(pDnode, &epSet, pReq);
-}

source/dnode/mgmt/main/CMakeLists.txt

@@ -1,5 +1,5 @@
-aux_source_directory(src DAEMON_SRC)
+aux_source_directory(src EXEC_SRC)
-add_executable(taosd ${DAEMON_SRC})
+add_executable(taosd ${EXEC_SRC})
 
 target_include_directories(
     taosd

source/dnode/mgmt/main/inc/dndMain.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -13,40 +14,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_INT_H_
+#ifndef _TD_DND_MAIN_H_
-#define _TD_DND_INT_H_
+#define _TD_DND_MAIN_H_
+
+#include "dnode.h"
+
+#include "taoserror.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "version.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "os.h"
-
-#include "cJSON.h"
-#include "monitor.h"
-#include "tcache.h"
-#include "tcrc32c.h"
-#include "tdatablock.h"
-#include "tglobal.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tlog.h"
-#include "tmsg.h"
-#include "tqueue.h"
-#include "trpc.h"
-#include "tthread.h"
-#include "ttime.h"
-#include "tworker.h"
-
-#include "dnode.h"
-
-#include "bnode.h"
-#include "mnode.h"
-#include "qnode.h"
-#include "snode.h"
-#include "vnode.h"
-#include "tfs.h"
-
 #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
 #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
 #define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
@@ -54,20 +36,13 @@ extern "C" {
 #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
 #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
 
-typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
+void      dndDumpCfg();
-typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
+void      dndPrintVersion();
-typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat;
+void      dndGenerateGrant();
-typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
+SDnodeOpt dndGetOpt();
-
-EStat       dndGetStat(SDnode *pDnode);
-void        dndSetStat(SDnode *pDnode, EStat stat);
-const char *dndStatStr(EStat stat);
-
-void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_INT_H_*/
+#endif /*_TD_DND_MAIN_H_*/

source/dnode/mgmt/main/src/dndMain.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndMain.h"
+
+static struct {
+  bool    dumpConfig;
+  bool    generateGrant;
+  bool    printAuth;
+  bool    printVersion;
+  char    envFile[PATH_MAX];
+  char    apolloUrl[PATH_MAX];
+  SDnode *pDnode;
+} global = {0};
+
+static void dndSigintHandle(int signum, void *info, void *ctx) {
+  dInfo("signal:%d is received", signum);
+  SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
+  if (pDnode != NULL) {
+    dndHandleEvent(pDnode, DND_EVENT_STOP);
+  }
+}
+
+static void dndSetSignalHandle() {
+  taosSetSignal(SIGTERM, dndSigintHandle);
+  taosSetSignal(SIGHUP, dndSigintHandle);
+  taosSetSignal(SIGINT, dndSigintHandle);
+  taosSetSignal(SIGABRT, dndSigintHandle);
+  taosSetSignal(SIGBREAK, dndSigintHandle);
+}
+
+static int32_t dndParseOption(int32_t argc, char const *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-C") == 0) {
+      global.dumpConfig = true;
+    } else if (strcmp(argv[i], "-k") == 0) {
+      global.generateGrant = true;
+    } else if (strcmp(argv[i], "-V") == 0) {
+      global.printVersion = true;
+    } else {
+    }
+  }
+
+  return 0;
+}
+
+static int32_t dndRunDnode() {
+  if (dndInit() != 0) {
+    dInfo("failed to initialize dnode environment since %s", terrstr());
+    return -1;
+  }
+
+  SDnodeOpt option = dndGetOpt();
+
+  SDnode *pDnode = dndCreate(&option);
+  if (pDnode == NULL) {
+    dError("failed to to create dnode object since %s", terrstr());
+    return -1;
+  } else {
+    global.pDnode = pDnode;
+    dndSetSignalHandle();
+  }
+
+  dInfo("start the TDengine service");
+  int32_t code = dndRun(pDnode);
+  dInfo("start shutting down the TDengine service");
+
+  global.pDnode = NULL;
+  dndClose(pDnode);
+  dndCleanup();
+  taosCloseLog();
+  taosCleanupCfg();
+  return code;
+}
+
+int main(int argc, char const *argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    dError("failed to start TDengine since on non-small-end machines");
+    return -1;
+  }
+
+  if (dndParseOption(argc, argv) != 0) {
+    return -1;
+  }
+
+  if (global.generateGrant) {
+    dndGenerateGrant();
+    return 0;
+  }
+
+  if (global.printVersion) {
+    dndPrintVersion();
+    return 0;
+  }
+
+  if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
+    dError("failed to start TDengine since read log config error");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
+    dError("failed to start TDengine since read config error");
+    return -1;
+  }
+
+  if (global.dumpConfig) {
+    dndDumpCfg();
+    taosCleanupCfg();
+    taosCloseLog();
+    return 0;
+  }
+
+  return dndRunDnode();
+}

source/dnode/mgmt/main/src/dndUtil.c

@@ -14,15 +14,15 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dmnInt.h"
+#include "dndMain.h"
 
-void dmnGenerateGrant() {
+void dndGenerateGrant() {
 #if 0
   grantParseParameter();
 #endif
 }
 
-void dmnPrintVersion() {
+void dndPrintVersion() {
 #ifdef TD_ENTERPRISE
   char *releaseName = "enterprise";
 #else
@@ -32,3 +32,24 @@ void dmnPrintVersion() {
   printf("gitinfo: %s\n", gitinfo);
   printf("builuInfo: %s\n", buildinfo);
 }
+
+void dndDumpCfg() {
+  SConfig *pCfg = taosGetCfg();
+  cfgDumpCfg(pCfg, 0, 1);
+}
+
+SDnodeOpt dndGetOpt() {
+  SConfig  *pCfg = taosGetCfg();
+  SDnodeOpt option = {0};
+
+  option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
+  tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
+  tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
+  tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
+  option.serverPort = tsServerPort;
+  tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
+  snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
+  option.pDisks = tsDiskCfg;
+  option.numOfDisks = tsDiskCfgNum;
+  return option;
+}

source/dnode/mgmt/mnode/inc/mm.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_MNODE_H_
+#define _TD_DND_MNODE_H_
+
+#include "dnd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void mmGetMgmtFp(SMgmtWrapper *pMgmt);
+
+int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey);
+int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                         SMonGrantInfo *pGrantInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_MNODE_H_*/

source/dnode/mgmt/mnode/inc/mmInt.h

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_MNODE_INT_H_
+#define _TD_DND_MNODE_INT_H_
+
+#include "mm.h"
+#include "mnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SMnodeMgmt {
+  SMnode       *pMnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  readWorker;
+  SDnodeWorker  writeWorker;
+  SDnodeWorker  syncWorker;
+  SReplica      replicas[TSDB_MAX_REPLICA];
+  int8_t        replica;
+  int8_t        selfIndex;
+} SMnodeMgmt;
+
+// mmFile.c
+int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed);
+int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed);
+
+// mmInt.c
+int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq);
+int32_t mmDrop(SMgmtWrapper *pWrapper);
+int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq);
+
+// mmMsg.c
+void    mmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+// mmWorker.c
+int32_t mmStartWorker(SMnodeMgmt *pMgmt);
+void    mmStopWorker(SMnodeMgmt *pMgmt);
+int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
+int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_MNODE_INT_H_*/

source/dnode/mgmt/mnode/src/mmFile.c

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 4096;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_MNODE_OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_Number) {
+    dError("failed to read %s since deployed not found", file);
+    goto PRASE_MNODE_OVER;
+  }
+  *pDeployed = deployed->valueint;
+
+  cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
+  if (!mnodes || mnodes->type != cJSON_Array) {
+    dError("failed to read %s since nodes not found", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  pMgmt->replica = cJSON_GetArraySize(mnodes);
+  if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
+    dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
+    goto PRASE_MNODE_OVER;
+  }
+
+  for (int32_t i = 0; i < pMgmt->replica; ++i) {
+    cJSON *node = cJSON_GetArrayItem(mnodes, i);
+    if (node == NULL) break;
+
+    SReplica *pReplica = &pMgmt->replicas[i];
+
+    cJSON *id = cJSON_GetObjectItem(node, "id");
+    if (!id || id->type != cJSON_Number) {
+      dError("failed to read %s since id not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    pReplica->id = id->valueint;
+
+    cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
+    if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+      dError("failed to read %s since fqdn not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *port = cJSON_GetObjectItem(node, "port");
+    if (!port || port->type != cJSON_Number) {
+      dError("failed to read %s since port not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    pReplica->port = port->valueint;
+  }
+
+  code = 0;
+  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+
+PRASE_MNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);;
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 4096;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", deployed);
+  len += snprintf(content + len, maxLen - len, "  \"mnodes\": [{\n");
+  for (int32_t i = 0; i < pMgmt->replica; ++i) {
+    SReplica *pReplica = &pMgmt->replicas[i];
+    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pReplica->id);
+    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pReplica->fqdn);
+    len += snprintf(content + len, maxLen - len, "    \"port\": %u\n", pReplica->port);
+    if (i < pMgmt->replica - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);;
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s, deployed:%d", realfile, deployed);
+  return 0;
+}

source/dnode/mgmt/mnode/src/mmInt.c

@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+static bool mmDeployRequired(SDnode *pDnode) {
+  if (pDnode->dnodeId > 0) return false;
+  if (pDnode->clusterId > 0) return false;
+  if (strcmp(pDnode->localEp, pDnode->firstEp) != 0) return false;
+  return true;
+}
+
+static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) {
+  SMnodeMgmt mgmt = {0};
+  mgmt.path = pWrapper->path;
+  if (mmReadFile(&mgmt, required) != 0) {
+    return -1;
+  }
+
+  if (!(*required)) {
+    *required = mmDeployRequired(pWrapper->pDnode);
+  }
+
+  return 0;
+}
+
+static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->putToWriteQFp = mmPutMsgToWriteQueue;
+  pOption->putToReadQFp = mmPutMsgToReadQueue;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  mmInitOption(pMgmt, pOption);
+  pOption->replica = 1;
+  pOption->selfIndex = 0;
+  SReplica *pReplica = &pOption->replicas[0];
+  pReplica->id = 1;
+  pReplica->port = pDnode->serverPort;
+  tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN);
+
+  pMgmt->selfIndex = pOption->selfIndex;
+  pMgmt->replica = pOption->replica;
+  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+}
+
+static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  mmInitOption(pMgmt, pOption);
+  pOption->selfIndex = pMgmt->selfIndex;
+  pOption->replica = pMgmt->replica;
+  memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+}
+
+static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
+  mmInitOption(pMgmt, pOption);
+
+  pOption->replica = pCreate->replica;
+  pOption->selfIndex = -1;
+  for (int32_t i = 0; i < pCreate->replica; ++i) {
+    SReplica *pReplica = &pOption->replicas[i];
+    pReplica->id = pCreate->replicas[i].id;
+    pReplica->port = pCreate->replicas[i].port;
+    memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
+    if (pReplica->id == pOption->dnodeId) {
+      pOption->selfIndex = i;
+    }
+  }
+
+  if (pOption->selfIndex == -1) {
+    dError("failed to build mnode options since %s", terrstr());
+    return -1;
+  }
+
+  pMgmt->selfIndex = pOption->selfIndex;
+  pMgmt->replica = pOption->replica;
+  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+  return 0;
+}
+
+static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) {
+  SMnodeOpt option = {0};
+  if (pReq != NULL) {
+    if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
+      return -1;
+    }
+  } else {
+    bool deployed = false;
+    if (mmReadFile(pMgmt, &deployed) != 0) {
+      dError("failed to read file since %s", terrstr());
+      return -1;
+    }
+
+    if (!deployed) {
+      dInfo("mnode start to deploy");
+      mmBuildOptionForDeploy(pMgmt, &option);
+    } else {
+      dInfo("mnode start to open");
+      mmBuildOptionForOpen(pMgmt, &option);
+    }
+  }
+
+  pMgmt->pMnode = mndOpen(pMgmt->path, &option);
+  if (pMgmt->pMnode == NULL) {
+    dError("failed to open mnode since %s", terrstr());
+    return -1;
+  }
+
+  if (mmStartWorker(pMgmt) != 0) {
+    dError("failed to start mnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (mmWriteFile(pMgmt, deployed) != 0) {
+    dError("failed to write mnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void mmCloseImp(SMnodeMgmt *pMgmt) {
+  if (pMgmt->pMnode != NULL) {
+    mmStopWorker(pMgmt);
+    mndClose(pMgmt->pMnode);
+    pMgmt->pMnode = NULL;
+  }
+}
+
+int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) {
+  SMnodeOpt option = {0};
+  if (pReq != NULL) {
+    if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
+      return -1;
+    }
+  }
+
+  return mndAlter(pMgmt->pMnode, &option);
+}
+
+int32_t mmDrop(SMgmtWrapper *pWrapper) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("mnode-mgmt start to drop");
+  bool deployed = false;
+  if (mmWriteFile(pMgmt, deployed) != 0) {
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  }
+
+  mmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("mnode-mgmt is dropped");
+  return 0;
+}
+
+static void mmClose(SMgmtWrapper *pWrapper) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("mnode-mgmt start to cleanup");
+  mmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("mnode-mgmt is cleaned up");
+}
+
+int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) {
+  dInfo("mnode-mgmt start to init");
+  if (walInit() != 0) {
+    dError("failed to init wal since %s", terrstr());
+    return -1;
+  }
+
+  SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = mmOpenImp(pMgmt, pReq);
+  if (code != 0) {
+    dError("failed to init mnode-mgmt since %s", terrstr());
+    mmClose(pWrapper);
+  } else {
+    dInfo("mnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+static int32_t mmOpen(SMgmtWrapper *pWrapper) {
+  return mmOpenFromMsg(pWrapper, NULL);
+}
+
+static int32_t mmStart(SMgmtWrapper *pWrapper) {
+  dDebug("mnode mgmt start to run");
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mndStart(pMgmt->pMnode);
+}
+
+void mmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = mmOpen;
+  mgmtFp.closeFp = mmClose;
+  mgmtFp.startFp = mmStart;
+  mgmtFp.createMsgFp = mmProcessCreateReq;
+  mgmtFp.dropMsgFp = mmProcessDropReq;
+  mgmtFp.requiredFp = mmRequire;
+
+  mmInitMsgHandles(pWrapper);
+  pWrapper->name = "mnode";
+  pWrapper->fp = mgmtFp;
+}
+
+int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+
+  int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey);
+  dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
+  return code;
+}
+
+int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                         SMonGrantInfo *pGrantInfo) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
+}

source/dnode/mgmt/mnode/src/mmMsg.c

@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateMnodeReq createReq = {0};
+  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.replica <= 1 || createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmOpenFromMsg(pWrapper, &createReq);
+  }
+}
+
+int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropMnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmDrop(pWrapper);
+  }
+}
+
+int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pMgmt->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDAlterMnodeReq alterReq = {0};
+  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (alterReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to alter mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmAlter(pMgmt, &alterReq);
+  }
+}
+
+void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by DNODE
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+
+  // Requests handled by MNODE
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg);
+
+  // Requests handled by VNODE
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+}

source/dnode/mgmt/mnode/src/mmWorker.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in mnode queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = -1;
+
+  if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) {
+    pMsg->pNode = pMgmt->pMnode;
+    code = mndProcessMsg(pMsg);
+  } else {
+    code = mmProcessAlterReq(pMgmt, pMsg);
+  }
+
+  if (pRpc->msgType & 1U) {
+    if (pRpc->handle == NULL) return;
+    if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+      if (code != 0) code = terrno;
+      SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
+      dndSendRsp(pMgmt->pWrapper, &rsp);
+    }
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode read worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode write worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode sync worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void mmStopWorker(SMnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->readWorker);
+  dndCleanupWorker(&pMgmt->writeWorker);
+  dndCleanupWorker(&pMgmt->syncWorker);
+}
+
+static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) {
+  dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg);
+}
+
+int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg);
+}
+
+int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg);
+}
+
+static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) {
+  SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
+  if (pMsg == NULL) {
+    return -1;
+  }
+
+  dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
+  pMsg->rpcMsg = *pRpc;
+
+  int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
+  if (code != 0) {
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pRpc->pCont);
+  }
+
+  return code;
+}
+
+int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc);
+}
+
+int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc);
+}

source/dnode/mgmt/qnode/inc/qm.h

@@ -1,4 +1,3 @@
-
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -14,28 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DMN_INT_H_
+#ifndef _TD_DND_QNODE_H_
-#define _TD_DMN_INT_H_
+#define _TD_DND_QNODE_H_
 
-#include "tconfig.h"
+#include "dnd.h"
-#include "dnode.h"
-#include "taoserror.h"
-#include "tglobal.h"
-#include "tlog.h"
-#include "version.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-SDnodeObjCfg dmnGetObjCfg();
+void qmGetMgmtFp(SMgmtWrapper *pMgmt);
-
-void dmnDumpCfg();
-void dmnPrintVersion();
-void dmnGenerateGrant();
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DMN_INT_H_*/
+#endif /*_TD_DND_QNODE_H_*/

source/dnode/mgmt/qnode/inc/qmInt.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_QNODE_INT_H_
+#define _TD_DND_QNODE_INT_H_
+
+#include "qm.h"
+#include "qnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SQnodeMgmt {
+  SQnode       *pQnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  queryWorker;
+  SDnodeWorker  fetchWorker;
+} SQnodeMgmt;
+
+// qmInt.c
+int32_t qmOpen(SMgmtWrapper *pWrapper);
+int32_t qmDrop(SMgmtWrapper *pWrapper);
+
+// qmMsg.c
+void    qmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// qmWorker.c
+int32_t qmStartWorker(SQnodeMgmt *pMgmt);
+void    qmStopWorker(SQnodeMgmt *pMgmt);
+int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_QNODE_INT_H_*/

source/dnode/mgmt/qnode/src/qmInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t qmOpenImp(SQnodeMgmt *pMgmt) {
+  SQnodeOpt option = {0};
+  qmInitOption(pMgmt, &option);
+
+  pMgmt->pQnode = qndOpen(&option);
+  if (pMgmt->pQnode == NULL) {
+    dError("failed to open qnode since %s", terrstr());
+    return -1;
+  }
+
+  if (qmStartWorker(pMgmt) != 0) {
+    dError("failed to start qnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write qnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void qmCloseImp(SQnodeMgmt *pMgmt) {
+  if (pMgmt->pQnode != NULL) {
+    qmStopWorker(pMgmt);
+    qndClose(pMgmt->pQnode);
+    pMgmt->pQnode = NULL;
+  }
+}
+
+int32_t qmDrop(SMgmtWrapper *pWrapper) {
+  SQnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("qnode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop qnode since %s", terrstr());
+    return -1;
+  }
+
+  qmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("qnode-mgmt is dropped");
+  return 0;
+}
+
+static void qmClose(SMgmtWrapper *pWrapper) {
+  SQnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("qnode-mgmt start to cleanup");
+  qmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("qnode-mgmt is cleaned up");
+}
+
+int32_t qmOpen(SMgmtWrapper *pWrapper) {
+  dInfo("qnode-mgmt start to init");
+  SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = qmOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init qnode-mgmt since %s", terrstr());
+    qmClose(pWrapper);
+  } else {
+    dInfo("qnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void qmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = qmOpen;
+  mgmtFp.closeFp = qmClose;
+  mgmtFp.createMsgFp = qmProcessCreateReq;
+  mgmtFp.dropMsgFp = qmProcessDropReq;
+  mgmtFp.requiredFp = qmRequire;
+
+  qmInitMsgHandles(pWrapper);
+  pWrapper->name = "qnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/qnode/src/qmMsg.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateQnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create qnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return qmOpen(pWrapper);
+  }
+}
+
+int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropQnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop qnode since %s", terrstr());
+    return -1;
+  } else {
+    return qmDrop(pWrapper);
+  }
+}
+
+void qmInitMsgHandles(SMgmtWrapper *pWrapper) {}

source/dnode/mgmt/qnode/src/qmWorker.c

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in qnode queue", pMsg);
+  SRpcMsg *pRsp = NULL;
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp);
+
+  if (pRpc->msgType & 1u) {
+    if (pRsp != NULL) {
+      pRsp->ahandle = pRpc->ahandle;
+      dndSendRsp(pMgmt->pWrapper, pRsp);
+      free(pRsp);
+    } else {
+      if (code != 0) code = terrno;
+      SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+      dndSendRsp(pMgmt->pWrapper, &rpcRsp);
+    }
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->queryWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->fetchWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) {
+    dError("failed to start qnode query worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) {
+    dError("failed to start qnode fetch worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void qmStopWorker(SQnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->queryWorker);
+  dndCleanupWorker(&pMgmt->fetchWorker);
+}

source/dnode/mgmt/snode/inc/sm.h

@@ -13,21 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_TRANSPORT_H_
+#ifndef _TD_DND_SNODE_H_
-#define _TD_DND_TRANSPORT_H_
+#define _TD_DND_SNODE_H_
+
+#include "dnd.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitTrans(SDnode *pDnode);
+void smGetMgmtFp(SMgmtWrapper *pWrapper);
-void    dndCleanupTrans(SDnode *pDnode);
-int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_TRANSPORT_H_*/
+#endif /*_TD_DND_SNODE_H_*/

source/dnode/mgmt/snode/inc/smInt.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_SNODE_INT_H_
+#define _TD_DND_SNODE_INT_H_
+
+#include "sm.h"
+#include "snode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SSnodeMgmt {
+  SSnode       *pSnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SRWLatch      latch;
+  int8_t        uniqueWorkerInUse;
+  SArray       *uniqueWorkers;  // SArray<SDnodeWorker*>
+  SDnodeWorker  sharedWorker;
+} SSnodeMgmt;
+
+// smInt.c
+int32_t smOpen(SMgmtWrapper *pWrapper);
+int32_t smDrop(SMgmtWrapper *pWrapper);
+
+// smMsg.c
+void    smInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// smWorker.c
+int32_t smStartWorker(SSnodeMgmt *pMgmt);
+void    smStopWorker(SSnodeMgmt *pMgmt);
+int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_SNODE_INT_H_*/

source/dnode/mgmt/snode/src/smInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t smOpenImp(SSnodeMgmt *pMgmt) {
+  SSnodeOpt option = {0};
+  smInitOption(pMgmt, &option);
+
+  pMgmt->pSnode = sndOpen(pMgmt->path, &option);
+  if (pMgmt->pSnode == NULL) {
+    dError("failed to open snode since %s", terrstr());
+    return -1;
+  }
+
+  if (smStartWorker(pMgmt) != 0) {
+    dError("failed to start snode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write snode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void smCloseImp(SSnodeMgmt *pMgmt) {
+  if (pMgmt->pSnode != NULL) {
+    smStopWorker(pMgmt);
+    sndClose(pMgmt->pSnode);
+    pMgmt->pSnode = NULL;
+  }
+}
+
+int32_t smDrop(SMgmtWrapper *pWrapper) {
+  SSnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("snode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop snode since %s", terrstr());
+    return -1;
+  }
+
+  smCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("snode-mgmt is dropped");
+  return 0;
+}
+
+static void smClose(SMgmtWrapper *pWrapper) {
+  SSnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("snode-mgmt start to cleanup");
+  smCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("snode-mgmt is cleaned up");
+}
+
+int32_t smOpen(SMgmtWrapper *pWrapper) {
+  dInfo("snode-mgmt start to init");
+  SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = smOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init snode-mgmt since %s", terrstr());
+    smClose(pWrapper);
+  } else {
+    dInfo("snode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void smGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = smOpen;
+  mgmtFp.closeFp = smClose;
+  mgmtFp.createMsgFp = smProcessCreateReq;
+  mgmtFp.dropMsgFp = smProcessDropReq;
+  mgmtFp.requiredFp = smRequire;
+
+  smInitMsgHandles(pWrapper);
+  pWrapper->name = "snode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/snode/src/smMsg.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateSnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create snode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return smOpen(pWrapper);
+  }
+}
+
+int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropSnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop snode since %s", terrstr());
+    return -1;
+  } else {
+    return smDrop(pWrapper);
+  }
+}
+
+void smInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by SNODE
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg);
+}

source/dnode/mgmt/snode/src/smWorker.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) {
+  for (int32_t i = 0; i < numOfMsgs; i++) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+
+    dTrace("msg:%p, will be processed in snode unique queue", pMsg);
+    sndProcessUMsg(pMgmt->pSnode, &pMsg->rpcMsg);
+
+    dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in snode shared queue", pMsg);
+  sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg);
+
+  dTrace("msg:%p, is freed", pMsg);
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t smStartWorker(SSnodeMgmt *pMgmt) {
+  pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *));
+  if (pMgmt->uniqueWorkers == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
+    SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker));
+    if (pUniqueWorker == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    if (dndInitWorker(pMgmt, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, smProcessSharedQueue) != 0) {
+      dError("failed to start snode unique worker since %s", terrstr());
+      return -1;
+    }
+    if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM,
+                    SND_SHARED_THREAD_NUM, smProcessSharedQueue)) {
+    dError("failed to start snode shared worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void smStopWorker(SSnodeMgmt *pMgmt) {
+  for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
+    SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i);
+    dndCleanupWorker(worker);
+  }
+  taosArrayDestroy(pMgmt->uniqueWorkers);
+  dndCleanupWorker(&pMgmt->sharedWorker);
+}
+
+static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
+  SMsgHead *pHead = pMsg->pCont;
+  pHead->streamTaskId = htonl(pHead->streamTaskId);
+  return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM;
+}
+
+static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
+  SStreamExecMsgHead *pHead = pMsg->pCont;
+  pHead->workerType = htonl(pHead->workerType);
+  return pHead->workerType;
+}
+
+int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
+  if (pWorker == NULL) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  int32_t       index = smGetSWIdFromMsg(&pMsg->rpcMsg);
+  SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
+  if (pWorker == NULL) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->sharedWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg);
+  if (workerType == SND_WORKER_TYPE__SHARED) {
+    return smProcessSharedMsg(pMgmt, pMsg);
+  } else {
+    return smProcessUniqueMsg(pMgmt, pMsg);
+  }
+}

source/dnode/mgmt/test/bnode/dbnode.cpp

@@ -13,7 +13,10 @@
 
 class DndTestBnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); }
+  static void SetUpTestSuite() {
+    test.Init("/tmp/dnode_test_bnode", 9112);
+    taosMsleep(1100);
+  }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;
@@ -36,7 +39,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -62,7 +65,7 @@ TEST_F(DndTestBnode, 01_Create_Bnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 
   test.Restart();
@@ -76,11 +79,11 @@ TEST_F(DndTestBnode, 01_Create_Bnode) {
     tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq);
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 }
 
-TEST_F(DndTestBnode, 01_Drop_Bnode) {
+TEST_F(DndTestBnode, 02_Drop_Bnode) {
   {
     SDDropBnodeReq dropReq = {0};
     dropReq.dnodeId = 2;
@@ -91,7 +94,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -117,7 +120,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   test.Restart();
@@ -132,7 +135,7 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   {

source/dnode/mgmt/test/mnode/dmnode.cpp

@@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 
   {
@@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 
   {
@@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 }
 
@@ -96,7 +96,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -113,7 +113,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -145,7 +145,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   {
@@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   {

source/dnode/mgmt/test/qnode/dqnode.cpp

@@ -36,7 +36,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 
   test.Restart();
@@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 }
 
@@ -92,7 +92,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -118,7 +118,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   test.Restart();
@@ -133,7 +133,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   {

source/dnode/mgmt/test/snode/dsnode.cpp

@@ -36,7 +36,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 
   test.Restart();
@@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
   }
 }
 
@@ -92,7 +92,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_INVALID_OPTION);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_INVALID_OPTION);
   }
 
   {
@@ -118,7 +118,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   test.Restart();
@@ -133,7 +133,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
   }
 
   {

source/dnode/mgmt/test/sut/inc/server.h

@@ -24,11 +24,11 @@ class TestServer {
   bool DoStart();
 
  private:
-  SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp);
+  SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp);
 
  private:
   SDnode*    pDnode;
-  TdThread* threadId;
+  TdThread   threadId;
   char       path[PATH_MAX];
   char       fqdn[TSDB_FQDN_LEN];
   char       firstEp[TSDB_EP_LEN];

source/dnode/mgmt/test/sut/src/server.cpp

@@ -16,36 +16,37 @@
 #include "sut.h"
 
 void* serverLoop(void* param) {
-  while (1) {
+  SDnode* pDnode = (SDnode*)param;
-    taosMsleep(100);
+  dndRun(pDnode);
-    taosThreadTestCancel();
+  return NULL;
-  }
 }
 
-SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
+SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
-  SDnodeObjCfg cfg = {0};
+  SDnodeOpt option = {0};
-  cfg.numOfSupportVnodes = 16;
+  option.numOfSupportVnodes = 16;
-  cfg.serverPort = port;
+  option.serverPort = port;
-  strcpy(cfg.dataDir, path);
+  strcpy(option.dataDir, path);
-  snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port);
+  snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port);
-  snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn);
+  snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn);
-  snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp);
+  snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp);
-  return cfg;
+  return option;
 }
 
 bool TestServer::DoStart() {
-  SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp);
+  SDnodeOpt option = BuildOption(path, fqdn, port, firstEp);
   taosMkDir(path);
 
-  pDnode = dndCreate(&cfg);
+  pDnode = dndCreate(&option);
-  if (pDnode != NULL) {
+  if (pDnode == NULL) {
     return false;
   }
 
-  threadId = taosCreateThread(serverLoop, NULL);
+  TdThreadAttr thAttr;
-  if (threadId != NULL) {
+  taosThreadAttrInit(&thAttr);
-    return false;
+  taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
-  }
+  taosThreadCreate(&threadId, &thAttr, serverLoop, pDnode);
+  taosThreadAttrDestroy(&thAttr);
+  taosMsleep(2100);
   return true;
 }
 
@@ -67,10 +68,8 @@ bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const
 }
 
 void TestServer::Stop() {
-  if (threadId != NULL) {
+  dndHandleEvent(pDnode, DND_EVENT_STOP);
-    taosDestoryThread(threadId);
+  taosThreadJoin(threadId, NULL);
-    threadId = NULL;
-  }
 
   if (pDnode != NULL) {
     dndClose(pDnode);

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -21,8 +21,8 @@ void Testbase::InitLog(const char* path) {
   mDebugFlag = 143;
   cDebugFlag = 0;
   jniDebugFlag = 0;
-  tmrDebugFlag = 143;
+  tmrDebugFlag = 135;
-  uDebugFlag = 143;
+  uDebugFlag = 135;
   rpcDebugFlag = 143;
   qDebugFlag = 0;
   wDebugFlag = 0;
@@ -49,7 +49,6 @@ void Testbase::Init(const char* path, int16_t port) {
   InitLog("/tmp/td");
   server.Start(path, fqdn, port, firstEp);
   client.Init("root", "taosdata", fqdn, port);
-  taosMsleep(1100);
 
   tFreeSTableMetaRsp(&metaRsp);
   showId = 0;

source/dnode/mgmt/vnode/inc/vm.h

@@ -13,24 +13,34 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_QNODE_H_
+#ifndef _TD_DND_VNODES_H_
-#define _TD_DND_QNODE_H_
+#define _TD_DND_VNODES_H_
+
+#include "dnd.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitQnode(SDnode *pDnode);
+typedef struct {
-void    dndCleanupQnode(SDnode *pDnode);
+  int32_t openVnodes;
+  int32_t totalVnodes;
+  int32_t masterNum;
+  int64_t numOfSelectReqs;
+  int64_t numOfInsertReqs;
+  int64_t numOfInsertSuccessReqs;
+  int64_t numOfBatchInsertReqs;
+  int64_t numOfBatchInsertSuccessReqs;
+} SVnodesStat;
 
-void    dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void vmGetMgmtFp(SMgmtWrapper *pWrapper);
-void    dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+
-int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+void    vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads);
-int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo);
+void    vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_QNODE_H_*/
+#endif /*_TD_DND_VNODES_H_*/

source/dnode/mgmt/vnode/inc/vmInt.h

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_VNODES_INT_H_
+#define _TD_DND_VNODES_INT_H_
+
+#include "sync.h"
+#include "vm.h"
+#include "vnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SVnodesMgmt {
+  SHashObj     *hash;
+  SRWLatch      latch;
+  SVnodesStat   state;
+  STfs         *pTfs;
+  SQWorkerPool  queryPool;
+  SFWorkerPool  fetchPool;
+  SWWorkerPool  syncPool;
+  SWWorkerPool  writePool;
+  const char   *path;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  SDnodeWorker  mgmtWorker;
+} SVnodesMgmt;
+
+typedef struct {
+  int32_t  vgId;
+  int32_t  vgVersion;
+  int8_t   dropped;
+  uint64_t dbUid;
+  char     db[TSDB_DB_FNAME_LEN];
+  char     path[PATH_MAX + 20];
+} SWrapperCfg;
+
+typedef struct {
+  int32_t       vgId;
+  int32_t       refCount;
+  int32_t       vgVersion;
+  int8_t        dropped;
+  int8_t        accessState;
+  uint64_t      dbUid;
+  char         *db;
+  char         *path;
+  SVnode       *pImpl;
+  STaosQueue   *pWriteQ;
+  STaosQueue   *pSyncQ;
+  STaosQueue   *pApplyQ;
+  STaosQueue   *pQueryQ;
+  STaosQueue   *pFetchQ;
+  SMgmtWrapper *pWrapper;
+} SVnodeObj;
+
+typedef struct {
+  int32_t      vnodeNum;
+  int32_t      opened;
+  int32_t      failed;
+  int32_t      threadIndex;
+  TdThread     thread;
+  SVnodesMgmt *pMgmt;
+  SWrapperCfg *pCfgs;
+} SVnodeThread;
+
+// vmInt.c
+SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId);
+void       vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
+int32_t    vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl);
+void       vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
+
+// vmMsg.c
+void    vmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+
+// vmFile.c
+int32_t     vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
+int32_t     vmWriteVnodesToFile(SVnodesMgmt *pMgmt);
+SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes);
+
+// vmWorker.c
+int32_t vmStartWorker(SVnodesMgmt *pMgmt);
+void    vmStopWorker(SVnodesMgmt *pMgmt);
+int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
+void    vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
+
+int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
+int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, int32_t vgId, SRpcMsg *pMsg);
+
+int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_VNODES_INT_H_*/

source/dnode/mgmt/vnode/src/vmFile.c

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "vmInt.h"
+
+SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) {
+  taosRLockLatch(&pMgmt->latch);
+
+  int32_t     num = 0;
+  int32_t     size = taosHashGetSize(pMgmt->hash);
+  SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *));
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode && num < size) {
+      int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+      dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+      pVnodes[num] = (*ppVnode);
+      num++;
+      pIter = taosHashIterate(pMgmt->hash, pIter);
+    } else {
+      taosHashCancelIterate(pMgmt->hash, pIter);
+    }
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+  *numOfVnodes = num;
+
+  return pVnodes;
+}
+
+int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
+  int32_t      code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t      len = 0;
+  int32_t      maxLen = 30000;
+  char        *content = calloc(1, maxLen + 1);
+  cJSON       *root = NULL;
+  FILE        *fp = NULL;
+  char         file[PATH_MAX];
+  SWrapperCfg *pCfgs = NULL;
+  TdFilePtr    pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
+
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_VNODE_OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes");
+  if (!vnodes || vnodes->type != cJSON_Array) {
+    dError("failed to read %s since vnodes not found", file);
+    goto PRASE_VNODE_OVER;
+  }
+
+  int32_t vnodesNum = cJSON_GetArraySize(vnodes);
+  if (vnodesNum > 0) {
+    pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg));
+    if (pCfgs == NULL) {
+      dError("failed to read %s since out of memory", file);
+      goto PRASE_VNODE_OVER;
+    }
+
+    for (int32_t i = 0; i < vnodesNum; ++i) {
+      cJSON       *vnode = cJSON_GetArrayItem(vnodes, i);
+      SWrapperCfg *pCfg = &pCfgs[i];
+
+      cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
+      if (!vgId || vgId->type != cJSON_Number) {
+        dError("failed to read %s since vgId not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->vgId = vgId->valueint;
+      snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
+
+      cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
+      if (!dropped || dropped->type != cJSON_Number) {
+        dError("failed to read %s since dropped not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->dropped = dropped->valueint;
+
+      cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion");
+      if (!vgVersion || vgVersion->type != cJSON_Number) {
+        dError("failed to read %s since vgVersion not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->vgVersion = vgVersion->valueint;
+
+      cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid");
+      if (!dbUid || dbUid->type != cJSON_String) {
+        dError("failed to read %s since dbUid not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->dbUid = atoll(dbUid->valuestring);
+
+      cJSON *db = cJSON_GetObjectItem(vnode, "db");
+      if (!db || db->type != cJSON_String) {
+        dError("failed to read %s since db not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN);
+    }
+
+    *ppCfgs = pCfgs;
+  }
+
+  *numOfVnodes = vnodesNum;
+  code = 0;
+  dInfo("succcessed to read file %s", file);
+
+PRASE_VNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) {
+  char file[PATH_MAX];
+  char realfile[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t     numOfVnodes = 0;
+  SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes);
+
+  int32_t len = 0;
+  int32_t maxLen = 65536;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"vnodes\": [\n");
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    SVnodeObj *pVnode = pVnodes[i];
+    len += snprintf(content + len, maxLen - len, "    {\n");
+    len += snprintf(content + len, maxLen - len, "      \"vgId\": %d,\n", pVnode->vgId);
+    len += snprintf(content + len, maxLen - len, "      \"dropped\": %d,\n", pVnode->dropped);
+    len += snprintf(content + len, maxLen - len, "      \"vgVersion\": %d,\n", pVnode->vgVersion);
+    len += snprintf(content + len, maxLen - len, "      \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid);
+    len += snprintf(content + len, maxLen - len, "      \"db\": \"%s\"\n", pVnode->db);
+    if (i < numOfVnodes - 1) {
+      len += snprintf(content + len, maxLen - len, "    },\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "    }\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "  ]\n");
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+  terrno = 0;
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    SVnodeObj *pVnode = pVnodes[i];
+    vmReleaseVnode(pMgmt, pVnode);
+  }
+
+  if (pVnodes != NULL) {
+    free(pVnodes);
+  }
+
+  dDebug("successed to write %s", realfile);
+  return taosRenameFile(file, realfile);
+}

source/dnode/mgmt/vnode/src/vmInt.c

@@ -0,0 +1,410 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "vmInt.h"
+
+SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) {
+  SVnodeObj *pVnode = NULL;
+  int32_t    refCount = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+  taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
+  if (pVnode == NULL) {
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+  } else {
+    refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+  }
+  taosRUnLockLatch(&pMgmt->latch);
+
+  if (pVnode != NULL) {
+    dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+  }
+
+  return pVnode;
+}
+
+void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
+  if (pVnode == NULL) return;
+
+  taosRLockLatch(&pMgmt->latch);
+  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+  taosRUnLockLatch(&pMgmt->latch);
+  dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
+}
+
+int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
+  SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
+  if (pVnode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pVnode->vgId = pCfg->vgId;
+  pVnode->refCount = 0;
+  pVnode->dropped = 0;
+  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
+  pVnode->pWrapper = pMgmt->pWrapper;
+  pVnode->pImpl = pImpl;
+  pVnode->vgVersion = pCfg->vgVersion;
+  pVnode->dbUid = pCfg->dbUid;
+  pVnode->db = tstrdup(pCfg->db);
+  pVnode->path = tstrdup(pCfg->path);
+
+  if (pVnode->path == NULL || pVnode->db == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (vmAllocQueue(pMgmt, pVnode) != 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  taosWLockLatch(&pMgmt->latch);
+  int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+  taosWUnLockLatch(&pMgmt->latch);
+
+  if (code != 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return code;
+}
+
+void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
+  taosWLockLatch(&pMgmt->latch);
+  taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
+  taosWUnLockLatch(&pMgmt->latch);
+
+  vmReleaseVnode(pMgmt, pVnode);
+  while (pVnode->refCount > 0) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+
+  vmFreeQueue(pMgmt, pVnode);
+  vnodeClose(pVnode->pImpl);
+  pVnode->pImpl = NULL;
+
+  dDebug("vgId:%d, vnode is closed", pVnode->vgId);
+
+  if (pVnode->dropped) {
+    dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped);
+    vnodeDestroy(pVnode->path);
+  }
+
+  free(pVnode->path);
+  free(pVnode->db);
+  free(pVnode);
+}
+
+static void *vmOpenVnodeFunc(void *param) {
+  SVnodeThread *pThread = param;
+  SVnodesMgmt  *pMgmt = pThread->pMgmt;
+  SDnode       *pDnode = pMgmt->pDnode;
+
+  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+  setThreadName("open-vnodes");
+
+  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+    SWrapperCfg *pCfg = &pThread->pCfgs[v];
+
+    char stepDesc[TSDB_STEP_DESC_LEN] = {0};
+    snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
+             pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
+    dndReportStartup(pDnode, "open-vnodes", stepDesc);
+
+    SVnodeCfg cfg = {.pWrapper = pMgmt->pWrapper, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
+    SVnode   *pImpl = vnodeOpen(pCfg->path, &cfg);
+    if (pImpl == NULL) {
+      dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
+      pThread->failed++;
+    } else {
+      vmOpenVnode(pMgmt, pCfg, pImpl);
+      dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
+      pThread->opened++;
+    }
+
+    atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
+  }
+
+  dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+         pThread->failed);
+  return NULL;
+}
+
+static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) {
+  SDnode *pDnode = pMgmt->pDnode;
+  taosInitRWLatch(&pMgmt->latch);
+
+  pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pMgmt->hash == NULL) {
+    dError("failed to init vnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  SWrapperCfg *pCfgs = NULL;
+  int32_t      numOfVnodes = 0;
+  if (vmGetVnodesFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) {
+    dInfo("failed to get vnode list from disk since %s", terrstr());
+    return -1;
+  }
+
+  pMgmt->state.totalVnodes = numOfVnodes;
+
+#if 0 
+  int32_t threadNum = tsNumOfCores;
+#else
+  int32_t threadNum = 1;
+#endif
+  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+
+  SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
+  for (int32_t t = 0; t < threadNum; ++t) {
+    threads[t].threadIndex = t;
+    threads[t].pMgmt = pMgmt;
+    threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg));
+  }
+
+  for (int32_t v = 0; v < numOfVnodes; ++v) {
+    int32_t       t = v % threadNum;
+    SVnodeThread *pThread = &threads[t];
+    pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v];
+  }
+
+  dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum == 0) continue;
+
+    TdThreadAttr thAttr;
+    taosThreadAttrInit(&thAttr);
+    taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
+    if (taosThreadCreate(&pThread->thread, &thAttr, vmOpenVnodeFunc, pThread) != 0) {
+      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+    }
+
+    taosThreadAttrDestroy(&thAttr);
+  }
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
+      taosThreadJoin(pThread->thread, NULL);
+    }
+    free(pThread->pCfgs);
+  }
+  free(threads);
+  free(pCfgs);
+
+  if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) {
+    dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes);
+    return -1;
+  } else {
+    dInfo("total vnodes:%d open successfully", pMgmt->state.totalVnodes);
+    return 0;
+  }
+}
+
+static void vmCloseVnodes(SVnodesMgmt *pMgmt) {
+  dInfo("start to close all vnodes");
+
+  int32_t     numOfVnodes = 0;
+  SVnodeObj **pVnodes = vmGetVnodesFromHash(pMgmt, &numOfVnodes);
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    vmCloseVnode(pMgmt, pVnodes[i]);
+  }
+
+  if (pVnodes != NULL) {
+    free(pVnodes);
+  }
+
+  if (pMgmt->hash != NULL) {
+    taosHashCleanup(pMgmt->hash);
+    pMgmt->hash = NULL;
+  }
+
+  dInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static void vmCleanup(SMgmtWrapper *pWrapper) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("vnodes-mgmt start to cleanup");
+  vmCloseVnodes(pMgmt);
+  vmStopWorker(pMgmt);
+  vnodeCleanup();
+  // walCleanUp();
+  free(pMgmt);
+  pWrapper->pMgmt = NULL;
+  dInfo("vnodes-mgmt is cleaned up");
+}
+
+static int32_t vmInit(SMgmtWrapper *pWrapper) {
+  SDnode      *pDnode = pWrapper->pDnode;
+  SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt));
+  int32_t      code = -1;
+  SVnodeOpt    vnodeOpt = {0};
+
+  dInfo("vnodes-mgmt start to init");
+  if (pMgmt == NULL) goto _OVER;
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  taosInitRWLatch(&pMgmt->latch);
+
+  SDiskCfg dCfg = {0};
+  tstrncpy(dCfg.dir, pDnode->dataDir, TSDB_FILENAME_LEN);
+  dCfg.level = 0;
+  dCfg.primary = 1;
+  SDiskCfg *pDisks = pDnode->pDisks;
+  int32_t   numOfDisks = pDnode->numOfDisks;
+  if (numOfDisks <= 0 || pDisks == NULL) {
+    pDisks = &dCfg;
+    numOfDisks = 1;
+  }
+
+  pMgmt->pTfs = tfsOpen(pDisks, numOfDisks);
+  if (pMgmt->pTfs == NULL) {
+    dError("failed to init tfs since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (walInit() != 0) {
+    dError("failed to init wal since %s", terrstr());
+    goto _OVER;
+  }
+
+  vnodeOpt.nthreads = tsNumOfCommitThreads;
+  vnodeOpt.putToQueryQFp = vmPutMsgToQueryQueue;
+  vnodeOpt.sendReqFp = dndSendReqToDnode;
+  if (vnodeInit(&vnodeOpt) != 0) {
+    dError("failed to init vnode since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (vmStartWorker(pMgmt) != 0) {
+    dError("failed to init workers since %s", terrstr()) goto _OVER;
+  }
+
+  if (vmOpenVnodes(pMgmt) != 0) {
+    dError("failed to open vnodes since %s", terrstr());
+    return -1;
+  }
+
+  code = 0;
+
+_OVER:
+  if (code == 0) {
+    pWrapper->pMgmt = pMgmt;
+    dInfo("vnodes-mgmt is initialized");
+  } else {
+    dError("failed to init vnodes-mgmt since %s", terrstr());
+    vmCleanup(pWrapper);
+  }
+
+  return 0;
+}
+
+static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) {
+  SDnode *pDnode = pWrapper->pDnode;
+  *required = pDnode->numOfSupportVnodes > 0;
+  return 0;
+}
+
+void vmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = vmInit;
+  mgmtFp.closeFp = vmCleanup;
+  mgmtFp.requiredFp = vmRequire;
+
+  vmInitMsgHandles(pWrapper);
+  pWrapper->name = "vnodes";
+  pWrapper->fp = mgmtFp;
+}
+
+int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return -1;
+
+  return tfsGetMonitorInfo(pMgmt->pTfs, pInfo);
+}
+
+void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  SVnodesStat *pStat = &pMgmt->state;
+  pInfo->req_select = pStat->numOfSelectReqs;
+  pInfo->req_insert = pStat->numOfInsertReqs;
+  pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
+  pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
+  pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
+  pInfo->errors = tsNumOfErrorLogs;
+  pInfo->vnodes_num = pStat->totalVnodes;
+  pInfo->masters = pStat->masterNum;
+}
+
+void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  SVnodesStat *pStat = &pMgmt->state;
+  int32_t      totalVnodes = 0;
+  int32_t      masterNum = 0;
+  int64_t      numOfSelectReqs = 0;
+  int64_t      numOfInsertReqs = 0;
+  int64_t      numOfInsertSuccessReqs = 0;
+  int64_t      numOfBatchInsertReqs = 0;
+  int64_t      numOfBatchInsertSuccessReqs = 0;
+
+  taosRLockLatch(&pMgmt->latch);
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    if (ppVnode == NULL || *ppVnode == NULL) continue;
+
+    SVnodeObj *pVnode = *ppVnode;
+    SVnodeLoad vload = {0};
+    vnodeGetLoad(pVnode->pImpl, &vload);
+    taosArrayPush(pLoads, &vload);
+
+    numOfSelectReqs += vload.numOfSelectReqs;
+    numOfInsertReqs += vload.numOfInsertReqs;
+    numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs;
+    numOfBatchInsertReqs += vload.numOfBatchInsertReqs;
+    numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs;
+    totalVnodes++;
+    if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++;
+
+    pIter = taosHashIterate(pMgmt->hash, pIter);
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+
+  pStat->totalVnodes = totalVnodes;
+  pStat->masterNum = masterNum;
+  pStat->numOfSelectReqs = numOfSelectReqs;
+  pStat->numOfInsertReqs = numOfInsertReqs;
+  pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
+  pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
+  pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
+}

source/dnode/mgmt/vnode/src/vmMsg.c

@@ -0,0 +1,274 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "vmInt.h"
+
+static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
+  pCfg->vgId = pCreate->vgId;
+  pCfg->wsize = pCreate->cacheBlockSize;
+  pCfg->ssize = pCreate->cacheBlockSize;
+  pCfg->lsize = pCreate->cacheBlockSize;
+  pCfg->isHeapAllocator = true;
+  pCfg->ttl = 4;
+  pCfg->keep = pCreate->daysToKeep0;
+  pCfg->streamMode = pCreate->streamMode;
+  pCfg->isWeak = true;
+  pCfg->tsdbCfg.keep = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
+  pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
+  pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
+  pCfg->walCfg.level = pCreate->walLevel;
+  pCfg->walCfg.retentionPeriod = 10;
+  pCfg->walCfg.retentionSize = 128;
+  pCfg->walCfg.rollPeriod = 128;
+  pCfg->walCfg.segSize = 128;
+  pCfg->walCfg.vgId = pCreate->vgId;
+  pCfg->hashBegin = pCreate->hashBegin;
+  pCfg->hashEnd = pCreate->hashEnd;
+  pCfg->hashMethod = pCreate->hashMethod;
+}
+
+static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
+  memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN);
+  pCfg->dbUid = pCreate->dbUid;
+  pCfg->dropped = 0;
+  snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId);
+  pCfg->vgId = pCreate->vgId;
+  pCfg->vgVersion = pCreate->vgVersion;
+}
+
+int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg        *pReq = &pMsg->rpcMsg;
+  SCreateVnodeReq createReq = {0};
+  if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dDebug("vgId:%d, create vnode req is received", createReq.vgId);
+
+  SVnodeCfg vnodeCfg = {0};
+  vmGenerateVnodeCfg(&createReq, &vnodeCfg);
+
+  SWrapperCfg wrapperCfg = {0};
+  vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
+
+  if (createReq.dnodeId != pMgmt->pDnode->dnodeId) {
+    terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION;
+    dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
+    return -1;
+  }
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
+  if (pVnode != NULL) {
+    dDebug("vgId:%d, already exist", createReq.vgId);
+    vmReleaseVnode(pMgmt, pVnode);
+    terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED;
+    return -1;
+  }
+
+  vnodeCfg.pWrapper = pMgmt->pWrapper;
+  vnodeCfg.pTfs = pMgmt->pTfs;
+  vnodeCfg.dbId = wrapperCfg.dbUid;
+  SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
+  if (pImpl == NULL) {
+    dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
+    return -1;
+  }
+
+  int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
+  if (code != 0) {
+    dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
+    vnodeClose(pImpl);
+    vnodeDestroy(wrapperCfg.path);
+    terrno = code;
+    return code;
+  }
+
+  code = vmWriteVnodesToFile(pMgmt);
+  if (code != 0) {
+    vnodeClose(pImpl);
+    vnodeDestroy(wrapperCfg.path);
+    terrno = code;
+    return code;
+  }
+
+  return 0;
+}
+
+int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg       *pReq = &pMsg->rpcMsg;
+  SAlterVnodeReq alterReq = {0};
+  if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dDebug("vgId:%d, alter vnode req is received", alterReq.vgId);
+
+  SVnodeCfg vnodeCfg = {0};
+  vmGenerateVnodeCfg(&alterReq, &vnodeCfg);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, alterReq.vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr());
+    return -1;
+  }
+
+  if (alterReq.vgVersion == pVnode->vgVersion) {
+    vmReleaseVnode(pMgmt, pVnode);
+    dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId);
+    return 0;
+  }
+
+  if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) {
+    dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr());
+    vmReleaseVnode(pMgmt, pVnode);
+    return -1;
+  }
+
+  int32_t oldVersion = pVnode->vgVersion;
+  pVnode->vgVersion = alterReq.vgVersion;
+  int32_t code = vmWriteVnodesToFile(pMgmt);
+  if (code != 0) {
+    pVnode->vgVersion = oldVersion;
+  }
+
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg      *pReq = &pMsg->rpcMsg;
+  SDropVnodeReq dropReq = {0};
+  if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  int32_t vgId = dropReq.vgId;
+  dDebug("vgId:%d, drop vnode req is received", vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
+    terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED;
+    return -1;
+  }
+
+  pVnode->dropped = 1;
+  if (vmWriteVnodesToFile(pMgmt) != 0) {
+    pVnode->dropped = 0;
+    vmReleaseVnode(pMgmt, pVnode);
+    return -1;
+  }
+
+  vmCloseVnode(pMgmt, pVnode);
+  vmWriteVnodesToFile(pMgmt);
+
+  return 0;
+}
+
+int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg      *pReq = &pMsg->rpcMsg;
+  SSyncVnodeReq syncReq = {0};
+  tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq);
+
+  int32_t vgId = syncReq.vgId;
+  dDebug("vgId:%d, sync vnode req is received", vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to sync since %s", vgId, terrstr());
+    return -1;
+  }
+
+  if (vnodeSync(pVnode->pImpl) != 0) {
+    dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr());
+    vmReleaseVnode(pMgmt, pVnode);
+    return -1;
+  }
+
+  vmReleaseVnode(pMgmt, pVnode);
+  return 0;
+}
+
+int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg         *pReq = &pMsg->rpcMsg;
+  SCompactVnodeReq compatcReq = {0};
+  tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq);
+
+  int32_t vgId = compatcReq.vgId;
+  dDebug("vgId:%d, compact vnode req is received", vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
+  if (pVnode == NULL) {
+    dDebug("vgId:%d, failed to compact since %s", vgId, terrstr());
+    return -1;
+  }
+
+  if (vnodeCompact(pVnode->pImpl) != 0) {
+    dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr());
+    vmReleaseVnode(pMgmt, pVnode);
+    return -1;
+  }
+
+  vmReleaseVnode(pMgmt, pVnode);
+  return 0;
+}
+
+void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by VNODE
+  dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg);
+
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg);
+}

source/dnode/mgmt/vnode/src/vmWorker.c

@@ -0,0 +1,299 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "vmInt.h"
+
+static void vmProcessQueryQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in vnode query queue", pMsg);
+  vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg);
+}
+
+static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in vnode fetch queue", pMsg);
+  vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg);
+}
+
+static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) {
+  SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+    dTrace("msg:%p, will be processed in vnode write queue", pMsg);
+    void *ptr = taosArrayPush(pArray, &pMsg);
+    assert(ptr != NULL);
+  }
+
+  vnodeProcessWMsgs(pVnode->pImpl, pArray);
+
+  for (size_t i = 0; i < numOfMsgs; i++) {
+    SRpcMsg  *pRsp = NULL;
+    SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    SRpcMsg  *pRpc = &pMsg->rpcMsg;
+    int32_t   code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp);
+    if (pRsp != NULL) {
+      pRsp->ahandle = pRpc->ahandle;
+      dndSendRsp(pVnode->pWrapper, pRsp);
+      free(pRsp);
+    } else {
+      if (code != 0) code = terrno;
+      SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+      dndSendRsp(pVnode->pWrapper, &rpcRsp);
+    }
+  }
+
+  for (size_t i = 0; i < numOfMsgs; i++) {
+    SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
+  }
+
+  taosArrayDestroy(pArray);
+}
+
+static void vmProcessApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) {
+  SNodeMsg *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    taosGetQitem(qall, (void **)&pMsg);
+
+    // todo
+    SRpcMsg *pRsp = NULL;
+    (void)vnodeApplyWMsg(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+  }
+}
+
+static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) {
+  SNodeMsg *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    taosGetQitem(qall, (void **)&pMsg);
+
+    // todo
+    SRpcMsg *pRsp = NULL;
+    (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+  }
+}
+
+static SVnodeObj *vmAcquireFromMsg(SVnodesMgmt *pMgmt, SNodeMsg *pNodeMsg) {
+  SRpcMsg *pMsg = &pNodeMsg->rpcMsg;
+
+  SMsgHead *pHead = pMsg->pCont;
+  pHead->contLen = htonl(pHead->contLen);
+  pHead->vgId = htonl(pHead->vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
+  if (pVnode == NULL) {
+    dError("vgId:%d, failed to acquire vnode while process req", pHead->vgId);
+  }
+
+  return pVnode;
+}
+
+int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg);
+  if (pVnode == NULL) return -1;
+
+  int32_t code = taosWriteQitem(pVnode->pWriteQ, pMsg);
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg);
+  if (pVnode == NULL) return -1;
+
+  int32_t code = taosWriteQitem(pVnode->pSyncQ, pMsg);
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg);
+  if (pVnode == NULL) return -1;
+
+  int32_t code = taosWriteQitem(pVnode->pQueryQ, pMsg);
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SVnodeObj *pVnode = vmAcquireFromMsg(pMgmt, pMsg);
+  if (pVnode == NULL) return -1;
+
+  int32_t code = taosWriteQitem(pVnode->pFetchQ, pMsg);
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+
+  int32_t   code = -1;
+  SMsgHead *pHead = pRpc->pCont;
+  // pHead->vgId = htonl(pHead->vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
+  if (pVnode == NULL) return -1;
+
+  SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
+  if (pMsg != NULL) {
+    pMsg->rpcMsg = *pRpc;
+    code = taosWriteQitem(pVnode->pQueryQ, pMsg);
+  }
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, int32_t vgId, SRpcMsg *pRpc) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+
+  int32_t   code = -1;
+  SMsgHead *pHead = pRpc->pCont;
+  // pHead->vgId = htonl(pHead->vgId);
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
+  if (pVnode == NULL) return -1;
+
+  SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
+  if (pMsg != NULL) {
+    pMsg->rpcMsg = *pRpc;
+    code = taosWriteQitem(pVnode->pApplyQ, pMsg);
+  }
+  vmReleaseVnode(pMgmt, pVnode);
+  return code;
+}
+
+int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
+  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue);
+  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue);
+  pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
+  pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
+  pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
+
+  if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL ||
+      pVnode->pQueryQ == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
+  tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
+  tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
+  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
+  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
+  tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
+  pVnode->pWriteQ = NULL;
+  pVnode->pApplyQ = NULL;
+  pVnode->pSyncQ = NULL;
+  pVnode->pFetchQ = NULL;
+  pVnode->pQueryQ = NULL;
+}
+
+static void vmProcessMgmtQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  int32_t code = -1;
+  tmsg_t  msgType = pMsg->rpcMsg.msgType;
+  dTrace("msg:%p, will be processed in vnode mgmt queue", pMsg);
+
+  switch (msgType) {
+    case TDMT_DND_CREATE_VNODE:
+      code = vmProcessCreateVnodeReq(pMgmt, pMsg);
+      break;
+    case TDMT_DND_ALTER_VNODE:
+      code = vmProcessAlterVnodeReq(pMgmt, pMsg);
+      break;
+    case TDMT_DND_DROP_VNODE:
+      code = vmProcessDropVnodeReq(pMgmt, pMsg);
+      break;
+    case TDMT_DND_SYNC_VNODE:
+      code = vmProcessSyncVnodeReq(pMgmt, pMsg);
+      break;
+    case TDMT_DND_COMPACT_VNODE:
+      code = vmProcessCompactVnodeReq(pMgmt, pMsg);
+      break;
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      dError("msg:%p, not processed in mgmt queue", pMsg);
+  }
+
+  if (msgType & 1u) {
+    if (code != 0) code = terrno;
+    SRpcMsg rsp = {.code = code, .handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle};
+    dndSendRsp(pMgmt->pWrapper, &rsp);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
+  int32_t maxFetchThreads = 4;
+  int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
+  int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
+  int32_t maxQueryThreads = minQueryThreads;
+  int32_t maxWriteThreads = TMAX(tsNumOfCores, 1);
+  int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1);
+
+  SQWorkerPool *pQPool = &pMgmt->queryPool;
+  pQPool->name = "vnode-query";
+  pQPool->min = minQueryThreads;
+  pQPool->max = maxQueryThreads;
+  if (tQWorkerInit(pQPool) != 0) return -1;
+
+  SFWorkerPool *pFPool = &pMgmt->fetchPool;
+  pFPool->name = "vnode-fetch";
+  pFPool->min = minFetchThreads;
+  pFPool->max = maxFetchThreads;
+  if (tFWorkerInit(pFPool) != 0) return -1;
+
+  SWWorkerPool *pWPool = &pMgmt->writePool;
+  pWPool->name = "vnode-write";
+  pWPool->max = maxWriteThreads;
+  if (tWWorkerInit(pWPool) != 0) return -1;
+
+  pWPool = &pMgmt->syncPool;
+  pWPool->name = "vnode-sync";
+  pWPool->max = maxSyncThreads;
+  if (tWWorkerInit(pWPool) != 0) return -1;
+
+  if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "vnode-mgmt", 1, 1, vmProcessMgmtQueue) != 0) {
+    dError("failed to start dnode mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  dDebug("vnode workers is initialized");
+  return 0;
+}
+
+void vmStopWorker(SVnodesMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->mgmtWorker);
+  tFWorkerCleanup(&pMgmt->fetchPool);
+  tQWorkerCleanup(&pMgmt->queryPool);
+  tWWorkerCleanup(&pMgmt->writePool);
+  tWWorkerCleanup(&pMgmt->syncPool);
+  dDebug("vnode workers is closed");
+}
+
+int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->mgmtWorker;
+  dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}

source/dnode/mnode/impl/inc/mndDef.h

@@ -702,16 +702,6 @@ typedef struct {
 int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);
 int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj);
 
-typedef struct SMnodeMsg {
-  char    user[TSDB_USER_LEN];
-  char    db[TSDB_DB_FNAME_LEN];
-  int32_t acctId;
-  SMnode* pMnode;
-  int64_t createdTime;
-  SRpcMsg rpcMsg;
-  int32_t contLen;
-  void*   pCont;
-} SMnodeMsg;
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndInt.h

@@ -38,11 +38,11 @@ extern "C" {
 #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
 #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
 
-typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
+typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg);
 typedef int32_t (*MndInitFp)(SMnode *pMnode);
 typedef void (*MndCleanupFp)(SMnode *pMnode);
-typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
+typedef int32_t (*ShowMetaFp)(SNodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
-typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
+typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
 typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
 
 typedef struct SMnodeLoad {
@@ -110,7 +110,7 @@ typedef struct SMnode {
   char             *path;
   int64_t           checkTime;
   SSdb             *pSdb;
-  SDnode           *pDnode;
+  SMgmtWrapper     *pWrapper;
   SArray           *pSteps;
   SShowMgmt         showMgmt;
   SProfileMgmt      profileMgmt;
@@ -119,16 +119,14 @@ typedef struct SMnode {
   SHashObj         *infosMeta;
   SGrantInfo        grant;
   MndMsgFp          msgFp[TDMT_MAX];
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp         sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp    sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  PutToQueueFp      putToWriteQFp;
-  PutReqToMWriteQFp putReqToMWriteQFp;
+  PutToQueueFp      putToReadQFp;
-  PutReqToMReadQFp  putReqToMReadQFp;
 } SMnode;
 
 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg);
 int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg);
-void    mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg);
 void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
 
 uint64_t mndGenerateUid(char *name, int32_t len);