Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

contrib/CMakeLists.txt

@@ -213,6 +213,7 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
+    add_compile_options(-Wno-sign-compare)
     if (${TD_WINDOWS})
         file(READ "libuv/include/uv.h" CONTENTS)
         string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}")

include/common/tglobal.h

@@ -51,6 +51,7 @@ extern int32_t tsCompatibleModel;
 extern bool    tsEnableSlaveQuery;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerDay[3];
+extern int32_t tsMultiProcess;
 
 // monitor
 extern bool     tsEnableMonitor;

include/common/tmsg.h

@@ -115,11 +115,12 @@ typedef enum _mgmt_table {
 #define TSDB_ALTER_TABLE_DROP_TAG            2
 #define TSDB_ALTER_TABLE_UPDATE_TAG_NAME     3
 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL      4
-
 #define TSDB_ALTER_TABLE_ADD_COLUMN          5
 #define TSDB_ALTER_TABLE_DROP_COLUMN         6
 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES    8
+#define TSDB_ALTER_TABLE_UPDATE_OPTIONS      9
+#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME  10
 
 #define TSDB_FILL_NONE      0
 #define TSDB_FILL_NULL      1
@@ -696,6 +697,7 @@ typedef struct {
 
 int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
 int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
+void    tFreeSStatusRsp(SStatusRsp* pRsp);
 
 typedef struct {
   int32_t reserved;
@@ -1127,10 +1129,10 @@ typedef struct {
 
 typedef struct {
   char   name[TSDB_TOPIC_FNAME_LEN];
+  char   outputTbName[TSDB_TABLE_NAME_LEN];
   int8_t igExists;
   char*  sql;
-  char*  physicalPlan;
+  char*  ast;
-  char*  logicalPlan;
 } SCMCreateStreamReq;
 
 typedef struct {
@@ -1919,12 +1921,13 @@ typedef struct {
 
 typedef struct {
   int8_t  type;  // 0 status report, 1 update data
-  char        indexName[TSDB_INDEX_NAME_LEN];  //
+  int64_t indexUid;
-  STimeWindow windows;
+  int64_t skey;  // start TS key of interval/sliding window
 } STSmaMsg;
 
 typedef struct {
   int64_t ver;  // use a general definition
+  int64_t indexUid;
   char    indexName[TSDB_INDEX_NAME_LEN];
 } SVDropTSmaReq;
 
@@ -2271,13 +2274,23 @@ enum {
   STREAM_TASK_STATUS__STOP,
 };
 
+typedef struct {
+  void*  inputHandle;
+  void** executor;
+} SStreamTaskParRunner;
+
 typedef struct {
   int64_t streamId;
   int32_t taskId;
   int32_t level;
   int8_t  status;
+  int8_t  pipeEnd;
+  int8_t  parallel;
+  SEpSet  NextOpEp;
   char*   qmsg;
-  void*   executor;
+  // not applied to encoder and decoder
+  SStreamTaskParRunner runner;
+  // void*                executor;
   // void*   stateStore;
   //  storage handle
 } SStreamTask;
@@ -2317,6 +2330,14 @@ typedef struct {
 
 #pragma pack(pop)
 
+struct SRpcMsg;
+struct SEpSet;
+struct SMgmtWrapper;
+typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq);
+typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq);
+typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq);
+typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp);
+
 #ifdef __cplusplus
 }
 #endif

include/common/tmsgdef.h

@@ -190,6 +190,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
 
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)

include/common/ttokendef.h

@@ -34,128 +34,147 @@
 #define TK_NK_REM                          16
 #define TK_NK_CONCAT                       17
 #define TK_CREATE                          18
-#define TK_USER                            19
+#define TK_ACCOUNT                         19
-#define TK_PASS                            20
+#define TK_NK_ID                           20
-#define TK_NK_STRING                       21
+#define TK_PASS                            21
-#define TK_ALTER                           22
+#define TK_NK_STRING                       22
-#define TK_PRIVILEGE                       23
+#define TK_ALTER                           23
-#define TK_DROP                            24
+#define TK_PPS                             24
-#define TK_SHOW                            25
+#define TK_TSERIES                         25
-#define TK_USERS                           26
+#define TK_STORAGE                         26
-#define TK_DNODE                           27
+#define TK_STREAMS                         27
-#define TK_PORT                            28
+#define TK_QTIME                           28
-#define TK_NK_INTEGER                      29
+#define TK_DBS                             29
-#define TK_DNODES                          30
+#define TK_USERS                           30
-#define TK_NK_ID                           31
+#define TK_CONNS                           31
-#define TK_NK_IPTOKEN                      32
+#define TK_STATE                           32
-#define TK_QNODE                           33
+#define TK_USER                            33
-#define TK_ON                              34
+#define TK_PRIVILEGE                       34
-#define TK_QNODES                          35
+#define TK_DROP                            35
-#define TK_DATABASE                        36
+#define TK_SHOW                            36
-#define TK_DATABASES                       37
+#define TK_DNODE                           37
-#define TK_USE                             38
+#define TK_PORT                            38
-#define TK_IF                              39
+#define TK_NK_INTEGER                      39
-#define TK_NOT                             40
+#define TK_DNODES                          40
-#define TK_EXISTS                          41
+#define TK_NK_IPTOKEN                      41
-#define TK_BLOCKS                          42
+#define TK_LOCAL                           42
-#define TK_CACHE                           43
+#define TK_QNODE                           43
-#define TK_CACHELAST                       44
+#define TK_ON                              44
-#define TK_COMP                            45
+#define TK_QNODES                          45
-#define TK_DAYS                            46
+#define TK_DATABASE                        46
-#define TK_FSYNC                           47
+#define TK_DATABASES                       47
-#define TK_MAXROWS                         48
+#define TK_USE                             48
-#define TK_MINROWS                         49
+#define TK_IF                              49
-#define TK_KEEP                            50
+#define TK_NOT                             50
-#define TK_PRECISION                       51
+#define TK_EXISTS                          51
-#define TK_QUORUM                          52
+#define TK_BLOCKS                          52
-#define TK_REPLICA                         53
+#define TK_CACHE                           53
-#define TK_TTL                             54
+#define TK_CACHELAST                       54
-#define TK_WAL                             55
+#define TK_COMP                            55
-#define TK_VGROUPS                         56
+#define TK_DAYS                            56
-#define TK_SINGLE_STABLE                   57
+#define TK_FSYNC                           57
-#define TK_STREAM_MODE                     58
+#define TK_MAXROWS                         58
-#define TK_TABLE                           59
+#define TK_MINROWS                         59
-#define TK_NK_LP                           60
+#define TK_KEEP                            60
-#define TK_NK_RP                           61
+#define TK_PRECISION                       61
-#define TK_STABLE                          62
+#define TK_QUORUM                          62
-#define TK_TABLES                          63
+#define TK_REPLICA                         63
-#define TK_STABLES                         64
+#define TK_TTL                             64
-#define TK_USING                           65
+#define TK_WAL                             65
-#define TK_TAGS                            66
+#define TK_VGROUPS                         66
-#define TK_NK_DOT                          67
+#define TK_SINGLE_STABLE                   67
-#define TK_NK_COMMA                        68
+#define TK_STREAM_MODE                     68
-#define TK_COMMENT                         69
+#define TK_RETENTIONS                      69
-#define TK_BOOL                            70
+#define TK_FILE_FACTOR                     70
-#define TK_TINYINT                         71
+#define TK_NK_FLOAT                        71
-#define TK_SMALLINT                        72
+#define TK_TABLE                           72
-#define TK_INT                             73
+#define TK_NK_LP                           73
-#define TK_INTEGER                         74
+#define TK_NK_RP                           74
-#define TK_BIGINT                          75
+#define TK_STABLE                          75
-#define TK_FLOAT                           76
+#define TK_TABLES                          76
-#define TK_DOUBLE                          77
+#define TK_STABLES                         77
-#define TK_BINARY                          78
+#define TK_ADD                             78
-#define TK_TIMESTAMP                       79
+#define TK_COLUMN                          79
-#define TK_NCHAR                           80
+#define TK_MODIFY                          80
-#define TK_UNSIGNED                        81
+#define TK_RENAME                          81
-#define TK_JSON                            82
+#define TK_TAG                             82
-#define TK_VARCHAR                         83
+#define TK_SET                             83
-#define TK_MEDIUMBLOB                      84
+#define TK_NK_EQ                           84
-#define TK_BLOB                            85
+#define TK_USING                           85
-#define TK_VARBINARY                       86
+#define TK_TAGS                            86
-#define TK_DECIMAL                         87
+#define TK_NK_DOT                          87
-#define TK_SMA                             88
+#define TK_NK_COMMA                        88
-#define TK_INDEX                           89
+#define TK_COMMENT                         89
-#define TK_FULLTEXT                        90
+#define TK_BOOL                            90
-#define TK_FUNCTION                        91
+#define TK_TINYINT                         91
-#define TK_INTERVAL                        92
+#define TK_SMALLINT                        92
-#define TK_TOPIC                           93
+#define TK_INT                             93
-#define TK_AS                              94
+#define TK_INTEGER                         94
-#define TK_MNODES                          95
+#define TK_BIGINT                          95
-#define TK_NK_FLOAT                        96
+#define TK_FLOAT                           96
-#define TK_NK_BOOL                         97
+#define TK_DOUBLE                          97
-#define TK_NK_VARIABLE                     98
+#define TK_BINARY                          98
-#define TK_BETWEEN                         99
+#define TK_TIMESTAMP                       99
-#define TK_IS                             100
+#define TK_NCHAR                          100
-#define TK_NULL                           101
+#define TK_UNSIGNED                       101
-#define TK_NK_LT                          102
+#define TK_JSON                           102
-#define TK_NK_GT                          103
+#define TK_VARCHAR                        103
-#define TK_NK_LE                          104
+#define TK_MEDIUMBLOB                     104
-#define TK_NK_GE                          105
+#define TK_BLOB                           105
-#define TK_NK_NE                          106
+#define TK_VARBINARY                      106
-#define TK_NK_EQ                          107
+#define TK_DECIMAL                        107
-#define TK_LIKE                           108
+#define TK_SMA                            108
-#define TK_MATCH                          109
+#define TK_ROLLUP                         109
-#define TK_NMATCH                         110
+#define TK_INDEX                          110
-#define TK_IN                             111
+#define TK_FULLTEXT                       111
-#define TK_FROM                           112
+#define TK_FUNCTION                       112
-#define TK_JOIN                           113
+#define TK_INTERVAL                       113
-#define TK_INNER                          114
+#define TK_TOPIC                          114
-#define TK_SELECT                         115
+#define TK_AS                             115
-#define TK_DISTINCT                       116
+#define TK_MNODES                         116
-#define TK_WHERE                          117
+#define TK_NK_BOOL                        117
-#define TK_PARTITION                      118
+#define TK_NK_VARIABLE                    118
-#define TK_BY                             119
+#define TK_BETWEEN                        119
-#define TK_SESSION                        120
+#define TK_IS                             120
-#define TK_STATE_WINDOW                   121
+#define TK_NULL                           121
-#define TK_SLIDING                        122
+#define TK_NK_LT                          122
-#define TK_FILL                           123
+#define TK_NK_GT                          123
-#define TK_VALUE                          124
+#define TK_NK_LE                          124
-#define TK_NONE                           125
+#define TK_NK_GE                          125
-#define TK_PREV                           126
+#define TK_NK_NE                          126
-#define TK_LINEAR                         127
+#define TK_LIKE                           127
-#define TK_NEXT                           128
+#define TK_MATCH                          128
-#define TK_GROUP                          129
+#define TK_NMATCH                         129
-#define TK_HAVING                         130
+#define TK_IN                             130
-#define TK_ORDER                          131
+#define TK_FROM                           131
-#define TK_SLIMIT                         132
+#define TK_JOIN                           132
-#define TK_SOFFSET                        133
+#define TK_INNER                          133
-#define TK_LIMIT                          134
+#define TK_SELECT                         134
-#define TK_OFFSET                         135
+#define TK_DISTINCT                       135
-#define TK_ASC                            136
+#define TK_WHERE                          136
-#define TK_DESC                           137
+#define TK_PARTITION                      137
-#define TK_NULLS                          138
+#define TK_BY                             138
-#define TK_FIRST                          139
+#define TK_SESSION                        139
-#define TK_LAST                           140
+#define TK_STATE_WINDOW                   140
+#define TK_SLIDING                        141
+#define TK_FILL                           142
+#define TK_VALUE                          143
+#define TK_NONE                           144
+#define TK_PREV                           145
+#define TK_LINEAR                         146
+#define TK_NEXT                           147
+#define TK_GROUP                          148
+#define TK_HAVING                         149
+#define TK_ORDER                          150
+#define TK_SLIMIT                         151
+#define TK_SOFFSET                        152
+#define TK_LIMIT                          153
+#define TK_OFFSET                         154
+#define TK_ASC                            155
+#define TK_DESC                           156
+#define TK_NULLS                          157
+#define TK_FIRST                          158
+#define TK_LAST                           159
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/common/ttypes.h

@@ -27,7 +27,7 @@ extern "C" {
 typedef int32_t  VarDataOffsetT;
 typedef uint32_t TDRowLenT;
 typedef uint8_t  TDRowValT;
-typedef uint16_t col_id_t;
+typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 
 #pragma pack(push, 1)

include/dnode/bnode/bnode.h

@@ -21,24 +21,19 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SBnode       SBnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
 } SBnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SBnodeOpt;
 
 /* ------------------------ SBnode ------------------------ */
@@ -76,13 +71,6 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad);
  */
 int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs);
 
-/**
- * @brief Drop a bnode.
- *
- * @param path Path of the bnode.
- */
-void bndDestroy(const char *path);
-
 #ifdef __cplusplus
 }
 #endif

include/dnode/mgmt/dnode.h

@@ -33,8 +33,7 @@ typedef struct SDnode SDnode;
 int32_t dndInit();
 
 /**
- * @brief clear the environment
+ * @brief Clear the environment
- *
  */
 void dndCleanup();
 
@@ -42,22 +41,24 @@ void dndCleanup();
 typedef struct {
   int32_t   numOfSupportVnodes;
   uint16_t  serverPort;
-  char      dataDir[TSDB_FILENAME_LEN];
+  char      dataDir[PATH_MAX];
   char      localEp[TSDB_EP_LEN];
   char      localFqdn[TSDB_FQDN_LEN];
   char      firstEp[TSDB_EP_LEN];
   char      secondEp[TSDB_EP_LEN];
   SDiskCfg *pDisks;
   int32_t   numOfDisks;
-} SDnodeObjCfg;
+} SDnodeOpt;
+
+typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent;
 
 /**
  * @brief Initialize and start the dnode.
  *
- * @param pCfg Config of the dnode.
+ * @param pOption Option of the dnode.
  * @return SDnode* The dnode object.
  */
-SDnode *dndCreate(SDnodeObjCfg *pCfg);
+SDnode *dndCreate(const SDnodeOpt *pOption);
 
 /**
  * @brief Stop and cleanup the dnode.
@@ -66,6 +67,21 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg);
  */
 void dndClose(SDnode *pDnode);
 
+/**
+ * @brief Run dnode until specific event is receive.
+ *
+ * @param pDnode The dnode object to run.
+ */
+int32_t dndRun(SDnode *pDnode);
+
+/**
+ * @brief Handle event in the dnode.
+ *
+ * @param pDnode The dnode object to close.
+ * @param event The event to handle.
+ */
+void dndHandleEvent(SDnode *pDnode, EDndEvent event);
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/mnode/mnode.h

@@ -23,14 +23,8 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode    SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SMnode       SMnode;
-typedef struct SMnodeMsg SMnodeMsg;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
 
 typedef struct {
   int32_t        dnodeId;
@@ -38,12 +32,12 @@ typedef struct {
   int8_t         replica;
   int8_t         selfIndex;
   SReplica       replicas[TSDB_MAX_REPLICA];
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  PutReqToMWriteQFp putReqToMWriteQFp;
+  PutToQueueFp   putToWriteQFp;
-  PutReqToMReadQFp  putReqToMReadQFp;
+  PutToQueueFp   putToReadQFp;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SMnodeOpt;
 
 /* ------------------------ SMnode ------------------------ */
@@ -73,11 +67,11 @@ void mndClose(SMnode *pMnode);
 int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption);
 
 /**
- * @brief Drop a mnode.
+ * @brief Start mnode
  *
- * @param path Path of the mnode.
+ * @param pMnode The mnode object.
  */
-void mndDestroy(const char *path);
+int32_t mndStart(SMnode *pMnode);
 
 /**
  * @brief Get mnode monitor info.
@@ -104,37 +98,13 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
  */
 int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
 
-/**
- * @brief Initialize mnode msg.
- *
- * @param pMnode The mnode object.
- * @param pMsg The request rpc msg.
- * @return int32_t The created mnode msg.
- */
-SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- */
-void mndCleanupMsg(SMnodeMsg *pMsg);
-
-/**
- * @brief Cleanup mnode msg.
- *
- * @param pMsg The request msg.
- * @param code The error code.
- */
-void mndSendRsp(SMnodeMsg *pMsg, int32_t code);
-
 /**
  * @brief Process the read, write, sync request.
  *
  * @param pMsg The request msg.
  * @return int32_t 0 for success, -1 for failure.
  */
-void mndProcessMsg(SMnodeMsg *pMsg);
+int32_t mndProcessMsg(SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

include/dnode/qnode/qnode.h

@@ -21,11 +21,8 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SQnode       SQnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
   int64_t numOfStartTask;
@@ -39,13 +36,12 @@ typedef struct {
 } SQnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SQnodeOpt;
 
 /* ------------------------ SQnode ------------------------ */

include/dnode/snode/snode.h

@@ -25,24 +25,20 @@ extern "C" {
 #endif
 
 /* ------------------------ TYPES EXPOSED ------------------------ */
-typedef struct SDnode SDnode;
+typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SSnode       SSnode;
-typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg);
-typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
-typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg);
 
 typedef struct {
-  int64_t numOfErrors;
+  int32_t reserved;
 } SSnodeLoad;
 
 typedef struct {
-  int32_t           sver;
   int32_t        dnodeId;
   int64_t        clusterId;
-  SDnode           *pDnode;
+  SMgmtWrapper  *pWrapper;
-  SendReqToDnodeFp  sendReqToDnodeFp;
+  SendReqFp      sendReqFp;
-  SendReqToMnodeFp  sendReqToMnodeFp;
+  SendMnodeReqFp sendMnodeReqFp;
-  SendRedirectRspFp sendRedirectRspFp;
+  SendRspFp      sendRspFp;
 } SSnodeOpt;
 
 /* ------------------------ SSnode ------------------------ */
@@ -77,20 +73,9 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
  * @param pSnode The snode object.
  * @param pMsg The request message
  * @param pRsp The response message
- * @return int32_t 0 for success, -1 for failure
  */
-// int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
+void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
+void sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
-int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
-int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg);
-
-/**
- * @brief Drop a snode.
- *
- * @param path Path of the snode.
- */
-void sndDestroy(const char *path);
 
 #ifdef __cplusplus
 }

include/libs/nodes/cmdnodes.h

@@ -127,6 +127,18 @@ typedef struct SDropSuperTableStmt {
   bool ignoreNotExists;
 } SDropSuperTableStmt;
 
+typedef struct SAlterTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  int8_t alterType;
+  char colName[TSDB_COL_NAME_LEN];
+  char newColName[TSDB_COL_NAME_LEN];
+  STableOptions* pOptions;
+  SDataType dataType;
+  SValueNode* pVal;
+} SAlterTableStmt;
+
 typedef struct SCreateUserStmt {
   ENodeType type;
   char useName[TSDB_USER_LEN];
@@ -158,6 +170,13 @@ typedef struct SDropDnodeStmt {
   int32_t port;
 } SDropDnodeStmt;
 
+typedef struct SAlterDnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+  char config[TSDB_DNODE_CONFIG_LEN];
+  char value[TSDB_DNODE_VALUE_LEN];
+} SAlterDnodeStmt;
+
 typedef struct SShowStmt {
   ENodeType type;
   char dbName[TSDB_DB_NAME_LEN];
@@ -215,6 +234,12 @@ typedef struct SDropTopicStmt {
   bool ignoreNotExists;
 } SDropTopicStmt;
 
+typedef struct SAlterLocalStmt {
+  ENodeType type;
+  char config[TSDB_DNODE_CONFIG_LEN];
+  char value[TSDB_DNODE_VALUE_LEN];
+} SAlterLocalStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -85,6 +85,7 @@ typedef enum ENodeType {
   QUERY_NODE_DROP_TABLE_CLAUSE,
   QUERY_NODE_DROP_TABLE_STMT,
   QUERY_NODE_DROP_SUPER_TABLE_STMT,
+  QUERY_NODE_ALTER_TABLE_STMT,
   QUERY_NODE_SHOW_TABLES_STMT, // temp
   QUERY_NODE_SHOW_STABLES_STMT,
   QUERY_NODE_CREATE_USER_STMT,
@@ -94,6 +95,7 @@ typedef enum ENodeType {
   QUERY_NODE_USE_DATABASE_STMT,
   QUERY_NODE_CREATE_DNODE_STMT,
   QUERY_NODE_DROP_DNODE_STMT,
+  QUERY_NODE_ALTER_DNODE_STMT,
   QUERY_NODE_SHOW_DNODES_STMT,
   QUERY_NODE_SHOW_VGROUPS_STMT,
   QUERY_NODE_SHOW_MNODES_STMT,
@@ -104,6 +106,7 @@ typedef enum ENodeType {
   QUERY_NODE_DROP_QNODE_STMT,
   QUERY_NODE_CREATE_TOPIC_STMT,
   QUERY_NODE_DROP_TOPIC_STMT,
+  QUERY_NODE_ALTER_LOCAL_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -163,6 +166,7 @@ SNodeList* nodesMakeList();
 int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
+int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
 SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);

include/libs/nodes/plannodes.h

@@ -26,7 +26,6 @@ extern "C" {
 
 typedef struct SLogicNode {
   ENodeType type;
-  int32_t id;
   SNodeList* pTargets; // SColumnNode
   SNode* pConditions;
   SNodeList* pChildren;
@@ -37,6 +36,7 @@ typedef enum EScanType {
   SCAN_TYPE_TAG,
   SCAN_TYPE_TABLE,
   SCAN_TYPE_STABLE,
+  SCAN_TYPE_TOPIC,
   SCAN_TYPE_STREAM
 } EScanType;
 
@@ -167,6 +167,7 @@ typedef struct SScanPhysiNode {
 
 typedef SScanPhysiNode SSystemTableScanPhysiNode;
 typedef SScanPhysiNode STagScanPhysiNode;
+typedef SScanPhysiNode SStreamScanPhysiNode;
 
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;

include/libs/nodes/querynodes.h

@@ -23,6 +23,10 @@ extern "C" {
 #include "nodes.h"
 #include "tmsg.h"
 
+#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
+#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
+
 typedef struct SRawExprNode {
   ENodeType nodeType;
   char* p;

include/libs/parser/parser.h

@@ -26,6 +26,7 @@ typedef struct SParseContext {
   uint64_t         requestId;
   int32_t          acctId;
   const char      *db;
+  bool             topicQuery;
   void            *pTransporter;
   SEpSet           mgmtEpSet;
   const char      *pSql;           // sql string

include/libs/planner/planner.h

@@ -26,6 +26,7 @@ typedef struct SPlanContext {
   uint64_t queryId;
   int32_t acctId;
   SNode* pAstRoot;
+  bool topicQuery;
   bool streamQuery;
 } SPlanContext;
 

include/libs/qworker/qworker.h

@@ -49,10 +49,10 @@ typedef struct {
 } SQWorkerStat;
 
 typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
-typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *);
+typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *);
 
 int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj,
-                    putReqToQueryQFp fp1, sendReqToDnodeFp fp2);
+                    putReqToQueryQFp fp1, sendReqFp fp2);
 
 int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 

include/libs/transport/trpc.h

@@ -38,16 +38,24 @@ typedef struct SRpcConnInfo {
 
 typedef struct SRpcMsg {
   tmsg_t  msgType;
-  tmsg_t  expectMsgType;
   void *  pCont;
   int     contLen;
   int32_t code;
   void *  handle;         // rpc handle returned to app
   void *  ahandle;        // app handle set by client
-  int     noResp;   // has response or not(default 0 indicate resp);
+  int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
+  int     persistHandle;  // persist handle or not
 
 } SRpcMsg;
 
+typedef struct {
+  char    user[TSDB_USER_LEN];
+  SRpcMsg rpcMsg;
+  int32_t rspLen;
+  void   *pRsp;
+  void   *pNode;
+} SNodeMsg;
+
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
   char *   label;         // for debug purpose
@@ -69,18 +77,19 @@ typedef struct SRpcInit {
   // call back to retrieve the client auth info, for server app only
   int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
 
-  // call back to keep conn or not
-  bool (*pfp)(void *parent, tmsg_t msgType);
-
-  // to support Send messages multiple times on a link
-  void *(*mfp)(void *parent, tmsg_t msgType);
-
-  // call back  to handle except when query/fetch in progress
-  bool (*efp)(void *parent, tmsg_t msgType);
-
   void *parent;
 } SRpcInit;
 
+typedef struct {
+  void *  val;
+  int32_t len;
+  void (*free)(void *arg);
+} SRpcCtxVal;
+
+typedef struct {
+  SHashObj *args;
+} SRpcCtx;
+
 int32_t rpcInit();
 void    rpcCleanup();
 void *  rpcOpen(const SRpcInit *pRpc);
@@ -89,16 +98,17 @@ void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
 void *  rpcReallocCont(void *ptr, int contLen);
 void    rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
+void    rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
+
 void rpcSendResponse(const SRpcMsg *pMsg);
 void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
 int  rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
 int  rpcReportProgress(void *pConn, char *pCont, int contLen);
 void rpcCancelRequest(int64_t rid);
-
+void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
 // just release client conn to rpc instance, no close sock
-void rpcReleaseHandle(void *handle, int8_t type);
+void rpcReleaseHandle(void *handle, int8_t type);  //
-
 void rpcRefHandle(void *handle, int8_t type);
 void rpcUnrefHandle(void *handle, int8_t type);
 

include/libs/wal/wal.h

@@ -127,7 +127,7 @@ typedef struct SWal {
   int64_t lastRollSeq;
   // ctl
   int64_t         refId;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
   // path
   char path[WAL_PATH_LEN];
   // reusable write head

include/os/os.h

@@ -22,7 +22,6 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
-#include <pthread.h>
 #include <semaphore.h>
 
 #include <regex.h>
@@ -83,6 +82,7 @@ extern "C" {
 #include "osMath.h"
 #include "osMemory.h"
 #include "osRand.h"
+#include "osThread.h"
 #include "osSemaphore.h"
 #include "osSignal.h"
 #include "osSleep.h"
@@ -90,7 +90,6 @@ extern "C" {
 #include "osString.h"
 #include "osSysinfo.h"
 #include "osSystem.h"
-#include "osThread.h"
 #include "osTime.h"
 #include "osTimer.h"
 #include "osTimezone.h"

include/os/osEnv.h

@@ -45,6 +45,7 @@ extern SDiskSpace tsTempSpace;
 
 void osInit();
 void osUpdate();
+void osCleanup();
 bool osLogSpaceAvailable();
 void osSetTimezone(const char *timezone);
 

include/os/osSemaphore.h

@@ -20,7 +20,6 @@
 extern "C" {
 #endif
 
-#include <pthread.h>
 #include <semaphore.h>
 
 #if defined (_TD_DARWIN_64)
@@ -38,25 +37,25 @@ extern "C" {
 #endif
 
 #if defined (_TD_DARWIN_64)
-//  #define pthread_rwlock_t pthread_mutex_t
+//  #define TdThreadRwlock TdThreadMutex
-//  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+//  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-//  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+//  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
-// #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+// #define taosThreadRwlockWrlock(lock) taosThreadMutexLock(lock)
-//  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+//  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
-//  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+//  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define pthread_spinlock_t pthread_mutex_t
+  #define TdThreadSpinlock TdThreadMutex
-  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
-  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
-  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
 #endif
 
-bool    taosCheckPthreadValid(pthread_t thread);
+bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
-int64_t taosGetPthreadId(pthread_t thread);
+int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(pthread_t* thread);
+void    taosResetPthread(TdThread* thread);
-bool    taosComparePthread(pthread_t first, pthread_t second);
+bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
 int32_t taosGetAppName(char* name, int32_t* len);
 

include/os/osString.h

@@ -35,7 +35,7 @@ typedef int32_t TdUcs4;
     #define wctomb WCTOMB_FUNC_TAOS_FORBID
     #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
     #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
-    #define wchar_t WCHAR_T_FUNC_TAOS_FORBID
+    #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
 #endif
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

include/os/osThread.h

@@ -22,6 +22,94 @@
 extern "C" {
 #endif
 
+typedef pthread_t TdThread;
+typedef pthread_spinlock_t TdThreadSpinlock;
+typedef pthread_mutex_t TdThreadMutex;
+typedef pthread_mutexattr_t TdThreadMutexAttr;
+typedef pthread_rwlock_t TdThreadRwlock;
+typedef pthread_attr_t TdThreadAttr;
+typedef pthread_once_t TdThreadOnce;
+typedef pthread_rwlockattr_t TdThreadRwlockAttr;
+typedef pthread_cond_t TdThreadCond;
+typedef pthread_condattr_t TdThreadCondAttr;
+
+#define taosThreadCleanupPush pthread_cleanup_push
+#define taosThreadCleanupPop pthread_cleanup_pop
+
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID
+    #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID
+    #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID
+    #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID
+    #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID
+    #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID
+    #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID
+    #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID
+    #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
+    #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
+    #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID
+    #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
+    #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
+    #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
+    #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID
+    #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID
+    #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID
+    #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
+    #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
+    #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
+    #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
+    #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID
+#endif
+
+int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared);
+int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr);
+int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock);
+int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
+int32_t taosThreadSpinLock(TdThreadSpinlock *lock);
+int32_t taosThreadMutexLock(TdThreadMutex *mutex);
+int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock);
+int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock);
+int32_t taosThreadMutexUnlock(TdThreadMutex *mutex);
+int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock);
+int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock);
+void taosThreadTestCancel(void);
+int32_t taosThreadAttrInit(TdThreadAttr *attr);
+int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg);
+int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void));
+int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState);
+int32_t taosThreadAttrDestroy(TdThreadAttr *attr);
+int32_t taosThreadJoin(TdThread thread, void **pValue);
+int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr);
+int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock);
+int32_t taosThreadCondSignal(TdThreadCond *cond);
+int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr);
+int32_t taosThreadCondBroadcast(TdThreadCond *cond);
+int32_t taosThreadCondDestroy(TdThreadCond *cond);
+int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex);
+TdThread taosThreadSelf(void);
+int32_t taosThreadEqual(TdThread t1, TdThread t2);
+int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset);
+int32_t taosThreadCancel(TdThread thread);
+int32_t taosThreadKill(TdThread thread, int sig);
 #ifdef __cplusplus
 }
 #endif

include/os/osTimezone.h

@@ -20,6 +20,11 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define tzset TZSET_FUNC_TAOS_FORBID
+#endif
+
 void taosGetSystemTimezone(char *outTimezone);
 void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
 

include/util/taoserror.h

@@ -75,6 +75,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x010B)
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
 #define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
+#define TSDB_CODE_OUT_OF_SHM_MEM                TAOS_DEF_ERROR_CODE(0, 0x010E)
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
 #define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0112)
@@ -277,34 +278,14 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)
 #define TSDB_CODE_DND_OFFLINE                   TAOS_DEF_ERROR_CODE(0, 0x0401)
 #define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0402)
-#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0410)
+#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x0403)
-#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0411)
+#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x0404)
-#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0420)
+#define TSDB_CODE_NODE_PARSE_FILE_ERROR         TAOS_DEF_ERROR_CODE(0, 0x0405)
-#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0421)
+#define TSDB_CODE_NODE_INVALID_OPTION           TAOS_DEF_ERROR_CODE(0, 0x0406)
-#define TSDB_CODE_DND_MNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0422)
+#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0410)
-#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0423)
+#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0411)
-#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0424)
+#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0412)
-#define TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0430)
+#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0413)
-#define TSDB_CODE_DND_QNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0431)
-#define TSDB_CODE_DND_QNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0432)
-#define TSDB_CODE_DND_QNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0433)
-#define TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0434)
-#define TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0440)
-#define TSDB_CODE_DND_SNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0441)
-#define TSDB_CODE_DND_SNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0442)
-#define TSDB_CODE_DND_SNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0443)
-#define TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0444)
-#define TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0450)
-#define TSDB_CODE_DND_BNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0451)
-#define TSDB_CODE_DND_BNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0452)
-#define TSDB_CODE_DND_BNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0453)
-#define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0454)
-#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0460)
-#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0461)
-#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0462)
-#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR     TAOS_DEF_ERROR_CODE(0, 0x0463)
-#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR    TAOS_DEF_ERROR_CODE(0, 0x0464)
-#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0465)
 
 // vnode
 #define TSDB_CODE_VND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0500)
@@ -482,6 +463,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_PASSWD_EMPTY              TAOS_DEF_ERROR_CODE(0, 0x2611)
 #define TSDB_CODE_PAR_INVALID_PORT              TAOS_DEF_ERROR_CODE(0, 0x2612)
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
+#define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 
 #ifdef __cplusplus
 }

include/util/tidpool.h

@@ -27,7 +27,7 @@ typedef struct {
   int32_t         numOfFree;
   int32_t         freeSlot;
   bool           *freeList;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
 } id_pool_t;
 
 void   *taosInitIdPool(int32_t maxId);

include/util/tjson.h

@@ -22,6 +22,14 @@
 extern "C" {
 #endif
 
+#define tjsonGetNumberValue(pJson, pName, val) \
+  ({ \
+    uint64_t _tmp = 0; \
+    int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
+    val = _tmp; \
+    _code; \
+  })
+
 typedef void SJson;
 
 SJson* tjsonCreateObject();
@@ -44,6 +52,7 @@ int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);
 int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal);
 int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal);
 int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal);
+int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal);
 int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal);
 int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal);
 int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal);
@@ -60,6 +69,7 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void*
 typedef int32_t (*FToObject)(const SJson* pJson, void* pObj);
 
 int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj);
+int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize);
 int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize);
 
 char* tjsonToString(const SJson* pJson);

include/util/tprocess.h

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_PROCESS_H_
+#define _TD_UTIL_PROCESS_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SProcQueue SProcQueue;
+typedef struct SProcObj   SProcObj;
+typedef void *(*ProcMallocFp)(int32_t contLen);
+typedef void *(*ProcFreeFp)(void *pCont);
+typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+
+typedef struct {
+  int32_t       childQueueSize;
+  ProcConsumeFp childConsumeFp;
+  ProcMallocFp  childMallocHeadFp;
+  ProcFreeFp    childFreeHeadFp;
+  ProcMallocFp  childMallocBodyFp;
+  ProcFreeFp    childFreeBodyFp;
+  int32_t       parentQueueSize;
+  ProcConsumeFp parentConsumeFp;
+  ProcMallocFp  parentdMallocHeadFp;
+  ProcFreeFp    parentFreeHeadFp;
+  ProcMallocFp  parentMallocBodyFp;
+  ProcFreeFp    parentFreeBodyFp;
+  bool          testFlag;
+  void         *pParent;
+  const char   *name;
+} SProcCfg;
+
+SProcObj *taosProcInit(const SProcCfg *pCfg);
+void      taosProcCleanup(SProcObj *pProc);
+int32_t   taosProcRun(SProcObj *pProc);
+void      taosProcStop(SProcObj *pProc);
+bool      taosProcIsChild(SProcObj *pProc);
+
+int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_PROCESS_H_*/

include/util/tskiplist.h

@@ -57,7 +57,7 @@ typedef struct SSkipListNode {
  * @date   2017/11/12
  * the simple version of skip list.
  *
- * for multi-thread safe purpose, we employ pthread_rwlock_t to guarantee to generate
+ * for multi-thread safe purpose, we employ TdThreadRwlock to guarantee to generate
  * deterministic result. Later, we will remove the lock in SkipList to further enhance the performance.
  * In this case, one should use the concurrent skip list (by using michael-scott algorithm) instead of
  * this simple version in a multi-thread environment, to achieve higher performance of read/write operations.
@@ -106,7 +106,7 @@ typedef struct SSkipList {
   uint32_t          seed;
   __compar_fn_t     comparFn;
   __sl_key_fn_t     keyFn;
-  pthread_rwlock_t *lock;
+  TdThreadRwlock *lock;
   uint16_t          len;
   uint8_t           maxLevel;
   uint8_t           flags;

include/util/tthread.h

@@ -22,9 +22,11 @@
 extern "C" {
 #endif
 
-pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param);
+TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param);
-bool       taosDestoryThread(pthread_t* pthread);
+bool       taosDestoryThread(TdThread* pthread);
-bool       taosThreadRunning(pthread_t* pthread);
+bool       taosThreadRunning(TdThread* pthread);
+
+typedef void *(*ThreadFp)(void *param);
 
 #ifdef __cplusplus
 }

include/util/tworker.h

@@ -27,7 +27,7 @@ typedef struct SWWorkerPool SWWorkerPool;
 
 typedef struct SQWorker {
   int32_t       id;      // worker ID
-  pthread_t     thread;  // thread
+  TdThread     thread;  // thread
   SQWorkerPool *pool;
 } SQWorker, SFWorker;
 
@@ -38,12 +38,12 @@ typedef struct SQWorkerPool {
   STaosQset      *qset;
   const char     *name;
   SQWorker       *workers;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
 } SQWorkerPool, SFWorkerPool;
 
 typedef struct SWWorker {
   int32_t       id;      // worker id
-  pthread_t     thread;  // thread
+  TdThread     thread;  // thread
   STaosQall    *qall;
   STaosQset    *qset;  // queue set
   SWWorkerPool *pool;
@@ -54,7 +54,7 @@ typedef struct SWWorkerPool {
   int32_t         nextId;  // from 0 to max-1, cyclic
   const char     *name;
   SWWorker       *workers;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
 } SWWorkerPool;
 
 int32_t     tQWorkerInit(SQWorkerPool *pool);

source/client/inc/clientInt.h

@@ -77,8 +77,8 @@ typedef struct {
   int8_t inited;
   // ctl
   int8_t          threadStop;
-  pthread_t       thread;
+  TdThread       thread;
-  pthread_mutex_t lock;       // used when app init and cleanup
+  TdThreadMutex lock;       // used when app init and cleanup
   SArray*         appHbMgrs;  // SArray<SAppHbMgr*> one for each cluster
   FHbReqHandle    reqHandle[HEARTBEAT_TYPE_MAX];
   FHbRspHandle    rspHandle[HEARTBEAT_TYPE_MAX];
@@ -125,7 +125,7 @@ typedef struct SAppInfo {
   int32_t         pid;
   int32_t         numOfThreads;
   SHashObj*       pInstMap;
-  pthread_mutex_t mutex;
+  TdThreadMutex mutex;
 } SAppInfo;
 
 typedef struct STscObj {
@@ -137,7 +137,7 @@ typedef struct STscObj {
   uint32_t        connId;
   int32_t         connType;
   uint64_t        id;         // ref ID returned by taosAddRef
-  pthread_mutex_t mutex;      // used to protect the operation on db
+  TdThreadMutex mutex;      // used to protect the operation on db
   int32_t         numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo*   pAppInfo;
 } STscObj;
@@ -179,6 +179,7 @@ typedef struct SRequestObj {
   uint64_t             requestId;
   int32_t              type;  // request type
   STscObj*             pTscObj;
+  char*                pDb;
   char*                sqlstr;  // sql string
   int32_t              sqlLen;
   int64_t              self;
@@ -229,7 +230,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery);
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
 
 // --- heartbeat

source/client/src/clientEnv.c

@@ -33,7 +33,7 @@ SAppInfo appInfo;
 int32_t  clientReqRefPool = -1;
 int32_t  clientConnRefPool = -1;
 
-static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
+static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
 volatile int32_t      tscInitRes = 0;
 
 static void registerRequest(SRequestObj *pRequest) {
@@ -90,7 +90,6 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   rpcInit.label = "TSC";
   rpcInit.numOfThreads = numOfThread;
   rpcInit.cfp = processMsgFromServer;
-  rpcInit.pfp = persistConnForSpecificMsg;
   rpcInit.sessions = tsMaxConnections;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
@@ -115,7 +114,7 @@ void destroyTscObj(void *pObj) {
   hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
   atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
   tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
-  pthread_mutex_destroy(&pTscObj->mutex);
+  taosThreadMutexDestroy(&pTscObj->mutex);
   tfree(pTscObj);
 }
 
@@ -134,7 +133,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
     tstrncpy(pObj->db, db, tListLen(pObj->db));
   }
 
-  pthread_mutex_init(&pObj->mutex, NULL);
+  taosThreadMutexInit(&pObj->mutex, NULL);
   pObj->id = taosAddRef(clientConnRefPool, pObj);
 
   tscDebug("connObj created, 0x%" PRIx64, pObj->id);
@@ -150,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
     return NULL;
   }
 
+  pRequest->pDb = getDbOfConnection(pObj);
   pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampMs();
 
@@ -180,6 +180,7 @@ static void doDestroyRequest(void *p) {
   tfree(pRequest->msgBuf);
   tfree(pRequest->sqlstr);
   tfree(pRequest->pInfo);
+  tfree(pRequest->pDb);
 
   doFreeReqResultInfo(&pRequest->body.resInfo);
   qDestroyQueryPlan(pRequest->body.pDag);
@@ -241,7 +242,7 @@ void taos_init_imp(void) {
 
   // transDestroyBuffer(&conn->readBuf);
   taosGetAppName(appInfo.appName, NULL);
-  pthread_mutex_init(&appInfo.mutex, NULL);
+  taosThreadMutexInit(&appInfo.mutex, NULL);
 
   appInfo.pid = taosGetPId();
   appInfo.startTime = taosGetTimestampMs();
@@ -250,7 +251,7 @@ void taos_init_imp(void) {
 }
 
 int taos_init() {
-  pthread_once(&tscinit, taos_init_imp);
+  taosThreadOnce(&tscinit, taos_init_imp);
   return tscInitRes;
 }
 
@@ -506,9 +507,9 @@ static setConfRet taos_set_config_imp(const char *config){
 }
 
 setConfRet taos_set_config(const char *config){
-  pthread_mutex_lock(&setConfMutex);
+  taosThreadMutexLock(&setConfMutex);
   setConfRet ret = taos_set_config_imp(config);
-  pthread_mutex_unlock(&setConfMutex);
+  taosThreadMutexUnlock(&setConfMutex);
   return ret;
 }
 #endif

source/client/src/clientHb.c

@@ -372,7 +372,7 @@ static void *hbThreadFunc(void *param) {
       break;
     }
 
-    pthread_mutex_lock(&clientHbMgr.lock);
+    taosThreadMutexLock(&clientHbMgr.lock);
 
     int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
     for (int i = 0; i < sz; i++) {
@@ -423,7 +423,7 @@ static void *hbThreadFunc(void *param) {
       atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
     }
 
-    pthread_mutex_unlock(&clientHbMgr.lock);
+    taosThreadMutexUnlock(&clientHbMgr.lock);
 
     taosMsleep(HEARTBEAT_INTERVAL);
   }
@@ -431,15 +431,15 @@ static void *hbThreadFunc(void *param) {
 }
 
 static int32_t hbCreateThread() {
-  pthread_attr_t thAttr;
+  TdThreadAttr thAttr;
-  pthread_attr_init(&thAttr);
+  taosThreadAttrInit(&thAttr);
-  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+  taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-//  if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
+//  if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
 //    terrno = TAOS_SYSTEM_ERROR(errno);
 //    return -1;
 //  }
-//  pthread_attr_destroy(&thAttr);
+//  taosThreadAttrDestroy(&thAttr);
   return 0;
 }
 
@@ -492,15 +492,15 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
     return NULL;
   }
 
-  pthread_mutex_lock(&clientHbMgr.lock);
+  taosThreadMutexLock(&clientHbMgr.lock);
   taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
-  pthread_mutex_unlock(&clientHbMgr.lock);
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 
   return pAppHbMgr;
 }
 
 void appHbMgrCleanup(void) {
-  pthread_mutex_lock(&clientHbMgr.lock);
+  taosThreadMutexLock(&clientHbMgr.lock);
 
   int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
   for (int i = 0; i < sz; i++) {
@@ -511,7 +511,7 @@ void appHbMgrCleanup(void) {
     pTarget->connInfo = NULL;
   }
 
-  pthread_mutex_unlock(&clientHbMgr.lock);
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 }
 
 int hbMgrInit() {
@@ -520,7 +520,7 @@ int hbMgrInit() {
   if (old == 1) return 0;
 
   clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *));
-  pthread_mutex_init(&clientHbMgr.lock, NULL);
+  taosThreadMutexInit(&clientHbMgr.lock, NULL);
 
   // init handle funcs
   hbMgrInitHandle();
@@ -539,10 +539,10 @@ void hbMgrCleanUp() {
   int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);
   if (old == 0) return;
 
-  pthread_mutex_lock(&clientHbMgr.lock);
+  taosThreadMutexLock(&clientHbMgr.lock);
   appHbMgrCleanup();
   taosArrayDestroy(clientHbMgr.appHbMgrs);
-  pthread_mutex_unlock(&clientHbMgr.lock);
+  taosThreadMutexUnlock(&clientHbMgr.lock);
 
   clientHbMgr.appHbMgrs = NULL;
 #endif

source/client/src/clientImpl.c

@@ -95,7 +95,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
   char*          key = getClusterKey(user, secretEncrypt, ip, port);
   SAppInstInfo** pInst = NULL;
 
-  pthread_mutex_lock(&appInfo.mutex);
+  taosThreadMutexLock(&appInfo.mutex);
 
   pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
   SAppInstInfo* p = NULL;
@@ -109,7 +109,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
     pInst = &p;
   }
 
-  pthread_mutex_unlock(&appInfo.mutex);
+  taosThreadMutexUnlock(&appInfo.mutex);
 
   tfree(key);
   return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst);
@@ -137,13 +137,14 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
   STscObj* pTscObj = pRequest->pTscObj;
 
   SParseContext cxt = {
       .requestId = pRequest->requestId,
       .acctId = pTscObj->acctId,
-      .db = getDbOfConnection(pTscObj),
+      .db = pRequest->pDb,
+      .topicQuery = topicQuery,
       .pSql = pRequest->sqlstr,
       .sqlLen = pRequest->sqlLen,
       .pMsg = pRequest->msgBuf,
@@ -154,7 +155,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
   cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
   int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
   if (code != TSDB_CODE_SUCCESS) {
-    tfree(cxt.db);
     return code;
   }
 
@@ -163,7 +163,6 @@ int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
     setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
   }
 
-  tfree(cxt.db);
   return code;
 }
 
@@ -249,7 +248,7 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
 
   terrno = TSDB_CODE_SUCCESS;
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
 
   if (pQuery->directRpc) {
     CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
@@ -592,21 +591,21 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
 
 char* getDbOfConnection(STscObj* pObj) {
   char* p = NULL;
-  pthread_mutex_lock(&pObj->mutex);
+  taosThreadMutexLock(&pObj->mutex);
   size_t len = strlen(pObj->db);
   if (len > 0) {
     p = strndup(pObj->db, tListLen(pObj->db));
   }
 
-  pthread_mutex_unlock(&pObj->mutex);
+  taosThreadMutexUnlock(&pObj->mutex);
   return p;
 }
 
 void setConnectionDB(STscObj* pTscObj, const char* db) {
   assert(db != NULL && pTscObj != NULL);
-  pthread_mutex_lock(&pTscObj->mutex);
+  taosThreadMutexLock(&pTscObj->mutex);
   tstrncpy(pTscObj->db, db, tListLen(pTscObj->db));
-  pthread_mutex_unlock(&pTscObj->mutex);
+  taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
 void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {

source/client/src/tmq.c

@@ -456,11 +456,99 @@ _return:
 
 void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; }
 
+TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
+  STscObj*     pTscObj = (STscObj*)taos;
+  SRequestObj* pRequest = NULL;
+  SQuery*      pQueryNode = NULL;
+  char*        astStr = NULL;
+  int32_t      sqlLen;
+
+  terrno = TSDB_CODE_SUCCESS;
+  if (taos == NULL || streamName == NULL || sql == NULL) {
+    tscError("invalid parameters for creating stream, connObj:%p, stream name:%s, sql:%s", taos, streamName, sql);
+    terrno = TSDB_CODE_TSC_INVALID_INPUT;
+    goto _return;
+  }
+  sqlLen = strlen(sql);
+
+  if (strlen(streamName) >= TSDB_TABLE_NAME_LEN) {
+    tscError("stream name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
+    terrno = TSDB_CODE_TSC_INVALID_INPUT;
+    goto _return;
+  }
+
+  if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) {
+    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
+    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
+    goto _return;
+  }
+
+  tscDebug("start to create stream: %s", streamName);
+
+  CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode), _return);
+
+  // todo check for invalid sql statement and return with error code
+
+  CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
+
+  /*printf("%s\n", pStr);*/
+
+  SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
+  strcpy(name.dbname, pRequest->pDb);
+  strcpy(name.tname, streamName);
+
+  SCMCreateStreamReq req = {
+      .igExists = 1,
+      .ast = astStr,
+      .sql = (char*)sql,
+  };
+  tNameExtractFullName(&name, req.name);
+  strcpy(req.outputTbName, tbName);
+
+  int   tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
+  void* buf = malloc(tlen);
+  if (buf == NULL) {
+    goto _return;
+  }
+
+  tSerializeSCMCreateStreamReq(buf, tlen, &req);
+  /*printf("formatted: %s\n", dagStr);*/
+
+  pRequest->body.requestMsg = (SDataBuf){
+      .pData = buf,
+      .len = tlen,
+      .handle = NULL,
+  };
+  pRequest->type = TDMT_MND_CREATE_STREAM;
+
+  SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
+  SEpSet        epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
+
+  int64_t transporterId = 0;
+  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
+
+  tsem_wait(&pRequest->body.rspSem);
+
+_return:
+  tfree(astStr);
+  qDestroyQuery(pQueryNode);
+  /*if (sendInfo != NULL) {*/
+  /*destroySendMsgInfo(sendInfo);*/
+  /*}*/
+
+  if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
+    pRequest->code = terrno;
+  }
+
+  return pRequest;
+}
+
 TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
   SQuery*      pQueryNode = NULL;
-  char*        pStr = NULL;
+  char*        astStr = NULL;
 
   terrno = TSDB_CODE_SUCCESS;
   if (taos == NULL || topicName == NULL || sql == NULL) {
@@ -481,39 +569,25 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
     goto _return;
   }
 
-  tscDebug("start to create topic, %s", topicName);
+  tscDebug("start to create topic: %s", topicName);
-#if 0
-  CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
 
-  pQueryNode->streamQuery = true;
+  CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
 
   // todo check for invalid sql statement and return with error code
 
-  SSchema* schema = NULL;
+  CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
-  int32_t  numOfCols = 0;
-  CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, NULL), _return);
-
-  pStr = qQueryPlanToString(pRequest->body.pDag);
-  if (pStr == NULL) {
-    goto _return;
-  }
 
   /*printf("%s\n", pStr);*/
 
-  // The topic should be related to a database that the queried table is belonged to.
+  SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
-  SName name = {0};
+  strcpy(name.dbname, pRequest->pDb);
-  char  dbName[TSDB_DB_FNAME_LEN] = {0};
+  strcpy(name.tname, topicName);
-  // tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName);
-
-  tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
-  tNameFromString(&name, topicName, T_NAME_TABLE);
 
   SCMCreateTopicReq req = {
       .igExists = 1,
-      .physicalPlan = (char*)pStr,
+      .ast = astStr,
       .sql = (char*)sql,
-      .logicalPlan = (char*)"no logic plan",
   };
   tNameExtractFullName(&name, req.name);
 
@@ -526,7 +600,11 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   tSerializeSCMCreateTopicReq(buf, tlen, &req);
   /*printf("formatted: %s\n", dagStr);*/
 
-  pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL};
+  pRequest->body.requestMsg = (SDataBuf){
+      .pData = buf,
+      .len = tlen,
+      .handle = NULL,
+  };
   pRequest->type = TDMT_MND_CREATE_TOPIC;
 
   SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
@@ -536,8 +614,9 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
   tsem_wait(&pRequest->body.rspSem);
-#endif
+
 _return:
+  tfree(astStr);
   qDestroyQuery(pQueryNode);
   /*if (sendInfo != NULL) {*/
   /*destroySendMsgInfo(sendInfo);*/

source/common/src/tdatablock.c

@@ -1072,6 +1072,10 @@ void blockDataClearup(SSDataBlock* pDataBlock) {
 }
 
 int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
+  if (0 == numOfRows) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
     char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
     if (tmp == NULL) {
@@ -1092,7 +1096,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo
 
     pColumn->nullbitmap = tmp;
     memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
-
+    assert(pColumn->info.bytes);
     tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes);
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -1137,7 +1141,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
 
   taosArrayDestroy(pBlock->pDataBlock);
   tfree(pBlock->pBlockAgg);
-  tfree(pBlock);
+  // tfree(pBlock);
   return NULL;
 }
 
@@ -1190,7 +1194,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
     }
 
     int32_t len = colDataGetLength(pColData, rows);
-    taosEncodeFixedI32(buf, len);
+    tlen += taosEncodeFixedI32(buf, len);
 
     tlen += taosEncodeBinary(buf, pColData->pData, len);
   }

source/common/src/tglobal.c

@@ -45,6 +45,7 @@ float   tsRatioOfQueryCores = 1.0f;
 int32_t tsMaxBinaryDisplayWidth = 30;
 bool    tsEnableSlaveQuery = 1;
 bool    tsPrintAuth = 0;
+int32_t tsMultiProcess = 0;
 
 // monitor
 bool     tsEnableMonitor = 1;
@@ -309,7 +310,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1;
-  if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
 
   if (cfgAddString(pCfg, "version", version, 1) != 0) return -1;
   if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1;
@@ -340,6 +340,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1;
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
@@ -403,7 +404,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
     return -1;
   }
 
-  tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
+  tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
   tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
   tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
   tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
@@ -457,6 +458,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
   tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
   tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
+  tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32;
 
   tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
   tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;

source/common/src/tmsg.c

@@ -757,6 +757,8 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
   return 0;
 }
 
+void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); }
+
 int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -2639,27 +2641,36 @@ int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) {
   int32_t tlen = 0;
 
   tlen += taosEncodeFixedI64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->indexUid);
   tlen += taosEncodeString(buf, pReq->indexName);
 
   return tlen;
 }
 void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
   buf = taosDecodeFixedI64(buf, &(pReq->ver));
+  buf = taosDecodeFixedI64(buf, &(pReq->indexUid));
   buf = taosDecodeStringTo(buf, pReq->indexName);
 
   return buf;
 }
 
 int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
+  int32_t sqlLen = 0;
+  int32_t astLen = 0;
+  if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql);
+  if (pReq->ast != NULL) astLen = (int32_t)strlen(pReq->ast);
+
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
+  if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
+  if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -2668,15 +2679,30 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
 }
 
 int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) {
+  int32_t sqlLen = 0;
+  int32_t astLen = 0;
+
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->outputTbName) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
-  if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
+  if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
-  if (tDecodeCStrAlloc(&decoder, &pReq->physicalPlan) < 0) return -1;
+  if (tDecodeI32(&decoder, &astLen) < 0) return -1;
-  if (tDecodeCStrAlloc(&decoder, &pReq->logicalPlan) < 0) return -1;
+
+  if (sqlLen > 0) {
+    pReq->sql = calloc(1, sqlLen + 1);
+    if (pReq->sql == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1;
+  }
+
+  if (astLen > 0) {
+    pReq->ast = calloc(1, astLen + 1);
+    if (pReq->ast == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
+  }
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);
@@ -2685,8 +2711,7 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
 
 void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
   tfree(pReq->sql);
-  tfree(pReq->physicalPlan);
+  tfree(pReq->ast);
-  tfree(pReq->logicalPlan);
 }
 
 int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) {
@@ -2695,6 +2720,7 @@ int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) {
   if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->level) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1;
   if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1;
   tEndEncode(pEncoder);
   return pEncoder->pos;
@@ -2706,6 +2732,7 @@ int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) {
   if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1;
   tEndDecode(pDecoder);
   return 0;

source/dnode/bnode/src/bnode.c

@@ -25,5 +25,3 @@ void bndClose(SBnode *pBnode) { free(pBnode); }
 int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; }
 
 int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; }
-
-void bndDestroy(const char *path) {}

source/dnode/mgmt/CMakeLists.txt

@@ -1,2 +1,30 @@
-add_subdirectory(daemon)
+aux_source_directory(src DNODE_SRC)
-add_subdirectory(impl)
+aux_source_directory(dnode/src DNODE_SRC)
+aux_source_directory(qnode/src DNODE_SRC)
+aux_source_directory(bnode/src DNODE_SRC)
+aux_source_directory(snode/src DNODE_SRC)
+aux_source_directory(vnode/src DNODE_SRC)
+aux_source_directory(mnode/src DNODE_SRC)
+aux_source_directory(container/src DNODE_SRC)
+
+add_library(dnode STATIC ${DNODE_SRC})
+target_link_libraries(
+    dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
+)
+target_include_directories(
+    dnode
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc"
+)
+
+add_subdirectory(main)
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/dnode/mgmt/bnode/inc/bm.h

@@ -16,17 +16,13 @@
 #ifndef _TD_DND_BNODE_H_
 #define _TD_DND_BNODE_H_
 
+#include "dnd.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitBnode(SDnode *pDnode);
+void bmGetMgmtFp(SMgmtWrapper *pWrapper);
-void    dndCleanupBnode(SDnode *pDnode);
-
-void    dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/bnode/inc/bmInt.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_BNODE_INT_H_
+#define _TD_DND_BNODE_INT_H_
+
+#include "bm.h"
+#include "bnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBnodeMgmt {
+  SBnode       *pBnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  writeWorker;
+} SBnodeMgmt;
+
+// bmInt.c
+int32_t bmOpen(SMgmtWrapper *pWrapper);
+int32_t bmDrop(SMgmtWrapper *pWrapper);
+
+// bmMsg.c
+void    bmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// bmWorker.c
+int32_t bmStartWorker(SBnodeMgmt *pMgmt);
+void    bmStopWorker(SBnodeMgmt *pMgmt);
+int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_BNODE_INT_H_*/

source/dnode/mgmt/bnode/src/bmInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t bmOpenImp(SBnodeMgmt *pMgmt) {
+  SBnodeOpt option = {0};
+  bmInitOption(pMgmt, &option);
+
+  pMgmt->pBnode = bndOpen(pMgmt->path, &option);
+  if (pMgmt->pBnode == NULL) {
+    dError("failed to open bnode since %s", terrstr());
+    return -1;
+  }
+
+  if (bmStartWorker(pMgmt) != 0) {
+    dError("failed to start bnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write bnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void bmCloseImp(SBnodeMgmt *pMgmt) {
+  if (pMgmt->pBnode != NULL) {
+    bmStopWorker(pMgmt);
+    bndClose(pMgmt->pBnode);
+    pMgmt->pBnode = NULL;
+  }
+}
+
+int32_t bmDrop(SMgmtWrapper *pWrapper) {
+  SBnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("bnode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop bnode since %s", terrstr());
+    return -1;
+  }
+
+  bmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("bnode-mgmt is dropped");
+  return 0;
+}
+
+static void bmClose(SMgmtWrapper *pWrapper) {
+  SBnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("bnode-mgmt start to cleanup");
+  bmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("bnode-mgmt is cleaned up");
+}
+
+int32_t bmOpen(SMgmtWrapper *pWrapper) {
+  dInfo("bnode-mgmt start to init");
+  SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = bmOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init bnode-mgmt since %s", terrstr());
+    bmClose(pWrapper);
+  } else {
+    dInfo("bnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void bmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = bmOpen;
+  mgmtFp.closeFp = bmClose;
+  mgmtFp.createMsgFp = bmProcessCreateReq;
+  mgmtFp.dropMsgFp = bmProcessDropReq;
+  mgmtFp.requiredFp = bmRequire;
+
+  bmInitMsgHandles(pWrapper);
+  pWrapper->name = "bnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/bnode/src/bmMsg.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateBnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return bmOpen(pWrapper);
+  }
+}
+
+int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropBnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop bnode since %s", terrstr());
+    return -1;
+  } else {
+    return bmDrop(pWrapper);
+  }
+}
+
+void bmInitMsgHandles(SMgmtWrapper *pWrapper) {}

source/dnode/mgmt/bnode/src/bmWorker.c

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bmInt.h"
+
+static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+  SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
+  dndSendRsp(pWrapper, &rpcRsp);
+
+  dTrace("msg:%p, is freed", pMsg);
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t numOfMsgs, int32_t code) {
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+    bmSendErrorRsp(pWrapper, pMsg, code);
+  }
+}
+
+static void bmProcessQueue(SBnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) {
+  SMgmtWrapper *pWrapper = pMgmt->pWrapper;
+
+  SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
+  if (pArray == NULL) {
+    bmSendErrorRsps(pWrapper, qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
+    return;
+  }
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+    dTrace("msg:%p, will be processed in bnode queue", pMsg);
+    if (taosArrayPush(pArray, &pMsg) == NULL) {
+      bmSendErrorRsp(pWrapper, pMsg, TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
+
+  bndProcessWMsgs(pMgmt->pBnode, pArray);
+
+  for (size_t i = 0; i < numOfMsgs; i++) {
+    SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
+  }
+  taosArrayDestroy(pArray);
+}
+
+int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->writeWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, bmProcessQueue) != 0) {
+    dError("failed to start bnode write worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void bmStopWorker(SBnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); }

source/dnode/mgmt/container/inc/dnd.h

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_H_
+#define _TD_DND_H_
+
+#include "os.h"
+
+#include "cJSON.h"
+#include "monitor.h"
+#include "tcache.h"
+#include "tcrc32c.h"
+#include "tdatablock.h"
+#include "tglobal.h"
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+#include "tprocess.h"
+#include "tqueue.h"
+#include "trpc.h"
+#include "tthread.h"
+#include "ttime.h"
+#include "tworker.h"
+
+#include "dnode.h"
+#include "tfs.h"
+#include "wal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define dInfo(...)  { if (dDebugFlag & DEBUG_INFO)  { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
+#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
+
+typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType;
+typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus;
+typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus;
+typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
+typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType;
+
+typedef struct SMgmtFp      SMgmtFp;
+typedef struct SMgmtWrapper SMgmtWrapper;
+typedef struct SMsgHandle   SMsgHandle;
+typedef struct SDnodeMgmt   SDnodeMgmt;
+typedef struct SVnodesMgmt  SVnodesMgmt;
+typedef struct SMnodeMgmt   SMnodeMgmt;
+typedef struct SQnodeMgmt   SQnodeMgmt;
+typedef struct SSnodeMgmt   SSnodeMgmt;
+typedef struct SBnodeMgmt   SBnodeMgmt;
+
+typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg);
+typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper);
+typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper);
+typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper);
+typedef int32_t (*CreateNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required);
+
+typedef struct {
+  EWorkerType type;
+  const char *name;
+  int32_t     minNum;
+  int32_t     maxNum;
+  void       *queueFp;
+  void       *param;
+  STaosQueue *queue;
+  union {
+    SQWorkerPool pool;
+    SWWorkerPool mpool;
+  };
+} SDnodeWorker;
+
+typedef struct SMsgHandle {
+  NodeMsgFp     msgFp;
+  SMgmtWrapper *pWrapper;
+} SMsgHandle;
+
+typedef struct SMgmtFp {
+  OpenNodeFp    openFp;
+  CloseNodeFp   closeFp;
+  StartNodeFp   startFp;
+  CreateNodeFp  createMsgFp;
+  DropNodeFp    dropMsgFp;
+  RequireNodeFp requiredFp;
+} SMgmtFp;
+
+typedef struct SMgmtWrapper {
+  const char *name;
+  char       *path;
+  int32_t     refCount;
+  SRWLatch    latch;
+  bool        deployed;
+  bool        required;
+  EProcType   procType;
+  SProcObj   *pProc;
+  void       *pMgmt;
+  SDnode     *pDnode;
+  NodeMsgFp   msgFps[TDMT_MAX];
+  SMgmtFp     fp;
+} SMgmtWrapper;
+
+typedef struct {
+  void      *serverRpc;
+  void      *clientRpc;
+  SMsgHandle msgHandles[TDMT_MAX];
+} STransMgmt;
+
+typedef struct SDnode {
+  int64_t      clusterId;
+  int32_t      dnodeId;
+  int32_t      numOfSupportVnodes;
+  int64_t      rebootTime;
+  char        *localEp;
+  char        *localFqdn;
+  char        *firstEp;
+  char        *secondEp;
+  char        *dataDir;
+  SDiskCfg    *pDisks;
+  int32_t      numOfDisks;
+  uint16_t     serverPort;
+  bool         dropped;
+  EDndStatus   status;
+  EDndEvent    event;
+  EProcType    procType;
+  SStartupReq  startup;
+  TdFilePtr    pLockFile;
+  STransMgmt   trans;
+  SMgmtWrapper wrappers[NODE_MAX];
+} SDnode;
+
+EDndStatus    dndGetStatus(SDnode *pDnode);
+void          dndSetStatus(SDnode *pDnode, EDndStatus stat);
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
+void          dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp);
+void          dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
+void          dndSendMonitorReport(SDnode *pDnode);
+
+int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
+void    dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
+
+int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
+                      int32_t maxNum, void *queueFp);
+void    dndCleanupWorker(SDnodeWorker *pWorker);
+int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg);
+
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
+
+int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
+int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_H_*/

source/dnode/mgmt/container/inc/dndInt.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_INT_H_
+#define _TD_DND_INT_H_
+
+#include "dnd.h"
+
+#include "bm.h"
+#include "dm.h"
+#include "mm.h"
+#include "qm.h"
+#include "sm.h"
+#include "vm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// dndInt.c
+int32_t     dndInit();
+void        dndCleanup();
+const char *dndStatStr(EDndStatus stat);
+void        dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
+TdFilePtr   dndCheckRunning(char *dataDir);
+void        dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
+
+// dndMsg.c
+void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet);
+
+// dndExec.c
+int32_t dndOpenNode(SMgmtWrapper *pWrapper);
+void    dndCloseNode(SMgmtWrapper *pWrapper);
+int32_t dndRun(SDnode *pDnode);
+
+// dndObj.c
+SDnode *dndCreate(const SDnodeOpt *pOption);
+void    dndClose(SDnode *pDnode);
+void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
+
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
+int32_t       dndMarkWrapper(SMgmtWrapper *pWrapper);
+void          dndReleaseWrapper(SMgmtWrapper *pWrapper);
+
+// dndTransport.c
+int32_t dndInitServer(SDnode *pDnode);
+void    dndCleanupServer(SDnode *pDnode);
+int32_t dndInitClient(SDnode *pDnode);
+void    dndCleanupClient(SDnode *pDnode);
+int32_t dndInitMsgHandle(SDnode *pDnode);
+void    dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_INT_H_*/

source/dnode/mgmt/container/src/dndExec.c

@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static void dndResetLog(SMgmtWrapper *pMgmt) {
+  char logname[24] = {0};
+  snprintf(logname, sizeof(logname), "%slog", pMgmt->name);
+
+  dInfo("node:%s, reset log to %s", pMgmt->name, logname);
+  taosCloseLog();
+  taosInitLog(logname, 1);
+}
+
+static bool dndRequireNode(SMgmtWrapper *pWrapper) {
+  bool required = false;
+  int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required);
+  if (!required) {
+    dDebug("node:%s, no need to start", pWrapper->name);
+  } else {
+    dDebug("node:%s, need to start", pWrapper->name);
+  }
+  return required;
+}
+
+int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
+  int32_t code = (*pWrapper->fp.openFp)(pWrapper);
+  if (code != 0) {
+    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
+    return -1;
+  } else {
+    dDebug("node:%s, has been opened", pWrapper->name);
+  }
+
+  pWrapper->deployed = true;
+  return 0;
+}
+
+void dndCloseNode(SMgmtWrapper *pWrapper) {
+  taosWLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    (*pWrapper->fp.closeFp)(pWrapper);
+    pWrapper->deployed = false;
+  }
+  if (pWrapper->pProc) {
+    taosProcCleanup(pWrapper->pProc);
+    pWrapper->pProc = NULL;
+  }
+  taosWUnLockLatch(&pWrapper->latch);
+}
+
+static int32_t dndRunInSingleProcess(SDnode *pDnode) {
+  dInfo("dnode run in single process mode");
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    if (taosMkDir(pWrapper->path) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+      return -1;
+    }
+
+    dInfo("node:%s, will start in single process", pWrapper->name);
+    pWrapper->procType = PROC_SINGLE;
+    if (dndOpenNode(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+    if (pWrapper->fp.startFp == NULL) continue;
+    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
+  dndCleanupServer(pDnode);
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    if (except == n) continue;
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    dndCloseNode(pWrapper);
+  }
+}
+
+static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
+  dTrace("msg:%p, get from child queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  pRpc->pCont = pCont;
+
+  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
+  int32_t   code = (*msgFp)(pWrapper, pMsg);
+
+  if (code != 0) {
+    if (pRpc->msgType & 1U) {
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
+      dndSendRsp(pWrapper, &rsp);
+    }
+
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pCont);
+  }
+}
+
+static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) {
+  dTrace("msg:%p, get from parent queue", pRsp);
+  pRsp->pCont = pCont;
+  dndSendRpcRsp(pWrapper, pRsp);
+  free(pRsp);
+}
+
+static int32_t dndRunInMultiProcess(SDnode *pDnode) {
+  dInfo("dnode run in multi process mode");
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    if (taosMkDir(pWrapper->path) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+      return -1;
+    }
+
+    if (n == DNODE) {
+      dInfo("node:%s, will start in parent process", pWrapper->name);
+      pWrapper->procType = PROC_SINGLE;
+      if (dndOpenNode(pWrapper) != 0) {
+        dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+        return -1;
+      }
+      continue;
+    }
+
+    SProcCfg  cfg = {.childQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
+                     .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
+                     .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
+                     .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
+                     .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                     .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                     .parentQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
+                     .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
+                     .parentdMallocHeadFp = (ProcMallocFp)malloc,
+                     .parentFreeHeadFp = (ProcFreeFp)free,
+                     .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                     .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                     .testFlag = 0,
+                     .pParent = pWrapper,
+                     .name = pWrapper->name};
+    SProcObj *pProc = taosProcInit(&cfg);
+    if (pProc == NULL) {
+      dError("node:%s, failed to fork since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+
+    pWrapper->pProc = pProc;
+
+    if (taosProcIsChild(pProc)) {
+      dInfo("node:%s, will start in child process", pWrapper->name);
+      pWrapper->procType = PROC_CHILD;
+      dndResetLog(pWrapper);
+
+      dInfo("node:%s, clean up resources inherited from parent", pWrapper->name);
+      dndClearNodesExecpt(pDnode, n);
+
+      dInfo("node:%s, will be initialized in child process", pWrapper->name);
+      dndOpenNode(pWrapper);
+    } else {
+      dInfo("node:%s, will not start in parent process", pWrapper->name);
+      pWrapper->procType = PROC_PARENT;
+    }
+
+    if (taosProcRun(pProc) != 0) {
+      dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+#if 0
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+  if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) {
+    dndReleaseWrapper(pWrapper);
+    dError("failed to start dnode worker since %s", terrstr());
+    return -1;
+  }
+
+  dndReleaseWrapper(pWrapper);
+#endif
+  return 0;
+}
+
+int32_t dndRun(SDnode *pDnode) {
+  if (tsMultiProcess == 0) {
+    if (dndRunInSingleProcess(pDnode) != 0) {
+      dError("failed to run dnode in single process mode since %s", terrstr());
+      return -1;
+    }
+  } else {
+    if (dndRunInMultiProcess(pDnode) != 0) {
+      dError("failed to run dnode in multi process mode since %s", terrstr());
+      return -1;
+    }
+  }
+
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      break;
+    }
+    taosMsleep(100);
+  }
+
+  return 0;
+}

source/dnode/mgmt/container/src/dndFile.c

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 1024;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto _OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto _OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto _OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_Number) {
+    dError("failed to read %s since deployed not found", file);
+    goto _OVER;
+  }
+  *pDeployed = deployed->valueint != 0;
+
+  code = 0;
+  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+
+_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 1024;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d\n", deployed);
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s, deployed:%d", realfile, deployed);
+  return 0;
+}

source/dnode/mgmt/container/src/dndInt.c

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int8_t once = DND_ENV_INIT;
+
+int32_t dndInit() {
+  dDebug("start to init dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
+    terrno = TSDB_CODE_REPEAT_INIT;
+    dError("failed to init dnode env since %s", terrstr());
+    return -1;
+  }
+
+  taosIgnSIGPIPE();
+  taosBlockSIGPIPE();
+  taosResolveCRC();
+
+  if (rpcInit() != 0) {
+    dError("failed to init rpc since %s", terrstr());
+    dndCleanup();
+    return -1;
+  }
+
+  SMonCfg monCfg = {0};
+  monCfg.maxLogs = tsMonitorMaxLogs;
+  monCfg.port = tsMonitorPort;
+  monCfg.server = tsMonitorFqdn;
+  monCfg.comp = tsMonitorComp;
+  if (monInit(&monCfg) != 0) {
+    dError("failed to init monitor since %s", terrstr());
+    dndCleanup();
+    return -1;
+  }
+
+  dInfo("dnode env is initialized");
+  return 0;
+}
+
+void dndCleanup() {
+  dDebug("start to cleanup dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
+    dError("dnode env is already cleaned up");
+    return;
+  }
+
+  monCleanup();
+  rpcCleanup();
+  walCleanUp();
+  taosStopCacheRefreshWorker();
+  dInfo("dnode env is cleaned up");
+}
+
+void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) {
+  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
+}
+
+EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
+
+void dndSetStatus(SDnode *pDnode, EDndStatus status) {
+  if (pDnode->status != status) {
+    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
+    pDnode->status = status;
+  }
+}
+
+const char *dndStatStr(EDndStatus status) {
+  switch (status) {
+    case DND_STAT_INIT:
+      return "init";
+    case DND_STAT_RUNNING:
+      return "running";
+    case DND_STAT_STOPPED:
+      return "stopped";
+    default:
+      return "unknown";
+  }
+}
+
+void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
+  SStartupReq *pStartup = &pDnode->startup;
+  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
+  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
+  pStartup->finished = 0;
+}
+
+void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
+  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
+  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
+}
+
+TdFilePtr dndCheckRunning(char *dataDir) {
+  char filepath[PATH_MAX] = {0};
+  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
+
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    return NULL;
+  }
+
+  int32_t ret = taosLockFile(pFile);
+  if (ret != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    taosCloseFile(&pFile);
+    return NULL;
+  }
+
+  dDebug("file:%s is locked", filepath);
+  return pFile;
+}
+
+void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
+  dDebug("startup req is received");
+
+  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
+  dndGetStartup(pDnode, pStartup);
+
+  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
+
+  SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)};
+  rpcSendResponse(&rpcRsp);
+}

source/dnode/mgmt/container/src/dndMonitor.c

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
+  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
+  pInfo->logdir.size = tsLogSpace.size;
+  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
+  pInfo->tempdir.size = tsTempSpace.size;
+
+  return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo);
+}
+
+static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
+  pInfo->protocol = 1;
+  pInfo->dnode_id = pDnode->dnodeId;
+  pInfo->cluster_id = pDnode->clusterId;
+  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+}
+
+static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
+  pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
+  taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
+  taosGetCpuCores(&pInfo->cpu_cores);
+  taosGetProcMemory(&pInfo->mem_engine);
+  taosGetSysMemory(&pInfo->mem_system);
+  pInfo->mem_total = tsTotalMemoryKB;
+  pInfo->disk_engine = 0;
+  pInfo->disk_used = tsDataSpace.size.used;
+  pInfo->disk_total = tsDataSpace.size.total;
+  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
+  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
+
+  vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo);
+  pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required);
+}
+
+void dndSendMonitorReport(SDnode *pDnode) {
+  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+  SMonInfo *pMonitor = monCreateMonitorInfo();
+  if (pMonitor == NULL) return;
+
+  SMonBasicInfo basicInfo = {0};
+  dndGetMonitorBasicInfo(pDnode, &basicInfo);
+  monSetBasicInfo(pMonitor, &basicInfo);
+
+  SMonClusterInfo clusterInfo = {0};
+  SMonVgroupInfo  vgroupInfo = {0};
+  SMonGrantInfo   grantInfo = {0};
+  if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
+    monSetClusterInfo(pMonitor, &clusterInfo);
+    monSetVgroupInfo(pMonitor, &vgroupInfo);
+    monSetGrantInfo(pMonitor, &grantInfo);
+  }
+
+  SMonDnodeInfo dnodeInfo = {0};
+  dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
+  monSetDnodeInfo(pMonitor, &dnodeInfo);
+
+  SMonDiskInfo diskInfo = {0};
+  if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
+    monSetDiskInfo(pMonitor, &diskInfo);
+  }
+
+  taosArrayDestroy(clusterInfo.dnodes);
+  taosArrayDestroy(clusterInfo.mnodes);
+  taosArrayDestroy(vgroupInfo.vgroups);
+  taosArrayDestroy(diskInfo.datadirs);
+
+  monSendReport(pMonitor);
+  monCleanupMonitorInfo(pMonitor);
+}

source/dnode/mgmt/container/src/dndMsg.c

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+  if (pWrapper != NULL) {
+    dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet);
+  }
+  dndReleaseWrapper(pWrapper);
+}
+
+static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
+  if (msgFp == NULL) {
+    terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+  }
+
+  return msgFp;
+}
+
+static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
+  SRpcConnInfo connInfo = {0};
+  if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) {
+    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+    dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle);
+    return -1;
+  }
+
+  memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
+  memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
+
+  return 0;
+}
+
+void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
+  if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) {
+    dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet);
+  }
+
+  int32_t   code = -1;
+  SNodeMsg *pMsg = NULL;
+  NodeMsgFp msgFp = NULL;
+
+  if (dndMarkWrapper(pWrapper) != 0) goto _OVER;
+  if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
+  if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
+  if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER;
+
+  dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
+  if (pWrapper->procType == PROC_SINGLE) {
+    code = (*msgFp)(pWrapper->pMgmt, pMsg);
+  } else if (pWrapper->procType == PROC_PARENT) {
+    code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen);
+  } else {
+  }
+
+_OVER:
+  if (code == 0) {
+    if (pWrapper->procType == PROC_PARENT) {
+      dTrace("msg:%p, is freed", pMsg);
+      taosFreeQitem(pMsg);
+      rpcFreeCont(pRpc->pCont);
+    }
+  } else {
+    dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr());
+    if (pRpc->msgType & 1U) {
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
+      dndSendRsp(pWrapper, &rsp);
+    }
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pRpc->pCont);
+  }
+
+  dndReleaseWrapper(pWrapper);
+}
+
+static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper != NULL) {
+    dndReleaseWrapper(pWrapper);
+    terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+    dError("failed to create node since %s", terrstr());
+    return -1;
+  }
+
+  pWrapper = &pDnode->wrappers[ntype];
+
+  if (taosMkDir(pWrapper->path) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+    return -1;
+  }
+
+  int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
+  } else {
+    dDebug("node:%s, has been opened", pWrapper->name);
+    pWrapper->deployed = true;
+  }
+
+  return code;
+}
+
+static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper == NULL) {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    dError("failed to drop node since %s", terrstr());
+    return -1;
+  }
+
+  taosWLockLatch(&pWrapper->latch);
+  pWrapper->deployed = false;
+
+  int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    pWrapper->deployed = true;
+    dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
+  } else {
+    pWrapper->deployed = false;
+    dDebug("node:%s, has been dropped", pWrapper->name);
+  }
+
+  taosWUnLockLatch(&pWrapper->latch);
+  dndReleaseWrapper(pWrapper);
+  return code;
+}
+
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) {
+  switch (pMsg->rpcMsg.msgType) {
+    case TDMT_DND_CREATE_MNODE:
+      return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_DROP_MNODE:
+      return dndProcessDropNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_CREATE_QNODE:
+      return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_DROP_QNODE:
+      return dndProcessDropNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_CREATE_SNODE:
+      return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_DROP_SNODE:
+      return dndProcessDropNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_CREATE_BNODE:
+      return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg);
+    case TDMT_DND_DROP_BNODE:
+      return dndProcessDropNodeMsg(pDnode, BNODE, pMsg);
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      return -1;
+  }
+}

source/dnode/mgmt/container/src/dndObj.c

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) {
+  pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
+  pDnode->serverPort = pOption->serverPort;
+  pDnode->dataDir = strdup(pOption->dataDir);
+  pDnode->localEp = strdup(pOption->localEp);
+  pDnode->localFqdn = strdup(pOption->localFqdn);
+  pDnode->firstEp = strdup(pOption->firstEp);
+  pDnode->secondEp = strdup(pOption->secondEp);
+  pDnode->pDisks = pOption->pDisks;
+  pDnode->numOfDisks = pOption->numOfDisks;
+  pDnode->rebootTime = taosGetTimestampMs();
+
+  if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
+      pDnode->secondEp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
+static void dndClearMemory(SDnode *pDnode) {
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
+    tfree(pMgmt->path);
+  }
+  if (pDnode->pLockFile != NULL) {
+    taosUnLockFile(pDnode->pLockFile);
+    taosCloseFile(&pDnode->pLockFile);
+    pDnode->pLockFile = NULL;
+  }
+  tfree(pDnode->localEp);
+  tfree(pDnode->localFqdn);
+  tfree(pDnode->firstEp);
+  tfree(pDnode->secondEp);
+  tfree(pDnode->dataDir);
+  free(pDnode);
+  dDebug("dnode object memory is cleared, data:%p", pDnode);
+}
+
+SDnode *dndCreate(const SDnodeOpt *pOption) {
+  dInfo("start to create dnode object");
+  int32_t code = -1;
+  char    path[PATH_MAX];
+  SDnode *pDnode = NULL;
+
+  pDnode = calloc(1, sizeof(SDnode));
+  if (pDnode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _OVER;
+  }
+
+  if (dndInitMemory(pDnode, pOption) != 0) {
+    goto _OVER;
+  }
+
+  dndSetStatus(pDnode, DND_STAT_INIT);
+  pDnode->pLockFile = dndCheckRunning(pDnode->dataDir);
+  if (pDnode->pLockFile == NULL) {
+    goto _OVER;
+  }
+
+  if (dndInitServer(pDnode) != 0) {
+    dError("failed to init trans server since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (dndInitClient(pDnode) != 0) {
+    dError("failed to init trans client since %s", terrstr());
+    goto _OVER;
+  }
+
+  dmGetMgmtFp(&pDnode->wrappers[DNODE]);
+  mmGetMgmtFp(&pDnode->wrappers[MNODE]);
+  vmGetMgmtFp(&pDnode->wrappers[VNODES]);
+  qmGetMgmtFp(&pDnode->wrappers[QNODE]);
+  smGetMgmtFp(&pDnode->wrappers[SNODE]);
+  bmGetMgmtFp(&pDnode->wrappers[BNODE]);
+
+  if (dndInitMsgHandle(pDnode) != 0) {
+    goto _OVER;
+  }
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
+    pWrapper->path = strdup(path);
+    pWrapper->pDnode = pDnode;
+    if (pWrapper->path == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _OVER;
+    }
+
+    pWrapper->procType = PROC_SINGLE;
+    taosInitRWLatch(&pWrapper->latch);
+  }
+
+  code = 0;
+
+_OVER:
+  if (code != 0 && pDnode) {
+    dndClearMemory(pDnode);
+    dError("failed to create dnode object since %s", terrstr());
+  } else {
+    dInfo("dnode object is created, data:%p", pDnode);
+  }
+
+  return pDnode;
+}
+
+void dndClose(SDnode *pDnode) {
+  if (pDnode == NULL) return;
+
+  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
+    dError("dnode is shutting down, data:%p", pDnode);
+    return;
+  }
+
+  dInfo("start to close dnode, data:%p", pDnode);
+  dndSetStatus(pDnode, DND_STAT_STOPPED);
+
+  dndCleanupServer(pDnode);
+  dndCleanupClient(pDnode);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    dndCloseNode(pWrapper);
+  }
+
+  dndClearMemory(pDnode);
+  dInfo("dnode object is closed, data:%p", pDnode);
+}
+
+void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
+  dInfo("dnode object receive event %d, data:%p", event, pDnode);
+  pDnode->event = event;
+}
+
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
+  SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+  SMgmtWrapper *pRetWrapper = pWrapper;
+
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    pRetWrapper = NULL;
+  }
+  taosRUnLockLatch(&pWrapper->latch);
+
+  return pRetWrapper;
+}
+
+int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
+  int32_t code = 0;
+
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    code = -1;
+  }
+  taosRUnLockLatch(&pWrapper->latch);
+
+  return code;
+}
+
+void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
+  if (pWrapper == NULL) return;
+
+  taosRLockLatch(&pWrapper->latch);
+  int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
+  taosRUnLockLatch(&pWrapper->latch);
+  dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
+}

source/dnode/mgmt/container/src/dndTransport.c

@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+#define INTERNAL_USER   "_dnd"
+#define INTERNAL_CKEY   "_key"
+#define INTERNAL_SECRET "_pwd"
+
+static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
+  SDnode     *pDnode = parent;
+  STransMgmt *pMgmt = &pDnode->trans;
+  tmsg_t      msgType = pRsp->msgType;
+
+  if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+    // if (pRsp == NULL || pRsp->pCont == NULL) return;
+    dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
+    rpcFreeCont(pRsp->pCont);
+    return;
+  }
+
+  SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
+  if (pHandle->msgFp != NULL) {
+    dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType),
+           pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code));
+    dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet);
+  } else {
+    dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle);
+    rpcFreeCont(pRsp->pCont);
+  }
+}
+
+int32_t dndInitClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.label = "DND";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp = dndProcessResponse;
+  rpcInit.sessions = 1024;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.user = INTERNAL_USER;
+  rpcInit.ckey = INTERNAL_CKEY;
+  rpcInit.spi = 1;
+  rpcInit.parent = pDnode;
+
+  char pass[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
+  rpcInit.secret = pass;
+
+  pMgmt->clientRpc = rpcOpen(&rpcInit);
+  if (pMgmt->clientRpc == NULL) {
+    dError("failed to init dnode rpc client");
+    return -1;
+  }
+
+  dDebug("dnode rpc client is initialized");
+  return 0;
+}
+
+void dndCleanupClient(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+  if (pMgmt->clientRpc) {
+    rpcClose(pMgmt->clientRpc);
+    pMgmt->clientRpc = NULL;
+    dDebug("dnode rpc client is closed");
+  }
+}
+
+static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
+  SDnode     *pDnode = param;
+  STransMgmt *pMgmt = &pDnode->trans;
+  tmsg_t      msgType = pReq->msgType;
+
+  if (msgType == TDMT_DND_NETWORK_TEST) {
+    dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle);
+    dndProcessStartupReq(pDnode, pReq);
+    return;
+  }
+
+  if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+    dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pReq->pCont);
+    return;
+  }
+
+  if (pReq->pCont == NULL) {
+    dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    return;
+  }
+
+  SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)];
+  if (pHandle->msgFp != NULL) {
+    dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name,
+           pReq->handle, pReq->ahandle);
+    dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet);
+  } else {
+    dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle);
+    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
+    rpcSendResponse(&rspMsg);
+    rpcFreeCont(pReq->pCont);
+  }
+}
+
+static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  SEpSet epSet = {0};
+  dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
+  rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
+}
+
+static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  int32_t code = 0;
+  char    pass[TSDB_PASSWORD_LEN + 1] = {0};
+
+  if (strcmp(user, INTERNAL_USER) == 0) {
+    taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
+  } else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
+    taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
+  } else {
+    code = -1;
+  }
+
+  if (code == 0) {
+    memcpy(secret, pass, TSDB_PASSWORD_LEN);
+    *spi = 1;
+    *encrypt = 0;
+    *ckey = 0;
+  }
+
+  return code;
+}
+
+static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SDnode *pDnode = parent;
+
+  if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) {
+    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
+    return 0;
+  }
+
+  if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) {
+    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
+    return 0;
+  }
+
+  if (terrno != TSDB_CODE_APP_NOT_READY) {
+    dTrace("failed to get user auth from mnode since %s", terrstr());
+    return -1;
+  }
+
+  SAuthReq authReq = {0};
+  tstrncpy(authReq.user, user, TSDB_USER_LEN);
+  int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
+  void   *pReq = rpcMallocCont(contLen);
+  tSerializeSAuthReq(pReq, contLen, &authReq);
+
+  SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
+  SRpcMsg rpcRsp = {0};
+  dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
+  dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
+
+  if (rpcRsp.code != 0) {
+    terrno = rpcRsp.code;
+    dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
+  } else {
+    SAuthRsp authRsp = {0};
+    tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
+    memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
+    memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
+    *spi = authRsp.spi;
+    *encrypt = authRsp.encrypt;
+    dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
+           authRsp.encrypt);
+  }
+
+  rpcFreeCont(rpcRsp.pCont);
+  return rpcRsp.code;
+}
+
+int32_t dndInitServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
+  if (numOfThreads < 1) {
+    numOfThreads = 1;
+  }
+
+  SRpcInit rpcInit;
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort = pDnode->serverPort;
+  rpcInit.label = "DND";
+  rpcInit.numOfThreads = numOfThreads;
+  rpcInit.cfp = dndProcessRequest;
+  rpcInit.sessions = tsMaxShellConns;
+  rpcInit.connType = TAOS_CONN_SERVER;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.afp = dndRetrieveUserAuthInfo;
+  rpcInit.parent = pDnode;
+
+  pMgmt->serverRpc = rpcOpen(&rpcInit);
+  if (pMgmt->serverRpc == NULL) {
+    dError("failed to init dnode rpc server");
+    return -1;
+  }
+
+  dDebug("dnode rpc server is initialized");
+  return 0;
+}
+
+void dndCleanupServer(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+  if (pMgmt->serverRpc) {
+    rpcClose(pMgmt->serverRpc);
+    pMgmt->serverRpc = NULL;
+    dDebug("dnode rpc server is closed");
+  }
+}
+
+int32_t dndInitMsgHandle(SDnode *pDnode) {
+  STransMgmt *pMgmt = &pDnode->trans;
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+
+    for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) {
+      NodeMsgFp msgFp = pWrapper->msgFps[msgIndex];
+      if (msgFp == NULL) continue;
+
+      SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex];
+      if (pHandle->msgFp != NULL) {
+        dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex],
+               pHandle->pWrapper->name, pWrapper->name);
+        return -1;
+      } else {
+        dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name);
+        pHandle->msgFp = msgFp;
+        pHandle->pWrapper = pWrapper;
+      }
+    }
+  }
+
+  return 0;
+}
+
+static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
+  if (pMgmt->clientRpc == NULL) {
+    terrno = TSDB_CODE_DND_OFFLINE;
+    return -1;
+  }
+
+  rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL);
+  return 0;
+}
+
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) {
+  if (pWrapper->procType == PROC_CHILD) {
+  } else {
+    SDnode *pDnode = pWrapper->pDnode;
+    if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
+      terrno = TSDB_CODE_DND_OFFLINE;
+      dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle);
+      return -1;
+    }
+    return dndSendRpcReq(&pDnode->trans, pEpSet, pReq);
+  }
+}
+
+int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
+  if (pWrapper->procType == PROC_CHILD) {
+  } else {
+    SDnode     *pDnode = pWrapper->pDnode;
+    STransMgmt *pTrans = &pDnode->trans;
+    SEpSet      epSet = {0};
+    dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
+    return dndSendRpcReq(pTrans, &epSet, pReq);
+  }
+}
+
+void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+  if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
+    SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
+    dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp);
+  } else {
+    rpcSendResponse(pRsp);
+  }
+}
+
+void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+  if (pWrapper->procType == PROC_CHILD) {
+    int32_t code = -1;
+    do {
+      code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen);
+      if (code != 0) {
+        taosMsleep(10);
+      }
+    } while (code != 0);
+  } else {
+    dndSendRpcRsp(pWrapper, pRsp);
+  }
+}

source/dnode/mgmt/container/src/dndWorker.c

@@ -14,11 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dndWorker.h"
+#include "dndInt.h"
 
-int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
+int32_t dndInitWorker(void *param, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
                       int32_t maxNum, void *queueFp) {
-  if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
+  if (pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return -1;
   }
@@ -28,7 +28,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
   pWorker->minNum = minNum;
   pWorker->maxNum = maxNum;
   pWorker->queueFp = queueFp;
-  pWorker->pDnode = pDnode;
+  pWorker->param = param;
 
   if (pWorker->type == DND_WORKER_SINGLE) {
     SQWorkerPool *pPool = &pWorker->pool;
@@ -39,7 +39,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp);
+    pWorker->queue = tQWorkerAllocQueue(pPool, param, (FItem)queueFp);
     if (pWorker->queue == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -52,7 +52,7 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
-    pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp);
+    pWorker->queue = tWWorkerAllocQueue(pPool, param, (FItems)queueFp);
     if (pWorker->queue == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -65,6 +65,8 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
 }
 
 void dndCleanupWorker(SDnodeWorker *pWorker) {
+  if (pWorker->queue == NULL) return;
+
   while (!taosQueueEmpty(pWorker->queue)) {
     taosMsleep(10);
   }
@@ -79,31 +81,13 @@ void dndCleanupWorker(SDnodeWorker *pWorker) {
   }
 }
 
-int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) {
+int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pMsg) {
   if (pWorker == NULL || pWorker->queue == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return -1;
   }
 
-  void *pMsg = NULL;
-  if (contLen != 0) {
-    pMsg = taosAllocateQitem(contLen);
-    if (pMsg != NULL) {
-      memcpy(pMsg, pCont, contLen);
-    }
-  } else {
-    pMsg = pCont;
-  }
-
-  if (pMsg == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
   if (taosWriteQitem(pWorker->queue, pMsg) != 0) {
-    if (contLen != 0) {
-      taosFreeQitem(pMsg);
-    }
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }

source/dnode/mgmt/daemon/src/dmnCfg.c

@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dmnInt.h"
-#include "tconfig.h"
-
-SDnodeObjCfg dmnGetObjCfg() {
-  SConfig *pCfg = taosGetCfg();
-  SDnodeObjCfg objCfg = {0};
-
-  objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
-  tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir));
-  tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp));
-  tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp));
-  objCfg.serverPort = tsServerPort;
-  tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn));
-  snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort);
-  objCfg.pDisks = tsDiskCfg;
-  objCfg.numOfDisks = tsDiskCfgNum;
-  return objCfg;
-}
-
-void dmnDumpCfg() {
-  SConfig *pCfg = taosGetCfg();
-  cfgDumpCfg(pCfg, 0, 1);
-}

source/dnode/mgmt/daemon/src/dmnMain.c

@@ -1,136 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dmnInt.h"
-
-static struct {
-  bool stop;
-  bool dumpConfig;
-  bool generateGrant;
-  bool printAuth;
-  bool printVersion;
-  char envFile[PATH_MAX];
-  char apolloUrl[PATH_MAX];
-} dmn = {0};
-
-static void dmnSigintHandle(int signum, void *info, void *ctx) {
-  uInfo("signal:%d is received", signum);
-  dmn.stop = true;
-}
-
-static void dmnSetSignalHandle() {
-  taosSetSignal(SIGTERM, dmnSigintHandle);
-  taosSetSignal(SIGHUP, dmnSigintHandle);
-  taosSetSignal(SIGINT, dmnSigintHandle);
-  taosSetSignal(SIGABRT, dmnSigintHandle);
-  taosSetSignal(SIGBREAK, dmnSigintHandle);
-}
-
-static void dmnWaitSignal() {
-  dmnSetSignalHandle();
-  while (!dmn.stop) {
-    taosMsleep(100);
-  }
-}
-
-static int32_t dmnParseOption(int32_t argc, char const *argv[]) {
-  for (int32_t i = 1; i < argc; ++i) {
-    if (strcmp(argv[i], "-c") == 0) {
-      if (i < argc - 1) {
-        if (strlen(argv[++i]) >= PATH_MAX) {
-          printf("config file path overflow");
-          return -1;
-        }
-        tstrncpy(configDir, argv[i], PATH_MAX);
-      } else {
-        printf("'-c' requires a parameter, default is %s\n", configDir);
-        return -1;
-      }
-    } else if (strcmp(argv[i], "-C") == 0) {
-      dmn.dumpConfig = true;
-    } else if (strcmp(argv[i], "-k") == 0) {
-      dmn.generateGrant = true;
-    } else if (strcmp(argv[i], "-V") == 0) {
-      dmn.printVersion = true;
-    } else {
-    }
-  }
-
-  return 0;
-}
-
-int32_t dmnRunDnode() {
-  if (dndInit() != 0) {
-    uInfo("Failed to start TDengine, please check the log");
-    return -1;
-  }
-
-  SDnodeObjCfg objCfg = dmnGetObjCfg();
-  SDnode      *pDnode = dndCreate(&objCfg);
-  if (pDnode == NULL) {
-    uInfo("Failed to start TDengine, please check the log");
-    return -1;
-  }
-
-  uInfo("Started TDengine service successfully.");
-  dmnWaitSignal();
-  uInfo("TDengine is shut down!");
-
-  dndClose(pDnode);
-  dndCleanup();
-  taosCloseLog();
-  taosCleanupCfg();
-  return 0;
-}
-
-int main(int argc, char const *argv[]) {
-  if (!taosCheckSystemIsSmallEnd()) {
-    uError("TDengine does not run on non-small-end machines.");
-    return -1;
-  }
-
-  if (dmnParseOption(argc, argv) != 0) {
-    return -1;
-  }
-
-  if (dmn.generateGrant) {
-    dmnGenerateGrant();
-    return 0;
-  }
-
-  if (dmn.printVersion) {
-    dmnPrintVersion();
-    return 0;
-  }
-
-  if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) {
-    uInfo("Failed to start TDengine since read config error");
-    return -1;
-  }
-
-  if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) {
-    uInfo("Failed to start TDengine since read config error");
-    return -1;
-  }
-
-  if (dmn.dumpConfig) {
-    dmnDumpCfg();
-    taosCleanupCfg();
-    return 0;
-  }
-
-  return dmnRunDnode();
-}

source/dnode/mgmt/dnode/inc/dm.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_DNODE_H_
+#define _TD_DND_DNODE_H_
+
+#include "dnd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SDnodeMgmt SDnodeMgmt;
+
+void dmGetMgmtFp(SMgmtWrapper *pWrapper);
+void dmInitMsgHandles(SMgmtWrapper *pWrapper);
+
+void    dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void    dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void    dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_DNODE_H_*/

source/dnode/mgmt/dnode/inc/dmInt.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_DNODE_INT_H_
+#define _TD_DND_DNODE_INT_H_
+
+#include "dm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SDnodeMgmt {
+  int64_t       dver;
+  int64_t       updateTime;
+  int8_t        statusSent;
+  SEpSet        mnodeEpSet;
+  SHashObj     *dnodeHash;
+  SArray       *dnodeEps;
+  TdThread    *threadId;
+  SRWLatch      latch;
+  SDnodeWorker  mgmtWorker;
+  SDnodeWorker  statusWorker;
+  const char   *path;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+} SDnodeMgmt;
+
+// dmFile.c
+int32_t dmReadFile(SDnodeMgmt *pMgmt);
+int32_t dmWriteFile(SDnodeMgmt *pMgmt);
+void    dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps);
+
+// dmMsg.c
+void    dmSendStatusReq(SDnodeMgmt *pMgmt);
+int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+// dmWorker.c
+int32_t dmStartWorker(SDnodeMgmt *pMgmt);
+void    dmStopWorker(SDnodeMgmt *pMgmt);
+int32_t dmStartThread(SDnodeMgmt *pMgmt);
+int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_DNODE_INT_H_*/

source/dnode/mgmt/dnode/src/dmFile.c

@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+
+static void dmPrintDnodes(SDnodeMgmt *pMgmt);
+static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep);
+static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps);
+
+int32_t dmReadFile(SDnodeMgmt *pMgmt) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 256 * 1024;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+  SDnode   *pDnode = pMgmt->pDnode;
+
+  pMgmt->dnodeEps = taosArrayInit(1, sizeof(SDnodeEp));
+  if (pMgmt->dnodeEps == NULL) {
+    dError("failed to calloc dnodeEp array since %s", strerror(errno));
+    goto PRASE_DNODE_OVER;
+  }
+
+  snprintf(file, sizeof(file), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_DNODE_OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
+  if (!dnodeId || dnodeId->type != cJSON_Number) {
+    dError("failed to read %s since dnodeId not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dnodeId = dnodeId->valueint;
+
+  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
+  if (!clusterId || clusterId->type != cJSON_String) {
+    dError("failed to read %s since clusterId not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->clusterId = atoll(clusterId->valuestring);
+
+  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
+  if (!dropped || dropped->type != cJSON_Number) {
+    dError("failed to read %s since dropped not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+  pDnode->dropped = dropped->valueint;
+
+  cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes");
+  if (!dnodes || dnodes->type != cJSON_Array) {
+    dError("failed to read %s since dnodes not found", file);
+    goto PRASE_DNODE_OVER;
+  }
+
+  int32_t numOfDnodes = cJSON_GetArraySize(dnodes);
+  if (numOfDnodes <= 0) {
+    dError("failed to read %s since numOfDnodes:%d invalid", file, numOfDnodes);
+    goto PRASE_DNODE_OVER;
+  }
+
+  for (int32_t i = 0; i < numOfDnodes; ++i) {
+    cJSON *node = cJSON_GetArrayItem(dnodes, i);
+    if (node == NULL) break;
+
+    SDnodeEp dnodeEp = {0};
+
+    cJSON *did = cJSON_GetObjectItem(node, "id");
+    if (!did || did->type != cJSON_Number) {
+      dError("failed to read %s since dnodeId not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+
+    dnodeEp.id = dnodeId->valueint;
+
+    cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
+    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
+      dError("failed to read %s since dnodeFqdn not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+    tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
+    if (!dnodePort || dnodePort->type != cJSON_Number) {
+      dError("failed to read %s since dnodePort not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+
+    dnodeEp.ep.port = dnodePort->valueint;
+
+    cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
+    if (!isMnode || isMnode->type != cJSON_Number) {
+      dError("failed to read %s since isMnode not found", file);
+      goto PRASE_DNODE_OVER;
+    }
+    dnodeEp.isMnode = isMnode->valueint;
+
+    taosArrayPush(pMgmt->dnodeEps, &dnodeEp);
+  }
+
+  code = 0;
+  dInfo("succcessed to read file %s", file);
+  dmPrintDnodes(pMgmt);
+
+PRASE_DNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  if (dmIsEpChanged(pMgmt, pDnode->dnodeId, pDnode->localEp)) {
+    dError("localEp %s different with %s and need reconfigured", pDnode->localEp, file);
+    return -1;
+  }
+
+  if (taosArrayGetSize(pMgmt->dnodeEps) == 0) {
+    SDnodeEp dnodeEp = {0};
+    dnodeEp.isMnode = 1;
+    taosGetFqdnPortFromEp(pDnode->firstEp, &dnodeEp.ep);
+    taosArrayPush(pMgmt->dnodeEps, &dnodeEp);
+  }
+
+  dmResetDnodes(pMgmt, pMgmt->dnodeEps);
+
+  terrno = code;
+  return code;
+}
+
+int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    dError("failed to write %s since %s", file, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 256 * 1024;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", pDnode->dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pDnode->dropped);
+  len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
+
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  for (int32_t i = 0; i < numOfEps; ++i) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->dnodeEps, i);
+    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pDnodeEp->id);
+    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
+    len += snprintf(content + len, maxLen - len, "    \"port\": %u,\n", pDnodeEp->ep.port);
+    len += snprintf(content + len, maxLen - len, "    \"isMnode\": %d\n", pDnodeEp->isMnode);
+    if (i < numOfEps - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  pMgmt->updateTime = taosGetTimestampMs();
+  dDebug("successed to write %s", file);
+  return 0;
+}
+
+void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *dnodeEps) {
+  int32_t numOfEps = taosArrayGetSize(dnodeEps);
+  if (numOfEps <= 0) return;
+
+  taosWLockLatch(&pMgmt->latch);
+
+  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  if (numOfEps != numOfEpsOld) {
+    dmResetDnodes(pMgmt, dnodeEps);
+    dmWriteFile(pMgmt);
+  } else {
+    int32_t size = numOfEps * sizeof(SDnodeEp);
+    if (memcmp(pMgmt->dnodeEps->pData, dnodeEps->pData, size) != 0) {
+      dmResetDnodes(pMgmt, dnodeEps);
+      dmWriteFile(pMgmt);
+    }
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+}
+
+static void dmResetDnodes(SDnodeMgmt *pMgmt, SArray *dnodeEps) {
+  if (pMgmt->dnodeEps != dnodeEps) {
+    SArray *tmp = pMgmt->dnodeEps;
+    pMgmt->dnodeEps = taosArrayDup(dnodeEps);
+    taosArrayDestroy(tmp);
+  }
+
+  pMgmt->mnodeEpSet.inUse = 0;
+  pMgmt->mnodeEpSet.numOfEps = 0;
+
+  int32_t mIndex = 0;
+  int32_t numOfEps = (int32_t)taosArrayGetSize(dnodeEps);
+
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i);
+    if (!pDnodeEp->isMnode) continue;
+    if (mIndex >= TSDB_MAX_REPLICA) continue;
+    pMgmt->mnodeEpSet.numOfEps++;
+
+    pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep;
+    mIndex++;
+  }
+
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(dnodeEps, i);
+    taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
+  }
+
+  dmPrintDnodes(pMgmt);
+}
+
+static void dmPrintDnodes(SDnodeMgmt *pMgmt) {
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->dnodeEps);
+  dDebug("print dnode ep list, num:%d", numOfEps);
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pEp = taosArrayGet(pMgmt->dnodeEps, i);
+    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
+  }
+}
+
+static bool dmIsEpChanged(SDnodeMgmt *pMgmt, int32_t dnodeId, const char *ep) {
+  bool changed = false;
+  taosRLockLatch(&pMgmt->latch);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    char epstr[TSDB_EP_LEN + 1];
+    snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
+    changed = strcmp(ep, epstr) != 0;
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+  return changed;
+}

source/dnode/mgmt/dnode/src/dmInt.c

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+
+void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) {
+  taosRLockLatch(&pMgmt->latch);
+  *pEpSet = pMgmt->mnodeEpSet;
+  taosRUnLockLatch(&pMgmt->latch);
+}
+
+void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet) {
+  dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
+
+  taosWLockLatch(&pMgmt->latch);
+  pMgmt->mnodeEpSet = *pEpSet;
+  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
+    dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+}
+
+void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
+  SDnodeMgmt *pMgmt = pWrapper->pMgmt;
+  taosRLockLatch(&pMgmt->latch);
+
+  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
+  if (pDnodeEp != NULL) {
+    if (pPort != NULL) {
+      *pPort = pDnodeEp->ep.port;
+    }
+    if (pFqdn != NULL) {
+      tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
+    }
+    if (pEp != NULL) {
+      snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
+    }
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+}
+
+void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  SEpSet epSet = {0};
+  dmGetMnodeEpSet(pMgmt, &epSet);
+
+  dDebug("RPC %p, req is redirected, num:%d use:%d", pReq->handle, epSet.numOfEps, epSet.inUse);
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+    if (strcmp(epSet.eps[i].fqdn, pDnode->localFqdn) == 0 && epSet.eps[i].port == pDnode->serverPort) {
+      epSet.inUse = (i + 1) % epSet.numOfEps;
+    }
+
+    epSet.eps[i].port = htons(epSet.eps[i].port);
+  }
+
+  rpcSendRedirectRsp(pReq->handle, &epSet);
+}
+
+static int32_t dmStart(SMgmtWrapper *pWrapper) {
+  dDebug("dnode mgmt start to run");
+  return dmStartThread(pWrapper->pMgmt);
+}
+
+int32_t dmInit(SMgmtWrapper *pWrapper) {
+  SDnode     *pDnode = pWrapper->pDnode;
+  SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt));
+  dInfo("dnode-mgmt is initialized");
+
+  pDnode->dnodeId = 0;
+  pDnode->dropped = 0;
+  pDnode->clusterId = 0;
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pDnode;
+  pMgmt->pWrapper = pWrapper;
+  taosInitRWLatch(&pMgmt->latch);
+
+  pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pMgmt->dnodeHash == NULL) {
+    dError("failed to init dnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dmReadFile(pMgmt) != 0) {
+    dError("failed to read file since %s", terrstr());
+    return -1;
+  }
+
+  if (pDnode->dropped) {
+    dError("dnode will not start since its already dropped");
+    return -1;
+  }
+
+  if (dmStartWorker(pMgmt) != 0) {
+    return -1;
+  }
+
+  pWrapper->pMgmt = pMgmt;
+  dInfo("dnode-mgmt is initialized");
+  return 0;
+}
+
+void dmCleanup(SMgmtWrapper *pWrapper) {
+  SDnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("dnode-mgmt start to clean up");
+  dmStopWorker(pMgmt);
+
+  taosWLockLatch(&pMgmt->latch);
+
+  if (pMgmt->dnodeEps != NULL) {
+    taosArrayDestroy(pMgmt->dnodeEps);
+    pMgmt->dnodeEps = NULL;
+  }
+
+  if (pMgmt->dnodeHash != NULL) {
+    taosHashCleanup(pMgmt->dnodeHash);
+    pMgmt->dnodeHash = NULL;
+  }
+
+  taosWUnLockLatch(&pMgmt->latch);
+
+  free(pMgmt);
+  pWrapper->pMgmt = NULL;
+  dInfo("dnode-mgmt is cleaned up");
+}
+
+int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) {
+  *required = true;
+  return 0;
+}
+
+void dmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = dmInit;
+  mgmtFp.closeFp = dmCleanup;
+  mgmtFp.startFp = dmStart;
+  mgmtFp.requiredFp = dmRequire;
+
+  dmInitMsgHandles(pWrapper);
+  pWrapper->name = "dnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/dnode/src/dmMsg.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmInt.h"
+#include "vm.h"
+
+void dmSendStatusReq(SDnodeMgmt *pMgmt) {
+  SDnode    *pDnode = pMgmt->pDnode;
+  SStatusReq req = {0};
+
+  taosRLockLatch(&pMgmt->latch);
+  req.sver = tsVersion;
+  req.dver = pMgmt->dver;
+  req.dnodeId = pDnode->dnodeId;
+  req.clusterId = pDnode->clusterId;
+  req.rebootTime = pDnode->rebootTime;
+  req.updateTime = pMgmt->updateTime;
+  req.numOfCores = tsNumOfCores;
+  req.numOfSupportVnodes = pDnode->numOfSupportVnodes;
+  tstrncpy(req.dnodeEp, pDnode->localEp, TSDB_EP_LEN);
+
+  req.clusterCfg.statusInterval = tsStatusInterval;
+  req.clusterCfg.checkTime = 0;
+  char timestr[32] = "1970-01-01 00:00:00.00";
+  (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN);
+  memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN);
+  memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
+  taosRUnLockLatch(&pMgmt->latch);
+
+  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
+  vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads);
+
+  int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
+  void   *pHead = rpcMallocCont(contLen);
+  tSerializeSStatusReq(pHead, contLen, &req);
+  taosArrayDestroy(req.pVloads);
+
+  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
+  pMgmt->statusSent = 1;
+
+  dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);
+  dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg);
+}
+
+static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  if (pDnode->dnodeId == 0) {
+    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
+    taosWLockLatch(&pMgmt->latch);
+    pDnode->dnodeId = pCfg->dnodeId;
+    pDnode->clusterId = pCfg->clusterId;
+    dmWriteFile(pMgmt);
+    taosWUnLockLatch(&pMgmt->latch);
+  }
+}
+
+int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pMgmt->pDnode;
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+
+  if (pRsp->code != TSDB_CODE_SUCCESS) {
+    if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pDnode->dropped && pDnode->dnodeId > 0) {
+      dInfo("dnode:%d, set to dropped since not exist in mnode", pDnode->dnodeId);
+      pDnode->dropped = 1;
+      dmWriteFile(pMgmt);
+    }
+  } else {
+    SStatusRsp statusRsp = {0};
+    if (pRsp->pCont != NULL && pRsp->contLen != 0 &&
+        tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
+      pMgmt->dver = statusRsp.dver;
+      dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
+      dmUpdateDnodeEps(pMgmt, statusRsp.pDnodeEps);
+    }
+    tFreeSStatusRsp(&statusRsp);
+  }
+
+  pMgmt->statusSent = 0;
+}
+
+int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+  dError("auth rsp is received, but not supported yet");
+  return 0;
+}
+
+int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg *pRsp = &pMsg->rpcMsg;
+  dError("grant rsp is received, but not supported yet");
+  return 0;
+}
+
+int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SRpcMsg       *pReq = &pMsg->rpcMsg;
+  SDCfgDnodeReq *pCfg = pReq->pCont;
+  dError("config req is received, but not supported yet");
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+}
+
+void dmInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by DNODE
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg);
+
+  // Requests handled by MNODE
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg);
+}

source/dnode/mgmt/dnode/src/dmWorker.c

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "bm.h"
+#include "dmInt.h"
+#include "mm.h"
+#include "qm.h"
+#include "sm.h"
+#include "vm.h"
+
+static void *dmThreadRoutine(void *param) {
+  SDnodeMgmt *pMgmt = param;
+  SDnode     *pDnode = pMgmt->pDnode;
+  int64_t     lastStatusTime = taosGetTimestampMs();
+  int64_t     lastMonitorTime = lastStatusTime;
+
+  setThreadName("dnode-hb");
+
+  while (true) {
+    taosThreadTestCancel();
+    taosMsleep(200);
+    if (dndGetStatus(pDnode) != DND_STAT_RUNNING || pDnode->dropped) {
+      continue;
+    }
+
+    int64_t curTime = taosGetTimestampMs();
+
+    float statusInterval = (curTime - lastStatusTime) / 1000.0f;
+    if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
+      dmSendStatusReq(pMgmt);
+      lastStatusTime = curTime;
+    }
+
+    float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
+    if (monitorInterval >= tsMonitorInterval) {
+      dndSendMonitorReport(pDnode);
+      lastMonitorTime = curTime;
+    }
+  }
+}
+
+static void dmProcessQueue(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode *pDnode = pMgmt->pDnode;
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t code = -1;
+  dTrace("msg:%p, will be processed in dnode queue", pMsg);
+
+  switch (pRpc->msgType) {
+    case TDMT_DND_CREATE_MNODE:
+    case TDMT_DND_CREATE_QNODE:
+    case TDMT_DND_CREATE_SNODE:
+    case TDMT_DND_CREATE_BNODE:
+    case TDMT_DND_DROP_MNODE:
+    case TDMT_DND_DROP_QNODE:
+    case TDMT_DND_DROP_SNODE:
+    case TDMT_DND_DROP_BNODE:
+      code = dndProcessNodeMsg(pMgmt->pDnode, pMsg);
+      break;
+    case TDMT_DND_CONFIG_DNODE:
+      code = dmProcessConfigReq(pMgmt, pMsg);
+      break;
+    case TDMT_MND_STATUS_RSP:
+      code = dmProcessStatusRsp(pMgmt, pMsg);
+      break;
+    case TDMT_MND_AUTH_RSP:
+      code = dmProcessAuthRsp(pMgmt, pMsg);
+      break;
+    case TDMT_MND_GRANT_RSP:
+      code = dmProcessGrantRsp(pMgmt, pMsg);
+      break;
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType));
+  }
+
+  if (pRpc->msgType & 1u) {
+    if (code != 0) code = terrno;
+    SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dmProcessQueue) != 0) {
+    dError("failed to start dnode mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dmProcessQueue) != 0) {
+    dError("failed to start dnode mgmt worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t dmStartThread(SDnodeMgmt *pMgmt) {
+  pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt);
+  if (pMgmt->threadId == NULL) {
+    dError("failed to init dnode thread");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+void dmStopWorker(SDnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->mgmtWorker);
+  dndCleanupWorker(&pMgmt->statusWorker);
+
+  if (pMgmt->threadId != NULL) {
+    taosDestoryThread(pMgmt->threadId);
+    pMgmt->threadId = NULL;
+  }
+}
+
+int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->mgmtWorker;
+  if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) {
+    pWorker = &pMgmt->statusWorker;
+  }
+
+  dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}

source/dnode/mgmt/impl/CMakeLists.txt

@@ -1,14 +0,0 @@
-aux_source_directory(src DNODE_SRC)
-add_library(dnode STATIC ${DNODE_SRC})
-target_link_libraries(
-    dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
-)
-target_include_directories(
-    dnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-if(${BUILD_TEST})
-    add_subdirectory(test)
-endif(${BUILD_TEST})

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_ENV_H_
-#define _TD_DND_ENV_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "dndInt.h"
-
-typedef struct {
-  EWorkerType type;
-  const char *name;
-  int32_t     minNum;
-  int32_t     maxNum;
-  void       *queueFp;
-  SDnode     *pDnode;
-  STaosQueue *queue;
-  union {
-    SQWorkerPool pool;
-    SWWorkerPool mpool;
-  };
-} SDnodeWorker;
-
-typedef struct {
-  char *dnode;
-  char *mnode;
-  char *snode;
-  char *bnode;
-  char *vnodes;
-} SDnodeDir;
-
-typedef struct {
-  int32_t      dnodeId;
-  int32_t      dropped;
-  int64_t      clusterId;
-  int64_t      dver;
-  int64_t      rebootTime;
-  int64_t      updateTime;
-  int8_t       statusSent;
-  SEpSet       mnodeEpSet;
-  char        *file;
-  SHashObj    *dnodeHash;
-  SArray      *pDnodeEps;
-  pthread_t   *threadId;
-  SRWLatch     latch;
-  SDnodeWorker mgmtWorker;
-  SDnodeWorker statusWorker;
-} SDnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SMnode      *pMnode;
-  SRWLatch     latch;
-  SDnodeWorker readWorker;
-  SDnodeWorker writeWorker;
-  SDnodeWorker syncWorker;
-  int8_t       replica;
-  int8_t       selfIndex;
-  SReplica     replicas[TSDB_MAX_REPLICA];
-} SMnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SQnode      *pQnode;
-  SRWLatch     latch;
-  SDnodeWorker queryWorker;
-  SDnodeWorker fetchWorker;
-} SQnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  int8_t       uniqueWorkerInUse;
-  SSnode      *pSnode;
-  SRWLatch     latch;
-  SArray      *uniqueWorkers;  // SArray<SDnodeWorker*>
-  SDnodeWorker sharedWorker;
-} SSnodeMgmt;
-
-typedef struct {
-  int32_t      refCount;
-  int8_t       deployed;
-  int8_t       dropped;
-  SBnode      *pBnode;
-  SRWLatch     latch;
-  SDnodeWorker writeWorker;
-} SBnodeMgmt;
-
-typedef struct {
-  int32_t openVnodes;
-  int32_t totalVnodes;
-  int32_t masterNum;
-  int64_t numOfSelectReqs;
-  int64_t numOfInsertReqs;
-  int64_t numOfInsertSuccessReqs;
-  int64_t numOfBatchInsertReqs;
-  int64_t numOfBatchInsertSuccessReqs;
-} SVnodesStat;
-
-typedef struct {
-  SVnodesStat  stat;
-  SHashObj    *hash;
-  SRWLatch     latch;
-  SQWorkerPool queryPool;
-  SFWorkerPool fetchPool;
-  SWWorkerPool syncPool;
-  SWWorkerPool writePool;
-} SVnodesMgmt;
-
-typedef struct {
-  void    *serverRpc;
-  void    *clientRpc;
-  DndMsgFp msgFp[TDMT_MAX];
-} STransMgmt;
-
-typedef struct SDnode {
-  EStat        stat;
-  SDnodeObjCfg cfg;
-  SDnodeDir    dir;
-  TdFilePtr    pLockFile;
-  SDnodeMgmt   dmgmt;
-  SMnodeMgmt   mmgmt;
-  SQnodeMgmt   qmgmt;
-  SSnodeMgmt   smgmt;
-  SBnodeMgmt   bmgmt;
-  SVnodesMgmt  vmgmt;
-  STransMgmt   tmgmt;
-  STfs        *pTfs;
-  SStartupReq  startup;
-} SDnode;
-
-int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_ENV_H_*/

source/dnode/mgmt/impl/inc/dndMgmt.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_DNODE_H_
-#define _TD_DND_DNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitMgmt(SDnode *pDnode);
-void    dndStopMgmt(SDnode *pDnode);
-void    dndCleanupMgmt(SDnode *pDnode);
-
-int32_t dndGetDnodeId(SDnode *pDnode);
-int64_t dndGetClusterId(SDnode *pDnode);
-void    dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
-void    dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet);
-
-void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg);
-void dndSendStatusReq(SDnode *pDnode);
-void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet);
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_DNODE_H_*/

source/dnode/mgmt/impl/inc/dndMnode.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_MNODE_H_
-#define _TD_DND_MNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitMnode(SDnode *pDnode);
-void    dndCleanupMnode(SDnode *pDnode);
-
-int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
-void    dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-
-int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                               SMonGrantInfo *pGrantInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_MNODE_H_*/

source/dnode/mgmt/impl/inc/dndSnode.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_SNODE_H_
-#define _TD_DND_SNODE_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitSnode(SDnode *pDnode);
-void    dndCleanupSnode(SDnode *pDnode);
-
-// void    dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
-
-void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_SNODE_H_*/

source/dnode/mgmt/impl/inc/dndVnodes.h

@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_VNODES_H_
-#define _TD_DND_VNODES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "dndEnv.h"
-
-int32_t dndInitVnodes(SDnode *pDnode);
-void    dndCleanupVnodes(SDnode *pDnode);
-void    dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads);
-void    dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-void    dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
-
-int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-
-int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_VNODES_H_*/

source/dnode/mgmt/impl/src/dndBnode.c

@@ -1,392 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndBnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs);
-
-static SBnode *dndAcquireBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  SBnode     *pBnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pBnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pBnode = pMgmt->pBnode;
-  } else {
-    terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pBnode != NULL) {
-    dTrace("acquire bnode, refCount:%d", refCount);
-  }
-  return pBnode;
-}
-
-static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) {
-  if (pBnode == NULL) return;
-
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release bnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadBnodeFile(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  int32_t     code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_BNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_BNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_BNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_BNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_BNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_BNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteBnodeFile(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartBnodeWorker(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) {
-    dError("failed to start bnode write worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopBnodeWorker(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->writeWorker);
-}
-
-static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  SBnode     *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode != NULL) {
-    dndReleaseBnode(pDnode, pBnode);
-    terrno = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED;
-    dError("failed to create bnode since %s", terrstr());
-    return -1;
-  }
-
-  SBnodeOpt option = {0};
-  dndBuildBnodeOption(pDnode, &option);
-
-  pBnode = bndOpen(pDnode->dir.bnode, &option);
-  if (pBnode == NULL) {
-    dError("failed to open bnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartBnodeWorker(pDnode) != 0) {
-    dError("failed to start bnode worker since %s", terrstr());
-    bndClose(pBnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteBnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write bnode file since %s", terrstr());
-    dndStopBnodeWorker(pDnode);
-    bndClose(pBnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pBnode = pBnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("bnode open successfully");
-  return 0;
-}
-
-static int32_t dndDropBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode == NULL) {
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteBnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseBnode(pDnode, pBnode);
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseBnode(pDnode, pBnode);
-  dndStopBnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteBnodeFile(pDnode);
-  bndClose(pBnode);
-  pMgmt->pBnode = NULL;
-  bndDestroy(pDnode->dir.bnode);
-
-  return 0;
-}
-
-int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateBnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION;
-    dError("failed to create bnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenBnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropBnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION;
-    dError("failed to drop bnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropBnode(pDnode);
-  }
-}
-
-static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) {
-  SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-  rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) {
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    SRpcMsg *pMsg = NULL;
-    taosGetQitem(qall, (void **)&pMsg);
-    dndSendBnodeErrorRsp(pMsg, code);
-  }
-}
-
-static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode == NULL) {
-    dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
-    return;
-  }
-
-  SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
-  if (pArray == NULL) {
-    dndReleaseBnode(pDnode, pBnode);
-    dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
-    return;
-  }
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    SRpcMsg *pMsg = NULL;
-    taosGetQitem(qall, (void **)&pMsg);
-    void *ptr = taosArrayPush(pArray, &pMsg);
-    if (ptr == NULL) {
-      dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
-    }
-  }
-
-  bndProcessWMsgs(pBnode, pArray);
-
-  for (size_t i = 0; i < numOfMsgs; i++) {
-    SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-  taosArrayDestroy(pArray);
-  dndReleaseBnode(pDnode, pBnode);
-}
-
-static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
-
-  SBnode *pBnode = dndAcquireBnode(pDnode);
-  if (pBnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseBnode(pDnode, pBnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg);
-}
-
-int32_t dndInitBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadBnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("bnode has been deployed and needs to be deleted");
-    bndDestroy(pDnode->dir.bnode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenBnode(pDnode);
-}
-
-void dndCleanupBnode(SDnode *pDnode) {
-  SBnodeMgmt *pMgmt = &pDnode->bmgmt;
-  if (pMgmt->pBnode) {
-    dndStopBnodeWorker(pDnode);
-    bndClose(pMgmt->pBnode);
-    pMgmt->pBnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndEnv.c

@@ -1,334 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndBnode.h"
-#include "dndMgmt.h"
-#include "dndMnode.h"
-#include "dndQnode.h"
-#include "dndSnode.h"
-#include "dndTransport.h"
-#include "dndVnodes.h"
-#include "monitor.h"
-#include "sync.h"
-#include "tfs.h"
-#include "wal.h"
-
-static int8_t once = DND_ENV_INIT;
-
-EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
-
-void dndSetStat(SDnode *pDnode, EStat stat) {
-  dDebug("dnode status set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat));
-  pDnode->stat = stat;
-}
-
-const char *dndStatStr(EStat stat) {
-  switch (stat) {
-    case DND_STAT_INIT:
-      return "init";
-    case DND_STAT_RUNNING:
-      return "running";
-    case DND_STAT_STOPPED:
-      return "stopped";
-    default:
-      return "unknown";
-  }
-}
-
-void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
-  SStartupReq *pStartup = &pDnode->startup;
-  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
-  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
-  pStartup->finished = 0;
-}
-
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
-  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
-  pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
-}
-
-static TdFilePtr dndCheckRunning(char *dataDir) {
-  char filepath[PATH_MAX] = {0};
-  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
-
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    return NULL;
-  }
-
-  int32_t ret = taosLockFile(pFile);
-  if (ret != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
-    taosCloseFile(&pFile);
-    return NULL;
-  }
-
-  return pFile;
-}
-
-static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) {
-  pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
-  if (pDnode->pLockFile == NULL) {
-    return -1;
-  }
-
-  char path[PATH_MAX + 100];
-  snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.mnode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.vnodes = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.dnode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.snode = tstrdup(path);
-  snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP);
-  pDnode->dir.bnode = tstrdup(path);
-
-  if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL ||
-      pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) {
-    dError("failed to malloc dir object");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.dnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.dnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.mnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.mnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.vnodes) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.vnodes, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.snode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.snode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosMkDir(pDnode->dir.bnode) != 0) {
-    dError("failed to create dir:%s since %s", pDnode->dir.bnode, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg));
-  return 0;
-}
-
-static void dndCloseDir(SDnode *pDnode) {
-  tfree(pDnode->dir.mnode);
-  tfree(pDnode->dir.vnodes);
-  tfree(pDnode->dir.dnode);
-  tfree(pDnode->dir.snode);
-  tfree(pDnode->dir.bnode);
-
-  if (pDnode->pLockFile != NULL) {
-    taosUnLockFile(pDnode->pLockFile);
-    taosCloseFile(&pDnode->pLockFile);
-    pDnode->pLockFile = NULL;
-  }
-}
-
-SDnode *dndCreate(SDnodeObjCfg *pCfg) {
-  dInfo("start to create dnode object");
-
-  SDnode *pDnode = calloc(1, sizeof(SDnode));
-  if (pDnode == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    dError("failed to create dnode object since %s", terrstr());
-    return NULL;
-  }
-
-  dndSetStat(pDnode, DND_STAT_INIT);
-
-  if (dndInitDir(pDnode, pCfg) != 0) {
-    dError("failed to init dnode dir since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  SDiskCfg dCfg = {0};
-  tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN);
-  dCfg.level = 0;
-  dCfg.primary = 1;
-  SDiskCfg *pDisks = pDnode->cfg.pDisks;
-  int32_t   numOfDisks = pDnode->cfg.numOfDisks;
-  if (numOfDisks <= 0 || pDisks == NULL) {
-    pDisks = &dCfg;
-    numOfDisks = 1;
-  }
-
-  pDnode->pTfs = tfsOpen(pDisks, numOfDisks);
-  if (pDnode->pTfs == NULL) {
-    dError("failed to init tfs since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitMgmt(pDnode) != 0) {
-    dError("failed to init mgmt since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitVnodes(pDnode) != 0) {
-    dError("failed to init vnodes since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitQnode(pDnode) != 0) {
-    dError("failed to init qnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitSnode(pDnode) != 0) {
-    dError("failed to init snode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitBnode(pDnode) != 0) {
-    dError("failed to init bnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitMnode(pDnode) != 0) {
-    dError("failed to init mnode since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  if (dndInitTrans(pDnode) != 0) {
-    dError("failed to init transport since %s", terrstr());
-    dndClose(pDnode);
-    return NULL;
-  }
-
-  dndSetStat(pDnode, DND_STAT_RUNNING);
-  dndSendStatusReq(pDnode);
-  dndReportStartup(pDnode, "TDengine", "initialized successfully");
-  dInfo("dnode object is created, data:%p", pDnode);
-
-  return pDnode;
-}
-
-void dndClose(SDnode *pDnode) {
-  if (pDnode == NULL) return;
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    dError("dnode is shutting down, data:%p", pDnode);
-    return;
-  }
-
-  dInfo("start to close dnode, data:%p", pDnode);
-  dndSetStat(pDnode, DND_STAT_STOPPED);
-  dndCleanupTrans(pDnode);
-  dndStopMgmt(pDnode);
-  dndCleanupMnode(pDnode);
-  dndCleanupBnode(pDnode);
-  dndCleanupSnode(pDnode);
-  dndCleanupQnode(pDnode);
-  dndCleanupVnodes(pDnode);
-  dndCleanupMgmt(pDnode);
-  tfsClose(pDnode->pTfs);
-
-  dndCloseDir(pDnode);
-  free(pDnode);
-  dInfo("dnode object is closed, data:%p", pDnode);
-}
-
-int32_t dndInit() {
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
-    terrno = TSDB_CODE_REPEAT_INIT;
-    dError("failed to init dnode env since %s", terrstr());
-    return -1;
-  }
-
-  taosIgnSIGPIPE();
-  taosBlockSIGPIPE();
-  taosResolveCRC();
-
-  if (rpcInit() != 0) {
-    dError("failed to init rpc since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  if (walInit() != 0) {
-    dError("failed to init wal since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  SVnodeOpt vnodeOpt = {
-      .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode};
-
-  if (vnodeInit(&vnodeOpt) != 0) {
-    dError("failed to init vnode since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp};
-  if (monInit(&monCfg) != 0) {
-    dError("failed to init monitor since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
-  dInfo("dnode env is initialized");
-  return 0;
-}
-
-void dndCleanup() {
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
-    dError("dnode env is already cleaned up");
-    return;
-  }
-
-  walCleanUp();
-  vnodeCleanup();
-  rpcCleanup();
-  monCleanup();
-
-  taosStopCacheRefreshWorker();
-  dInfo("dnode env is cleaned up");
-}
-
-int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
-  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
-  pInfo->logdir.size = tsLogSpace.size;
-  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
-  pInfo->tempdir.size = tsTempSpace.size;
-
-  return tfsGetMonitorInfo(pDnode->pTfs, pInfo);
-}

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -1,763 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndMgmt.h"
-#include "dndBnode.h"
-#include "dndMnode.h"
-#include "dndQnode.h"
-#include "dndSnode.h"
-#include "dndTransport.h"
-#include "dndVnodes.h"
-#include "dndWorker.h"
-#include "monitor.h"
-
-static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static int32_t dndReadDnodes(SDnode *pDnode);
-static int32_t dndWriteDnodes(SDnode *pDnode);
-static void   *dnodeThreadRoutine(void *param);
-
-static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq);
-static void    dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp);
-static void    dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp);
-static void    dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp);
-
-int32_t dndGetDnodeId(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t dnodeId = pMgmt->dnodeId;
-  taosRUnLockLatch(&pMgmt->latch);
-  return dnodeId;
-}
-
-int64_t dndGetClusterId(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int64_t clusterId = pMgmt->clusterId;
-  taosRUnLockLatch(&pMgmt->latch);
-  return clusterId;
-}
-
-void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-
-  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pDnodeEp != NULL) {
-    if (pPort != NULL) {
-      *pPort = pDnodeEp->ep.port;
-    }
-    if (pFqdn != NULL) {
-      tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
-    }
-    if (pEp != NULL) {
-      snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    }
-  }
-
-  taosRUnLockLatch(&pMgmt->latch);
-}
-
-void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  *pEpSet = pMgmt->mnodeEpSet;
-  taosRUnLockLatch(&pMgmt->latch);
-}
-
-void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) {
-  tmsg_t msgType = pReq->msgType;
-
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-
-  dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse);
-  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
-    if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) {
-      epSet.inUse = (i + 1) % epSet.numOfEps;
-    }
-
-    epSet.eps[i].port = htons(epSet.eps[i].port);
-  }
-
-  rpcSendRedirectRsp(pReq->handle, &epSet);
-}
-
-static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
-  dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  pMgmt->mnodeEpSet = *pEpSet;
-  for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
-    dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-}
-
-static void dndPrintDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  dDebug("print dnode ep list, num:%d", numOfEps);
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i);
-    dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
-  }
-}
-
-static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pMgmt->pDnodeEps != pDnodeEps) {
-    SArray *tmp = pMgmt->pDnodeEps;
-    pMgmt->pDnodeEps = taosArrayDup(pDnodeEps);
-    taosArrayDestroy(tmp);
-  }
-
-  pMgmt->mnodeEpSet.inUse = 0;
-  pMgmt->mnodeEpSet.numOfEps = 0;
-
-  int32_t mIndex = 0;
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps);
-
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
-    if (!pDnodeEp->isMnode) continue;
-    if (mIndex >= TSDB_MAX_REPLICA) continue;
-    pMgmt->mnodeEpSet.numOfEps++;
-
-    pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep;
-    mIndex++;
-  }
-
-  for (int32_t i = 0; i < numOfEps; i++) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
-    taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
-  }
-
-  dndPrintDnodes(pDnode);
-}
-
-static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) {
-  bool changed = false;
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-
-  SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t));
-  if (pDnodeEp != NULL) {
-    char epstr[TSDB_EP_LEN + 1];
-    snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    changed = strcmp(pEp, epstr) != 0;
-  }
-
-  taosRUnLockLatch(&pMgmt->latch);
-  return changed;
-}
-
-static int32_t dndReadDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp));
-  if (pMgmt->pDnodeEps == NULL) {
-    dError("failed to calloc dnodeEp array since %s", strerror(errno));
-    goto PRASE_DNODE_OVER;
-  }
-
-  int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR;
-  int32_t len = 0;
-  int32_t maxLen = 256 * 1024;
-  char   *content = calloc(1, maxLen + 1);
-  cJSON  *root = NULL;
-
-  // fp = fopen(pMgmt->file, "r");
-  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", pMgmt->file);
-    code = 0;
-    goto PRASE_DNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
-  if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read %s since dnodeId not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->dnodeId = dnodeId->valueint;
-
-  cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read %s since clusterId not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->clusterId = atoll(clusterId->valuestring);
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  cJSON *dnodes = cJSON_GetObjectItem(root, "dnodes");
-  if (!dnodes || dnodes->type != cJSON_Array) {
-    dError("failed to read %s since dnodes not found", pMgmt->file);
-    goto PRASE_DNODE_OVER;
-  }
-
-  int32_t numOfDnodes = cJSON_GetArraySize(dnodes);
-  if (numOfDnodes <= 0) {
-    dError("failed to read %s since numOfDnodes:%d invalid", pMgmt->file, numOfDnodes);
-    goto PRASE_DNODE_OVER;
-  }
-
-  for (int32_t i = 0; i < numOfDnodes; ++i) {
-    cJSON *node = cJSON_GetArrayItem(dnodes, i);
-    if (node == NULL) break;
-
-    SDnodeEp dnodeEp = {0};
-
-    cJSON *did = cJSON_GetObjectItem(node, "id");
-    if (!did || did->type != cJSON_Number) {
-      dError("failed to read %s since dnodeId not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-
-    dnodeEp.id = dnodeId->valueint;
-
-    cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
-    if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
-      dError("failed to read %s since dnodeFqdn not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-    tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
-
-    cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
-    if (!dnodePort || dnodePort->type != cJSON_Number) {
-      dError("failed to read %s since dnodePort not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-
-    dnodeEp.ep.port = dnodePort->valueint;
-
-    cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
-    if (!isMnode || isMnode->type != cJSON_Number) {
-      dError("failed to read %s since isMnode not found", pMgmt->file);
-      goto PRASE_DNODE_OVER;
-    }
-    dnodeEp.isMnode = isMnode->valueint;
-
-    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
-  }
-
-  code = 0;
-  dInfo("succcessed to read file %s", pMgmt->file);
-  dndPrintDnodes(pDnode);
-
-PRASE_DNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
-    dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
-    return -1;
-  }
-
-  if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
-    SDnodeEp dnodeEp = {0};
-    dnodeEp.isMnode = 1;
-    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
-    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
-  }
-
-  dndResetDnodes(pDnode, pMgmt->pDnodeEps);
-
-  terrno = 0;
-  return 0;
-}
-
-static int32_t dndWriteDnodes(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  // FILE *fp = fopen(pMgmt->file, "w");
-  TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    dError("failed to write %s since %s", pMgmt->file, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 256 * 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", pMgmt->dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
-
-  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  for (int32_t i = 0; i < numOfEps; ++i) {
-    SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i);
-    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pDnodeEp->id);
-    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
-    len += snprintf(content + len, maxLen - len, "    \"port\": %u,\n", pDnodeEp->ep.port);
-    len += snprintf(content + len, maxLen - len, "    \"isMnode\": %d\n", pDnodeEp->isMnode);
-    if (i < numOfEps - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-  terrno = 0;
-
-  pMgmt->updateTime = taosGetTimestampMs();
-  dDebug("successed to write %s", pMgmt->file);
-  return 0;
-}
-
-void dndSendStatusReq(SDnode *pDnode) {
-  SStatusReq req = {0};
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  req.sver = tsVersion;
-  req.dver = pMgmt->dver;
-  req.dnodeId = pMgmt->dnodeId;
-  req.clusterId = pMgmt->clusterId;
-  req.rebootTime = pMgmt->rebootTime;
-  req.updateTime = pMgmt->updateTime;
-  req.numOfCores = tsNumOfCores;
-  req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes;
-  memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN);
-
-  req.clusterCfg.statusInterval = tsStatusInterval;
-  req.clusterCfg.checkTime = 0;
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN);
-  memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN);
-  memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
-  taosRUnLockLatch(&pMgmt->latch);
-
-  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
-  dndGetVnodeLoads(pDnode, req.pVloads);
-
-  int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
-  void   *pHead = rpcMallocCont(contLen);
-  tSerializeSStatusReq(pHead, contLen, &req);
-  taosArrayDestroy(req.pVloads);
-
-  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
-  pMgmt->statusSent = 1;
-
-  dTrace("pDnode:%p, send status req to mnode", pDnode);
-  dndSendReqToMnode(pDnode, &rpcMsg);
-}
-
-static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  if (pMgmt->dnodeId == 0) {
-    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
-    taosWLockLatch(&pMgmt->latch);
-    pMgmt->dnodeId = pCfg->dnodeId;
-    pMgmt->clusterId = pCfg->clusterId;
-    dndWriteDnodes(pDnode);
-    taosWUnLockLatch(&pMgmt->latch);
-  }
-}
-
-static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) {
-  int32_t numOfEps = taosArrayGetSize(pDnodeEps);
-  if (numOfEps <= 0) return;
-
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
-  if (numOfEps != numOfEpsOld) {
-    dndResetDnodes(pDnode, pDnodeEps);
-    dndWriteDnodes(pDnode);
-  } else {
-    int32_t size = numOfEps * sizeof(SDnodeEp);
-    if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) {
-      dndResetDnodes(pDnode, pDnodeEps);
-      dndWriteDnodes(pDnode);
-    }
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-}
-
-static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pRsp->code != TSDB_CODE_SUCCESS) {
-    if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) {
-      dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId);
-      pMgmt->dropped = 1;
-      dndWriteDnodes(pDnode);
-    }
-  } else {
-    SStatusRsp statusRsp = {0};
-    if (pRsp->pCont != NULL && pRsp->contLen != 0 &&
-        tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
-      pMgmt->dver = statusRsp.dver;
-      dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg);
-      dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps);
-    }
-    taosArrayDestroy(statusRsp.pDnodeEps);
-  }
-
-  pMgmt->statusSent = 0;
-}
-
-static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); }
-
-static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) {
-  dError("grant rsp is received, but not supported yet");
-}
-
-static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dError("config req is received, but not supported yet");
-  SDCfgDnodeReq *pCfg = pReq->pCont;
-  return TSDB_CODE_OPS_NOT_SUPPORT;
-}
-
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dDebug("startup req is received");
-
-  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
-  dndGetStartup(pDnode, pStartup);
-
-  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-
-  SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)};
-  rpcSendResponse(&rpcRsp);
-}
-
-static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
-  pInfo->dnode_id = dndGetDnodeId(pDnode);
-  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
-  pInfo->cluster_id = dndGetClusterId(pDnode);
-  pInfo->protocol = 1;
-}
-
-static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
-  pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f);
-  taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
-  pInfo->cpu_cores = tsNumOfCores;
-  taosGetProcMemory(&pInfo->mem_engine);
-  taosGetSysMemory(&pInfo->mem_system);
-  pInfo->mem_total = tsTotalMemoryKB;
-  pInfo->disk_engine = 0;
-  pInfo->disk_used = tsDataSpace.size.used;
-  pInfo->disk_total = tsDataSpace.size.total;
-  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
-  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
-
-  SVnodesStat *pStat = &pDnode->vmgmt.stat;
-  pInfo->req_select = pStat->numOfSelectReqs;
-  pInfo->req_insert = pStat->numOfInsertReqs;
-  pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
-  pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
-  pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
-  pInfo->errors = tsNumOfErrorLogs;
-  pInfo->vnodes_num = pStat->totalVnodes;
-  pInfo->masters = pStat->masterNum;
-  pInfo->has_mnode = pDnode->mmgmt.deployed;
-}
-
-static void dndSendMonitorReport(SDnode *pDnode) {
-  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
-  dTrace("pDnode:%p, send monitor report to %s:%u", pDnode, tsMonitorFqdn, tsMonitorPort);
-
-  SMonInfo *pMonitor = monCreateMonitorInfo();
-  if (pMonitor == NULL) return;
-
-  SMonBasicInfo basicInfo = {0};
-  dndGetMonitorBasicInfo(pDnode, &basicInfo);
-  monSetBasicInfo(pMonitor, &basicInfo);
-
-  SMonClusterInfo clusterInfo = {0};
-  SMonVgroupInfo  vgroupInfo = {0};
-  SMonGrantInfo   grantInfo = {0};
-  if (dndGetMnodeMonitorInfo(pDnode, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
-    monSetClusterInfo(pMonitor, &clusterInfo);
-    monSetVgroupInfo(pMonitor, &vgroupInfo);
-    monSetGrantInfo(pMonitor, &grantInfo);
-  }
-
-  SMonDnodeInfo dnodeInfo = {0};
-  dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
-  monSetDnodeInfo(pMonitor, &dnodeInfo);
-
-  SMonDiskInfo diskInfo = {0};
-  if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
-    monSetDiskInfo(pMonitor, &diskInfo);
-  }
-
-  taosArrayDestroy(clusterInfo.dnodes);
-  taosArrayDestroy(clusterInfo.mnodes);
-  taosArrayDestroy(vgroupInfo.vgroups);
-  taosArrayDestroy(diskInfo.datadirs);
-
-  monSendReport(pMonitor);
-  monCleanupMonitorInfo(pMonitor);
-}
-
-static void *dnodeThreadRoutine(void *param) {
-  SDnode     *pDnode = param;
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  int64_t     lastStatusTime = taosGetTimestampMs();
-  int64_t     lastMonitorTime = lastStatusTime;
-
-  setThreadName("dnode-hb");
-
-  while (true) {
-    pthread_testcancel();
-    taosMsleep(200);
-    if (dndGetStat(pDnode) != DND_STAT_RUNNING || pMgmt->dropped) {
-      continue;
-    }
-
-    int64_t curTime = taosGetTimestampMs();
-
-    float statusInterval = (curTime - lastStatusTime) / 1000.0f;
-    if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
-      dndSendStatusReq(pDnode);
-      lastStatusTime = curTime;
-    }
-
-    float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
-    if (monitorInterval >= tsMonitorInterval) {
-      dndSendMonitorReport(pDnode);
-      lastMonitorTime = curTime;
-    }
-  }
-}
-
-int32_t dndInitMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  pMgmt->dnodeId = 0;
-  pMgmt->rebootTime = taosGetTimestampMs();
-  pMgmt->dropped = 0;
-  pMgmt->clusterId = 0;
-  taosInitRWLatch(&pMgmt->latch);
-
-  char path[PATH_MAX];
-  snprintf(path, PATH_MAX, "%s/dnode.json", pDnode->dir.dnode);
-  pMgmt->file = strdup(path);
-  if (pMgmt->file == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pMgmt->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  if (pMgmt->dnodeHash == NULL) {
-    dError("failed to init dnode hash");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (dndReadDnodes(pDnode) != 0) {
-    dError("failed to read file:%s since %s", pMgmt->file, terrstr());
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dError("dnode not start since its already dropped");
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) {
-    dError("failed to start dnode mgmt worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) {
-    dError("failed to start dnode mgmt worker since %s", terrstr());
-    return -1;
-  }
-
-  pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode);
-  if (pMgmt->threadId == NULL) {
-    dError("failed to init dnode thread");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  dInfo("dnode-mgmt is initialized");
-  return 0;
-}
-
-void dndStopMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  dndCleanupWorker(&pMgmt->mgmtWorker);
-  dndCleanupWorker(&pMgmt->statusWorker);
-
-  if (pMgmt->threadId != NULL) {
-    taosDestoryThread(pMgmt->threadId);
-    pMgmt->threadId = NULL;
-  }
-}
-
-void dndCleanupMgmt(SDnode *pDnode) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-  taosWLockLatch(&pMgmt->latch);
-
-  if (pMgmt->pDnodeEps != NULL) {
-    taosArrayDestroy(pMgmt->pDnodeEps);
-    pMgmt->pDnodeEps = NULL;
-  }
-
-  if (pMgmt->dnodeHash != NULL) {
-    taosHashCleanup(pMgmt->dnodeHash);
-    pMgmt->dnodeHash = NULL;
-  }
-
-  if (pMgmt->file != NULL) {
-    free(pMgmt->file);
-    pMgmt->file = NULL;
-  }
-
-  taosWUnLockLatch(&pMgmt->latch);
-  dInfo("dnode-mgmt is cleaned up");
-}
-
-void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
-
-  if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) {
-    dndUpdateMnodeEpSet(pDnode, pEpSet);
-  }
-
-  SDnodeWorker *pWorker = &pMgmt->mgmtWorker;
-  if (pMsg->msgType == TDMT_MND_STATUS_RSP) {
-    pWorker = &pMgmt->statusWorker;
-  }
-
-  if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-}
-
-static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = 0;
-
-  switch (pMsg->msgType) {
-    case TDMT_DND_CREATE_MNODE:
-      code = dndProcessCreateMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_ALTER_MNODE:
-      code = dndProcessAlterMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_MNODE:
-      code = dndProcessDropMnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_QNODE:
-      code = dndProcessCreateQnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_QNODE:
-      code = dndProcessDropQnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_SNODE:
-      code = dndProcessCreateSnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_SNODE:
-      code = dndProcessDropSnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_BNODE:
-      code = dndProcessCreateBnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_BNODE:
-      code = dndProcessDropBnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_CONFIG_DNODE:
-      code = dndProcessConfigDnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_MND_STATUS_RSP:
-      dndProcessStatusRsp(pDnode, pMsg);
-      break;
-    case TDMT_MND_AUTH_RSP:
-      dndProcessAuthRsp(pDnode, pMsg);
-      break;
-    case TDMT_MND_GRANT_RSP:
-      dndProcessGrantRsp(pDnode, pMsg);
-      break;
-    case TDMT_DND_CREATE_VNODE:
-      code = dndProcessCreateVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_ALTER_VNODE:
-      code = dndProcessAlterVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_DROP_VNODE:
-      code = dndProcessDropVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_SYNC_VNODE:
-      code = dndProcessSyncVnodeReq(pDnode, pMsg);
-      break;
-    case TDMT_DND_COMPACT_VNODE:
-      code = dndProcessCompactVnodeReq(pDnode, pMsg);
-      break;
-    default:
-      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-      code = -1;
-      dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType));
-      break;
-  }
-
-  if (pMsg->msgType & 1u) {
-    if (code != 0) code = terrno;
-    SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle};
-    rpcSendResponse(&rsp);
-  }
-
-  rpcFreeCont(pMsg->pCont);
-  pMsg->pCont = NULL;
-  taosFreeQitem(pMsg);
-}

source/dnode/mgmt/impl/src/dndMnode.c

@@ -1,642 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndMnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg);
-static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg);
-
-static SMnode *dndAcquireMnode(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  SMnode     *pMnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pMnode = pMgmt->pMnode;
-  } else {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pMnode != NULL) {
-    dTrace("acquire mnode, refCount:%d", refCount);
-  }
-  return pMnode;
-}
-
-static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) {
-  if (pMnode == NULL) return;
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release mnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadMnodeFile(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  int32_t     code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 4096;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_MNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
-  if (!mnodes || mnodes->type != cJSON_Array) {
-    dError("failed to read %s since nodes not found", file);
-    goto PRASE_MNODE_OVER;
-  }
-
-  pMgmt->replica = cJSON_GetArraySize(mnodes);
-  if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
-    dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
-    goto PRASE_MNODE_OVER;
-  }
-
-  for (int32_t i = 0; i < pMgmt->replica; ++i) {
-    cJSON *node = cJSON_GetArrayItem(mnodes, i);
-    if (node == NULL) break;
-
-    SReplica *pReplica = &pMgmt->replicas[i];
-
-    cJSON *id = cJSON_GetObjectItem(node, "id");
-    if (!id || id->type != cJSON_Number) {
-      dError("failed to read %s since id not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    pReplica->id = id->valueint;
-
-    cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
-    if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
-      dError("failed to read %s since fqdn not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
-
-    cJSON *port = cJSON_GetObjectItem(node, "port");
-    if (!port || port->type != cJSON_Number) {
-      dError("failed to read %s since port not found", file);
-      goto PRASE_MNODE_OVER;
-    }
-    pReplica->port = port->valueint;
-  }
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_MNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteMnodeFile(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 4096;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"mnodes\": [{\n");
-  for (int32_t i = 0; i < pMgmt->replica; ++i) {
-    SReplica *pReplica = &pMgmt->replicas[i];
-    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pReplica->id);
-    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pReplica->fqdn);
-    len += snprintf(content + len, maxLen - len, "    \"port\": %u\n", pReplica->port);
-    if (i < pMgmt->replica - 1) {
-      len += snprintf(content + len, maxLen - len, "  },{\n");
-    } else {
-      len += snprintf(content + len, maxLen - len, "  }]\n");
-    }
-  }
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartMnodeWorker(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode read worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode write worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) {
-    dError("failed to start mnode sync worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopMnodeWorker(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 1) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->readWorker);
-  dndCleanupWorker(&pMgmt->writeWorker);
-  dndCleanupWorker(&pMgmt->syncWorker);
-}
-
-static bool dndNeedDeployMnode(SDnode *pDnode) {
-  if (dndGetDnodeId(pDnode) > 0) {
-    return false;
-  }
-
-  if (dndGetClusterId(pDnode) > 0) {
-    return false;
-  }
-
-  if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) {
-    return false;
-  }
-
-  return true;
-}
-
-static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg);
-  return 0;
-}
-
-static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg);
-  return 0;
-}
-
-static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ;
-  pOption->putReqToMReadQFp = dndPutMsgToMReadQ;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-}
-
-static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  dndInitMnodeOption(pDnode, pOption);
-  pOption->replica = 1;
-  pOption->selfIndex = 0;
-  SReplica *pReplica = &pOption->replicas[0];
-  pReplica->id = 1;
-  pReplica->port = pDnode->cfg.serverPort;
-  memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN);
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pMgmt->selfIndex = pOption->selfIndex;
-  pMgmt->replica = pOption->replica;
-  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-}
-
-static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) {
-  dndInitMnodeOption(pDnode, pOption);
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pOption->selfIndex = pMgmt->selfIndex;
-  pOption->replica = pMgmt->replica;
-  memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-}
-
-static int32_t dndBuildMnodeOptionFromReq(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
-  dndInitMnodeOption(pDnode, pOption);
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-
-  pOption->replica = pCreate->replica;
-  pOption->selfIndex = -1;
-  for (int32_t i = 0; i < pCreate->replica; ++i) {
-    SReplica *pReplica = &pOption->replicas[i];
-    pReplica->id = pCreate->replicas[i].id;
-    pReplica->port = pCreate->replicas[i].port;
-    memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-    if (pReplica->id == pOption->dnodeId) {
-      pOption->selfIndex = i;
-    }
-  }
-
-  if (pOption->selfIndex == -1) {
-    dError("failed to build mnode options since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  pMgmt->selfIndex = pOption->selfIndex;
-  pMgmt->replica = pOption->replica;
-  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
-  return 0;
-}
-
-static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = mndOpen(pDnode->dir.mnode, pOption);
-  if (pMnode == NULL) {
-    dError("failed to open mnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartMnodeWorker(pDnode) != 0) {
-    dError("failed to start mnode worker since %s", terrstr());
-    mndClose(pMnode);
-    mndDestroy(pDnode->dir.mnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteMnodeFile(pDnode) != 0) {
-    dError("failed to write mnode file since %s", terrstr());
-    pMgmt->deployed = 0;
-    dndStopMnodeWorker(pDnode);
-    mndClose(pMnode);
-    mndDestroy(pDnode->dir.mnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pMnode = pMnode;
-  pMgmt->deployed = 1;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("mnode open successfully");
-  return 0;
-}
-
-static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  if (mndAlter(pMnode, pOption) != 0) {
-    dError("failed to alter mnode since %s", terrstr());
-    dndReleaseMnode(pDnode, pMnode);
-    return -1;
-  }
-
-  dndReleaseMnode(pDnode, pMnode);
-  return 0;
-}
-
-static int32_t dndDropMnode(SDnode *pDnode) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteMnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseMnode(pDnode, pMnode);
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseMnode(pDnode, pMnode);
-  dndStopMnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteMnodeFile(pDnode);
-  mndClose(pMnode);
-  pMgmt->pMnode = NULL;
-  mndDestroy(pDnode->dir.mnode);
-
-  return 0;
-}
-
-
-int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateMnodeReq createReq = {0};
-  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeOpt option = {0};
-  if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    dndReleaseMnode(pDnode, pMnode);
-    terrno = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED;
-    dError("failed to create mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to create mnode");
-  return dndOpenMnode(pDnode, &option);
-}
-
-int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDAlterMnodeReq alterReq = {0};
-  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (alterReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnodeOpt option = {0};
-  if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-    dError("failed to alter mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to alter mnode");
-  int32_t code = dndAlterMnode(pDnode, &option);
-  dndReleaseMnode(pDnode, pMnode);
-
-  return code;
-}
-
-int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropMnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION;
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-    dError("failed to drop mnode since %s", terrstr());
-    return -1;
-  }
-
-  dDebug("start to drop mnode");
-  int32_t code = dndDropMnode(pDnode);
-  dndReleaseMnode(pDnode, pMnode);
-
-  return code;
-}
-
-static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    mndProcessMsg(pMsg);
-    dndReleaseMnode(pDnode, pMnode);
-  } else {
-    mndSendRsp(pMsg, terrno);
-  }
-
-  mndCleanupMsg(pMsg);
-}
-
-static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) {
-  int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode != NULL) {
-    SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
-    if (pMsg == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-    } else {
-      code = dndWriteMsgToWorker(pWorker, pMsg, 0);
-      if (code != 0) code = terrno;
-    }
-
-    if (code != 0) {
-      mndCleanupMsg(pMsg);
-    }
-  }
-  dndReleaseMnode(pDnode, pMnode);
-
-  if (code != 0) {
-    if (pRpcMsg->msgType & 1u) {
-      if (code == TSDB_CODE_DND_MNODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) {
-        dndSendRedirectRsp(pDnode, pRpcMsg);
-      } else {
-        SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code};
-        rpcSendResponse(&rsp);
-      }
-    }
-    rpcFreeCont(pRpcMsg->pCont);
-  }
-}
-
-void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg);
-}
-
-void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg);
-}
-
-void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg);
-}
-
-int32_t dndInitMnode(SDnode *pDnode) {
-  dInfo("dnode-mnode start to init");
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadMnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("mnode has been deployed and needs to be deleted");
-    mndDestroy(pDnode->dir.mnode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) {
-    bool needDeploy = dndNeedDeployMnode(pDnode);
-    if (!needDeploy) {
-      dDebug("mnode does not need to be deployed");
-      return 0;
-    }
-
-    dInfo("start to deploy mnode");
-    SMnodeOpt option = {0};
-    dndBuildMnodeDeployOption(pDnode, &option);
-    return dndOpenMnode(pDnode, &option);
-  } else {
-    dInfo("start to open mnode");
-    SMnodeOpt option = {0};
-    dndBuildMnodeOpenOption(pDnode, &option);
-    return dndOpenMnode(pDnode, &option);
-  }
-}
-
-void dndCleanupMnode(SDnode *pDnode) {
-  dInfo("dnode-mnode start to clean up");
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-  if (pMgmt->pMnode) {
-    dndStopMnodeWorker(pDnode);
-    mndClose(pMgmt->pMnode);
-    pMgmt->pMnode = NULL;
-  }
-  dInfo("dnode-mnode is cleaned up");
-}
-
-int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  SMnodeMgmt *pMgmt = &pDnode->mmgmt;
-
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) {
-    terrno = TSDB_CODE_APP_NOT_READY;
-    dTrace("failed to get user auth since %s", terrstr());
-    return -1;
-  }
-
-  int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
-  dndReleaseMnode(pDnode, pMnode);
-
-  dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
-  return code;
-}
-
-int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                               SMonGrantInfo *pGrantInfo) {
-  SMnode *pMnode = dndAcquireMnode(pDnode);
-  if (pMnode == NULL) return -1;
-
-  int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
-  dndReleaseMnode(pDnode, pMnode);
-  return code;
-}

source/dnode/mgmt/impl/src/dndQnode.c

@@ -1,375 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndQnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static SQnode *dndAcquireQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  SQnode     *pQnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pQnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pQnode = pMgmt->pQnode;
-  } else {
-    terrno = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pQnode != NULL) {
-    dTrace("acquire qnode, refCount:%d", refCount);
-  }
-  return pQnode;
-}
-
-static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) {
-  if (pQnode == NULL) return;
-
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release qnode, refCount:%d", refCount);
-}
-
-static int32_t dndReadQnodeFile(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  int32_t     code = TSDB_CODE_DND_QNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_QNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_QNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_QNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_QNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_QNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_QNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteQnodeFile(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartQnodeWorker(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) {
-    dError("failed to start qnode query worker since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) {
-    dError("failed to start qnode fetch worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopQnodeWorker(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  dndCleanupWorker(&pMgmt->queryWorker);
-  dndCleanupWorker(&pMgmt->fetchWorker);
-}
-
-static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    dndReleaseQnode(pDnode, pQnode);
-    terrno = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED;
-    dError("failed to create qnode since %s", terrstr());
-    return -1;
-  }
-
-  SQnodeOpt option = {0};
-  dndBuildQnodeOption(pDnode, &option);
-
-  pQnode = qndOpen(&option);
-  if (pQnode == NULL) {
-    dError("failed to open qnode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartQnodeWorker(pDnode) != 0) {
-    dError("failed to start qnode worker since %s", terrstr());
-    qndClose(pQnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteQnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write qnode file since %s", terrstr());
-    dndStopQnodeWorker(pDnode);
-    qndClose(pQnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pQnode = pQnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("qnode open successfully");
-  return 0;
-}
-
-static int32_t dndDropQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode == NULL) {
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteQnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseQnode(pDnode, pQnode);
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseQnode(pDnode, pQnode);
-  dndStopQnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteQnodeFile(pDnode);
-  qndClose(pQnode);
-  pMgmt->pQnode = NULL;
-
-  return 0;
-}
-
-int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateQnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION;
-    dError("failed to create qnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenQnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropQnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION;
-    dError("failed to drop qnode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropQnode(pDnode);
-  }
-}
-
-static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  SRpcMsg    *pRsp = NULL;
-  int32_t     code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    code = qndProcessMsg(pQnode, pMsg, &pRsp);
-  }
-  dndReleaseQnode(pDnode, pQnode);
-
-  if (pMsg->msgType & 1u) {
-    if (pRsp != NULL) {
-      pRsp->ahandle = pMsg->ahandle;
-      rpcSendResponse(pRsp);
-      free(pRsp);
-    } else {
-      if (code != 0) code = terrno;
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-    }
-  }
-
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
-
-  SQnode *pQnode = dndAcquireQnode(pDnode);
-  if (pQnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseQnode(pDnode, pQnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
-}
-
-void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
-}
-
-int32_t dndInitQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadQnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) return 0;
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenQnode(pDnode);
-}
-
-void dndCleanupQnode(SDnode *pDnode) {
-  SQnodeMgmt *pMgmt = &pDnode->qmgmt;
-  if (pMgmt->pQnode) {
-    dndStopQnodeWorker(pDnode);
-    qndClose(pMgmt->pQnode);
-    pMgmt->pQnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndSnode.c

@@ -1,505 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndSnode.h"
-#include "dndMgmt.h"
-#include "dndTransport.h"
-#include "dndWorker.h"
-
-typedef struct {
-  int32_t     vgId;
-  int32_t     refCount;
-  int32_t     snVersion;
-  int8_t      dropped;
-  char       *path;
-  SSnode     *pImpl;
-  STaosQueue *pSharedQ;
-  STaosQueue *pUniqueQ;
-} SSnodeObj;
-
-static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg);
-
-static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs);
-
-static SSnode *dndAcquireSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  SSnode     *pSnode = NULL;
-  int32_t     refCount = 0;
-
-  taosRLockLatch(&pMgmt->latch);
-  if (pMgmt->deployed && !pMgmt->dropped && pMgmt->pSnode != NULL) {
-    refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
-    pSnode = pMgmt->pSnode;
-  } else {
-    terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-  }
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (pSnode != NULL) {
-    dTrace("acquire snode, refCount:%d", refCount);
-  }
-  return pSnode;
-}
-
-static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) {
-  if (pSnode == NULL) return;
-
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  taosRLockLatch(&pMgmt->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
-  taosRUnLockLatch(&pMgmt->latch);
-  dTrace("release snode, refCount:%d", refCount);
-}
-
-static int32_t dndReadSnodeFile(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  int32_t     code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR;
-  int32_t     len = 0;
-  int32_t     maxLen = 1024;
-  char       *content = calloc(1, maxLen + 1);
-  cJSON      *root = NULL;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "r");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto PRASE_SNODE_OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto PRASE_SNODE_OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto PRASE_SNODE_OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto PRASE_SNODE_OVER;
-  }
-  pMgmt->deployed = deployed->valueint;
-
-  cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
-  if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", file);
-    goto PRASE_SNODE_OVER;
-  }
-  pMgmt->dropped = dropped->valueint;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
-
-PRASE_SNODE_OVER:
-  if (content != NULL) free(content);
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-static int32_t dndWriteSnodeFile(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  char file[PATH_MAX + 20];
-  snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  // FILE *fp = fopen(file, "w");
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t len = 0;
-  int32_t maxLen = 1024;
-  char   *content = calloc(1, maxLen + 1);
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", pMgmt->deployed);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d\n", pMgmt->dropped);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-  free(content);
-
-  char realfile[PATH_MAX + 20];
-  snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
-  return 0;
-}
-
-static int32_t dndStartSnodeWorker(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *));
-  for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
-    SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker));
-    if (pUniqueWorker == NULL) {
-      return -1;
-    }
-    if (dndInitWorker(pDnode, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, dndProcessSnodeSharedQueue) != 0) {
-      dError("failed to start snode unique worker since %s", terrstr());
-      return -1;
-    }
-    taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker);
-  }
-  if (dndInitWorker(pDnode, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM,
-                    SND_SHARED_THREAD_NUM, dndProcessSnodeSharedQueue)) {
-    dError("failed to start snode shared worker since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-
-static void dndStopSnodeWorker(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->deployed = 0;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  while (pMgmt->refCount > 0) {
-    taosMsleep(10);
-  }
-
-  for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
-    SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i);
-    dndCleanupWorker(worker);
-  }
-  taosArrayDestroy(pMgmt->uniqueWorkers);
-}
-
-static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
-  pOption->pDnode = pDnode;
-  pOption->sendReqToDnodeFp = dndSendReqToDnode;
-  pOption->sendReqToMnodeFp = dndSendReqToMnode;
-  pOption->sendRedirectRspFp = dndSendRedirectRsp;
-  pOption->dnodeId = dndGetDnodeId(pDnode);
-  pOption->clusterId = dndGetClusterId(pDnode);
-  pOption->sver = tsVersion;
-}
-
-static int32_t dndOpenSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  SSnode     *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    dndReleaseSnode(pDnode, pSnode);
-    terrno = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED;
-    dError("failed to create snode since %s", terrstr());
-    return -1;
-  }
-
-  SSnodeOpt option = {0};
-  dndBuildSnodeOption(pDnode, &option);
-
-  pSnode = sndOpen(pDnode->dir.snode, &option);
-  if (pSnode == NULL) {
-    dError("failed to open snode since %s", terrstr());
-    return -1;
-  }
-
-  if (dndStartSnodeWorker(pDnode) != 0) {
-    dError("failed to start snode worker since %s", terrstr());
-    sndClose(pSnode);
-    return -1;
-  }
-
-  pMgmt->deployed = 1;
-  if (dndWriteSnodeFile(pDnode) != 0) {
-    pMgmt->deployed = 0;
-    dError("failed to write snode file since %s", terrstr());
-    dndStopSnodeWorker(pDnode);
-    sndClose(pSnode);
-    return -1;
-  }
-
-  taosWLockLatch(&pMgmt->latch);
-  pMgmt->pSnode = pSnode;
-  taosWUnLockLatch(&pMgmt->latch);
-
-  dInfo("snode open successfully");
-  return 0;
-}
-
-static int32_t dndDropSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode == NULL) {
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pMgmt->latch);
-  pMgmt->dropped = 1;
-  taosRUnLockLatch(&pMgmt->latch);
-
-  if (dndWriteSnodeFile(pDnode) != 0) {
-    taosRLockLatch(&pMgmt->latch);
-    pMgmt->dropped = 0;
-    taosRUnLockLatch(&pMgmt->latch);
-
-    dndReleaseSnode(pDnode, pSnode);
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  }
-
-  dndReleaseSnode(pDnode, pSnode);
-  dndStopSnodeWorker(pDnode);
-  pMgmt->deployed = 0;
-  dndWriteSnodeFile(pDnode);
-  sndClose(pSnode);
-  pMgmt->pSnode = NULL;
-  sndDestroy(pDnode->dir.snode);
-
-  return 0;
-}
-
-int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDCreateSnodeReq createReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (createReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION;
-    dError("failed to create snode since %s", terrstr());
-    return -1;
-  } else {
-    return dndOpenSnode(pDnode);
-  }
-}
-
-int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDDropSnodeReq dropReq = {0};
-  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  if (dropReq.dnodeId != dndGetDnodeId(pDnode)) {
-    terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION;
-    dError("failed to drop snode since %s", terrstr());
-    return -1;
-  } else {
-    return dndDropSnode(pDnode);
-  }
-}
-
-static void dndProcessSnodeUniqueQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) {
-  /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    for (int32_t i = 0; i < numOfMsgs; i++) {
-      SRpcMsg *pMsg = NULL;
-      taosGetQitem(qall, (void **)&pMsg);
-
-      sndProcessUMsg(pSnode, pMsg);
-
-      rpcFreeCont(pMsg->pCont);
-      taosFreeQitem(pMsg);
-    }
-    dndReleaseSnode(pDnode, pSnode);
-  } else {
-    for (int32_t i = 0; i < numOfMsgs; i++) {
-      SRpcMsg *pMsg = NULL;
-      taosGetQitem(qall, (void **)&pMsg);
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-
-      rpcFreeCont(pMsg->pCont);
-      taosFreeQitem(pMsg);
-    }
-  }
-}
-
-static void dndProcessSnodeSharedQueue(SDnode *pDnode, SRpcMsg *pMsg) {
-  /*SSnodeMgmt *pMgmt = &pDnode->smgmt;*/
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    sndProcessSMsg(pSnode, pMsg);
-    dndReleaseSnode(pDnode, pSnode);
-  } else {
-    SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-    rpcSendResponse(&rpcRsp);
-  }
-
-#if 0
-  if (pMsg->msgType & 1u) {
-    if (pRsp != NULL) {
-      pRsp->ahandle = pMsg->ahandle;
-      rpcSendResponse(pRsp);
-      free(pRsp);
-    } else {
-      if (code != 0) code = terrno;
-      SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rpcRsp);
-    }
-  }
-#endif
-
-  rpcFreeCont(pMsg->pCont);
-  taosFreeQitem(pMsg);
-}
-
-static FORCE_INLINE int32_t dndGetSWTypeFromMsg(SRpcMsg *pMsg) {
-  SStreamExecMsgHead *pHead = pMsg->pCont;
-  pHead->workerType = htonl(pHead->workerType);
-  return pHead->workerType;
-}
-
-static FORCE_INLINE int32_t dndGetSWIdFromMsg(SRpcMsg *pMsg) {
-  SMsgHead *pHead = pMsg->pCont;
-  pHead->streamTaskId = htonl(pHead->streamTaskId);
-  return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM;
-}
-
-static void dndWriteSnodeMsgToWorkerByMsg(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    int32_t       index = dndGetSWIdFromMsg(pMsg);
-    SDnodeWorker *pWorker = taosArrayGetP(pDnode->smgmt.uniqueWorkers, index);
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static void dndWriteSnodeMsgToMgmtWorker(SDnode *pDnode, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    SDnodeWorker *pWorker = taosArrayGet(pDnode->smgmt.uniqueWorkers, 0);
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
-  int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
-
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
-  }
-  dndReleaseSnode(pDnode, pSnode);
-
-  if (code != 0) {
-    if (pMsg->msgType & 1u) {
-      SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
-      rpcSendResponse(&rsp);
-    }
-    rpcFreeCont(pMsg->pCont);
-  }
-}
-
-void dndProcessSnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToMgmtWorker(pDnode, pMsg);
-}
-
-void dndProcessSnodeExecMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SSnode *pSnode = dndAcquireSnode(pDnode);
-  if (pSnode != NULL) {
-    int32_t workerType = dndGetSWTypeFromMsg(pMsg);
-    if (workerType == SND_WORKER_TYPE__SHARED) {
-      dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
-    } else {
-      dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
-    }
-  }
-}
-
-void dndProcessSnodeUniqueMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToWorkerByMsg(pDnode, pMsg);
-}
-
-void dndProcessSnodeSharedMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.sharedWorker, pMsg);
-}
-
-int32_t dndInitSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  taosInitRWLatch(&pMgmt->latch);
-
-  if (dndReadSnodeFile(pDnode) != 0) {
-    return -1;
-  }
-
-  if (pMgmt->dropped) {
-    dInfo("snode has been deployed and needs to be deleted");
-    sndDestroy(pDnode->dir.snode);
-    return 0;
-  }
-
-  if (!pMgmt->deployed) return 0;
-
-  return dndOpenSnode(pDnode);
-}
-
-void dndCleanupSnode(SDnode *pDnode) {
-  SSnodeMgmt *pMgmt = &pDnode->smgmt;
-  if (pMgmt->pSnode) {
-    dndStopSnodeWorker(pDnode);
-    sndClose(pMgmt->pSnode);
-    pMgmt->pSnode = NULL;
-  }
-}

source/dnode/mgmt/impl/src/dndTransport.c

@@ -1,422 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-/* this file is mainly responsible for the communication between DNODEs. Each
- * dnode works as both server and client. Dnode may send status, grant, config
- * messages to mnode, mnode may send create/alter/drop table/vnode messages
- * to dnode. All theses messages are handled from here
- */
-
-#define _DEFAULT_SOURCE
-#include "dndTransport.h"
-#include "dndMgmt.h"
-#include "dndMnode.h"
-#include "dndSnode.h"
-#include "dndVnodes.h"
-
-#define INTERNAL_USER   "_dnd"
-#define INTERNAL_CKEY   "_key"
-#define INTERNAL_SECRET "_pwd"
-
-static void dndInitMsgFp(STransMgmt *pMgmt) {
-  // Requests handled by DNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_DND_NETWORK_TEST)] = dndProcessMgmtMsg;
-
-  // Requests handled by MNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONNECT)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_ACCT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_USER)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CONFIG_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH_RSP)] = dndProcessMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg;
-
-  // Requests handled by VNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_DISCONNECT)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_RES_READY)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TASKS_STATUS)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CANCEL_TASK)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TASK)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_STB_RSP)] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg;
-
-  // Requests handled by SNODE
-  pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_DEPLOY)] = dndProcessSnodeMgmtMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_SND_TASK_EXEC)] = dndProcessSnodeExecMsg;
-}
-
-static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
-  SDnode     *pDnode = parent;
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  tmsg_t msgType = pRsp->msgType;
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    if (pRsp == NULL || pRsp->pCont == NULL) return;
-    dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
-    rpcFreeCont(pRsp->pCont);
-    return;
-  }
-
-  DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
-  if (fp != NULL) {
-    dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF,
-           pRsp->ahandle);
-    (*fp)(pDnode, pRsp, pEpSet);
-  } else {
-    dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
-    rpcFreeCont(pRsp->pCont);
-  }
-}
-
-static int32_t dndInitClient(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.label = "D-C";
-  rpcInit.numOfThreads = 1;
-  rpcInit.cfp = dndProcessResponse;
-  rpcInit.sessions = 1024;
-  rpcInit.connType = TAOS_CONN_CLIENT;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.user = INTERNAL_USER;
-  rpcInit.ckey = INTERNAL_CKEY;
-  rpcInit.spi = 1;
-  rpcInit.parent = pDnode;
-
-  char pass[TSDB_PASSWORD_LEN + 1] = {0};
-  taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
-  rpcInit.secret = pass;
-
-  pMgmt->clientRpc = rpcOpen(&rpcInit);
-  if (pMgmt->clientRpc == NULL) {
-    dError("failed to init dnode rpc client");
-    return -1;
-  }
-
-  dDebug("dnode rpc client is initialized");
-  return 0;
-}
-
-static void dndCleanupClient(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->clientRpc) {
-    rpcClose(pMgmt->clientRpc);
-    pMgmt->clientRpc = NULL;
-    dDebug("dnode rpc client is closed");
-  }
-}
-
-static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
-  SDnode     *pDnode = param;
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  tmsg_t msgType = pReq->msgType;
-  if (msgType == TDMT_DND_NETWORK_TEST) {
-    dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle);
-    dndProcessStartupReq(pDnode, pReq);
-    return;
-  }
-
-  if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
-    dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-    return;
-  } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) {
-    dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-    return;
-  }
-
-  if (pReq->pCont == NULL) {
-    dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    return;
-  }
-
-  DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
-  if (fp != NULL) {
-    dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    (*fp)(pDnode, pReq, pEpSet);
-  } else {
-    dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
-    SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
-    rpcSendResponse(&rspMsg);
-    rpcFreeCont(pReq->pCont);
-  }
-}
-
-static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-  rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
-}
-
-static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  if (strcmp(user, INTERNAL_USER) == 0) {
-    char pass[TSDB_PASSWORD_LEN + 1] = {0};
-    taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
-    memcpy(secret, pass, TSDB_PASSWORD_LEN);
-    *spi = 1;
-    *encrypt = 0;
-    *ckey = 0;
-    return 0;
-  } else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
-    char pass[TSDB_PASSWORD_LEN + 1] = {0};
-    taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
-    memcpy(secret, pass, TSDB_PASSWORD_LEN);
-    *spi = 1;
-    *encrypt = 0;
-    *ckey = 0;
-    return 0;
-  } else {
-    return -1;
-  }
-}
-
-static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  SDnode *pDnode = parent;
-
-  if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) {
-    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
-    return 0;
-  }
-
-  if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) {
-    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
-    return 0;
-  }
-
-  if (terrno != TSDB_CODE_APP_NOT_READY) {
-    dTrace("failed to get user auth from mnode since %s", terrstr());
-    return -1;
-  }
-
-  SAuthReq authReq = {0};
-  tstrncpy(authReq.user, user, TSDB_USER_LEN);
-  int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void   *pReq = rpcMallocCont(contLen);
-  tSerializeSAuthReq(pReq, contLen, &authReq);
-
-  SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
-  SRpcMsg rpcRsp = {0};
-  dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
-  dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
-
-  if (rpcRsp.code != 0) {
-    terrno = rpcRsp.code;
-    dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
-  } else {
-    SAuthRsp authRsp = {0};
-    tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
-    memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
-    memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
-    *spi = authRsp.spi;
-    *encrypt = authRsp.encrypt;
-    dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
-           authRsp.encrypt);
-  }
-
-  rpcFreeCont(rpcRsp.pCont);
-  return rpcRsp.code;
-}
-
-static int32_t dndInitServer(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  dndInitMsgFp(pMgmt);
-
-  int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
-  if (numOfThreads < 1) {
-    numOfThreads = 1;
-  }
-
-  SRpcInit rpcInit;
-  memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.localPort = pDnode->cfg.serverPort;
-  rpcInit.label = "D-S";
-  rpcInit.numOfThreads = numOfThreads;
-  rpcInit.cfp = dndProcessRequest;
-  rpcInit.sessions = tsMaxShellConns;
-  rpcInit.connType = TAOS_CONN_SERVER;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
-  rpcInit.afp = dndRetrieveUserAuthInfo;
-  rpcInit.parent = pDnode;
-
-  pMgmt->serverRpc = rpcOpen(&rpcInit);
-  if (pMgmt->serverRpc == NULL) {
-    dError("failed to init dnode rpc server");
-    return -1;
-  }
-
-  dDebug("dnode rpc server is initialized");
-  return 0;
-}
-
-static void dndCleanupServer(SDnode *pDnode) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->serverRpc) {
-    rpcClose(pMgmt->serverRpc);
-    pMgmt->serverRpc = NULL;
-    dDebug("dnode rpc server is closed");
-  }
-}
-
-int32_t dndInitTrans(SDnode *pDnode) {
-  if (dndInitClient(pDnode) != 0) {
-    return -1;
-  }
-
-  if (dndInitServer(pDnode) != 0) {
-    return -1;
-  }
-
-  dInfo("dnode-transport is initialized");
-  return 0;
-}
-
-void dndCleanupTrans(SDnode *pDnode) {
-  dInfo("dnode-transport start to clean up");
-  dndCleanupServer(pDnode);
-  dndCleanupClient(pDnode);
-  dInfo("dnode-transport is cleaned up");
-}
-
-int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq) {
-  STransMgmt *pMgmt = &pDnode->tmgmt;
-  if (pMgmt->clientRpc == NULL) {
-    terrno = TSDB_CODE_DND_OFFLINE;
-    return -1;
-  }
-
-  rpcSendRequest(pMgmt->clientRpc, pEpSet, pReq, NULL);
-  return 0;
-}
-
-int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pReq) {
-  SEpSet epSet = {0};
-  dndGetMnodeEpSet(pDnode, &epSet);
-  return dndSendReqToDnode(pDnode, &epSet, pReq);
-}

source/dnode/mgmt/main/CMakeLists.txt

@@ -1,5 +1,5 @@
-aux_source_directory(src DAEMON_SRC)
+aux_source_directory(src EXEC_SRC)
-add_executable(taosd ${DAEMON_SRC})
+add_executable(taosd ${EXEC_SRC})
 
 target_include_directories(
     taosd

source/dnode/mgmt/main/inc/dndMain.h

@@ -1,3 +1,4 @@
+
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -13,40 +14,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_INT_H_
+#ifndef _TD_DND_MAIN_H_
-#define _TD_DND_INT_H_
+#define _TD_DND_MAIN_H_
+
+#include "dnode.h"
+
+#include "taoserror.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tlog.h"
+#include "version.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "os.h"
-
-#include "cJSON.h"
-#include "monitor.h"
-#include "tcache.h"
-#include "tcrc32c.h"
-#include "tdatablock.h"
-#include "tglobal.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tlog.h"
-#include "tmsg.h"
-#include "tqueue.h"
-#include "trpc.h"
-#include "tthread.h"
-#include "ttime.h"
-#include "tworker.h"
-
-#include "dnode.h"
-
-#include "bnode.h"
-#include "mnode.h"
-#include "qnode.h"
-#include "snode.h"
-#include "vnode.h"
-#include "tfs.h"
-
 #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
 #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
 #define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
@@ -54,20 +36,13 @@ extern "C" {
 #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
 #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
 
-typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
+void      dndDumpCfg();
-typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
+void      dndPrintVersion();
-typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat;
+void      dndGenerateGrant();
-typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
+SDnodeOpt dndGetOpt();
-
-EStat       dndGetStat(SDnode *pDnode);
-void        dndSetStat(SDnode *pDnode, EStat stat);
-const char *dndStatStr(EStat stat);
-
-void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_INT_H_*/
+#endif /*_TD_DND_MAIN_H_*/

source/dnode/mgmt/main/src/dndMain.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndMain.h"
+
+static struct {
+  bool    dumpConfig;
+  bool    generateGrant;
+  bool    printAuth;
+  bool    printVersion;
+  char    envFile[PATH_MAX];
+  char    apolloUrl[PATH_MAX];
+  SDnode *pDnode;
+} global = {0};
+
+static void dndSigintHandle(int signum, void *info, void *ctx) {
+  dInfo("signal:%d is received", signum);
+  SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
+  if (pDnode != NULL) {
+    dndHandleEvent(pDnode, DND_EVENT_STOP);
+  }
+}
+
+static void dndSetSignalHandle() {
+  taosSetSignal(SIGTERM, dndSigintHandle);
+  taosSetSignal(SIGHUP, dndSigintHandle);
+  taosSetSignal(SIGINT, dndSigintHandle);
+  taosSetSignal(SIGABRT, dndSigintHandle);
+  taosSetSignal(SIGBREAK, dndSigintHandle);
+}
+
+static int32_t dndParseOption(int32_t argc, char const *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-C") == 0) {
+      global.dumpConfig = true;
+    } else if (strcmp(argv[i], "-k") == 0) {
+      global.generateGrant = true;
+    } else if (strcmp(argv[i], "-V") == 0) {
+      global.printVersion = true;
+    } else {
+    }
+  }
+
+  return 0;
+}
+
+static int32_t dndRunDnode() {
+  if (dndInit() != 0) {
+    dInfo("failed to initialize dnode environment since %s", terrstr());
+    return -1;
+  }
+
+  SDnodeOpt option = dndGetOpt();
+
+  SDnode *pDnode = dndCreate(&option);
+  if (pDnode == NULL) {
+    dError("failed to to create dnode object since %s", terrstr());
+    return -1;
+  } else {
+    global.pDnode = pDnode;
+    dndSetSignalHandle();
+  }
+
+  dInfo("start the TDengine service");
+  int32_t code = dndRun(pDnode);
+  dInfo("start shutting down the TDengine service");
+
+  global.pDnode = NULL;
+  dndClose(pDnode);
+  dndCleanup();
+  taosCloseLog();
+  taosCleanupCfg();
+  return code;
+}
+
+int main(int argc, char const *argv[]) {
+  if (!taosCheckSystemIsSmallEnd()) {
+    dError("failed to start TDengine since on non-small-end machines");
+    return -1;
+  }
+
+  if (dndParseOption(argc, argv) != 0) {
+    return -1;
+  }
+
+  if (global.generateGrant) {
+    dndGenerateGrant();
+    return 0;
+  }
+
+  if (global.printVersion) {
+    dndPrintVersion();
+    return 0;
+  }
+
+  if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
+    dError("failed to start TDengine since read log config error");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
+    dError("failed to start TDengine since read config error");
+    return -1;
+  }
+
+  if (global.dumpConfig) {
+    dndDumpCfg();
+    taosCleanupCfg();
+    taosCloseLog();
+    return 0;
+  }
+
+  return dndRunDnode();
+}

source/dnode/mgmt/main/src/dndUtil.c

@@ -14,15 +14,15 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dmnInt.h"
+#include "dndMain.h"
 
-void dmnGenerateGrant() {
+void dndGenerateGrant() {
 #if 0
   grantParseParameter();
 #endif
 }
 
-void dmnPrintVersion() {
+void dndPrintVersion() {
 #ifdef TD_ENTERPRISE
   char *releaseName = "enterprise";
 #else
@@ -32,3 +32,24 @@ void dmnPrintVersion() {
   printf("gitinfo: %s\n", gitinfo);
   printf("builuInfo: %s\n", buildinfo);
 }
+
+void dndDumpCfg() {
+  SConfig *pCfg = taosGetCfg();
+  cfgDumpCfg(pCfg, 0, 1);
+}
+
+SDnodeOpt dndGetOpt() {
+  SConfig  *pCfg = taosGetCfg();
+  SDnodeOpt option = {0};
+
+  option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
+  tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
+  tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
+  tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
+  option.serverPort = tsServerPort;
+  tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
+  snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
+  option.pDisks = tsDiskCfg;
+  option.numOfDisks = tsDiskCfgNum;
+  return option;
+}

source/dnode/mgmt/mnode/inc/mm.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_MNODE_H_
+#define _TD_DND_MNODE_H_
+
+#include "dnd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void mmGetMgmtFp(SMgmtWrapper *pMgmt);
+
+int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey);
+int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                         SMonGrantInfo *pGrantInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_MNODE_H_*/

source/dnode/mgmt/mnode/inc/mmInt.h

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_MNODE_INT_H_
+#define _TD_DND_MNODE_INT_H_
+
+#include "mm.h"
+#include "mnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SMnodeMgmt {
+  SMnode       *pMnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  readWorker;
+  SDnodeWorker  writeWorker;
+  SDnodeWorker  syncWorker;
+  SReplica      replicas[TSDB_MAX_REPLICA];
+  int8_t        replica;
+  int8_t        selfIndex;
+} SMnodeMgmt;
+
+// mmFile.c
+int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed);
+int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed);
+
+// mmInt.c
+int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq);
+int32_t mmDrop(SMgmtWrapper *pWrapper);
+int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq);
+
+// mmMsg.c
+void    mmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+// mmWorker.c
+int32_t mmStartWorker(SMnodeMgmt *pMgmt);
+void    mmStopWorker(SMnodeMgmt *pMgmt);
+int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
+int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_MNODE_INT_H_*/

source/dnode/mgmt/mnode/src/mmFile.c

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int32_t   len = 0;
+  int32_t   maxLen = 4096;
+  char     *content = calloc(1, maxLen + 1);
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto PRASE_MNODE_OVER;
+  }
+
+  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  content[len] = 0;
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_Number) {
+    dError("failed to read %s since deployed not found", file);
+    goto PRASE_MNODE_OVER;
+  }
+  *pDeployed = deployed->valueint;
+
+  cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
+  if (!mnodes || mnodes->type != cJSON_Array) {
+    dError("failed to read %s since nodes not found", file);
+    goto PRASE_MNODE_OVER;
+  }
+
+  pMgmt->replica = cJSON_GetArraySize(mnodes);
+  if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
+    dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
+    goto PRASE_MNODE_OVER;
+  }
+
+  for (int32_t i = 0; i < pMgmt->replica; ++i) {
+    cJSON *node = cJSON_GetArrayItem(mnodes, i);
+    if (node == NULL) break;
+
+    SReplica *pReplica = &pMgmt->replicas[i];
+
+    cJSON *id = cJSON_GetObjectItem(node, "id");
+    if (!id || id->type != cJSON_Number) {
+      dError("failed to read %s since id not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    pReplica->id = id->valueint;
+
+    cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
+    if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
+      dError("failed to read %s since fqdn not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
+
+    cJSON *port = cJSON_GetObjectItem(node, "port");
+    if (!port || port->type != cJSON_Number) {
+      dError("failed to read %s since port not found", file);
+      goto PRASE_MNODE_OVER;
+    }
+    pReplica->port = port->valueint;
+  }
+
+  code = 0;
+  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+
+PRASE_MNODE_OVER:
+  if (content != NULL) free(content);
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
+  char file[PATH_MAX];
+  snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);;
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+
+  int32_t len = 0;
+  int32_t maxLen = 4096;
+  char   *content = calloc(1, maxLen + 1);
+
+  len += snprintf(content + len, maxLen - len, "{\n");
+  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d,\n", deployed);
+  len += snprintf(content + len, maxLen - len, "  \"mnodes\": [{\n");
+  for (int32_t i = 0; i < pMgmt->replica; ++i) {
+    SReplica *pReplica = &pMgmt->replicas[i];
+    len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pReplica->id);
+    len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pReplica->fqdn);
+    len += snprintf(content + len, maxLen - len, "    \"port\": %u\n", pReplica->port);
+    if (i < pMgmt->replica - 1) {
+      len += snprintf(content + len, maxLen - len, "  },{\n");
+    } else {
+      len += snprintf(content + len, maxLen - len, "  }]\n");
+    }
+  }
+  len += snprintf(content + len, maxLen - len, "}\n");
+
+  taosWriteFile(pFile, content, len);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  free(content);
+
+  char realfile[PATH_MAX];
+  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);;
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s, deployed:%d", realfile, deployed);
+  return 0;
+}

source/dnode/mgmt/mnode/src/mmInt.c

@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+static bool mmDeployRequired(SDnode *pDnode) {
+  if (pDnode->dnodeId > 0) return false;
+  if (pDnode->clusterId > 0) return false;
+  if (strcmp(pDnode->localEp, pDnode->firstEp) != 0) return false;
+  return true;
+}
+
+static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) {
+  SMnodeMgmt mgmt = {0};
+  mgmt.path = pWrapper->path;
+  if (mmReadFile(&mgmt, required) != 0) {
+    return -1;
+  }
+
+  if (!(*required)) {
+    *required = mmDeployRequired(pWrapper->pDnode);
+  }
+
+  return 0;
+}
+
+static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->putToWriteQFp = mmPutMsgToWriteQueue;
+  pOption->putToReadQFp = mmPutMsgToReadQueue;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+
+  mmInitOption(pMgmt, pOption);
+  pOption->replica = 1;
+  pOption->selfIndex = 0;
+  SReplica *pReplica = &pOption->replicas[0];
+  pReplica->id = 1;
+  pReplica->port = pDnode->serverPort;
+  tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN);
+
+  pMgmt->selfIndex = pOption->selfIndex;
+  pMgmt->replica = pOption->replica;
+  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+}
+
+static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
+  mmInitOption(pMgmt, pOption);
+  pOption->selfIndex = pMgmt->selfIndex;
+  pOption->replica = pMgmt->replica;
+  memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+}
+
+static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
+  mmInitOption(pMgmt, pOption);
+
+  pOption->replica = pCreate->replica;
+  pOption->selfIndex = -1;
+  for (int32_t i = 0; i < pCreate->replica; ++i) {
+    SReplica *pReplica = &pOption->replicas[i];
+    pReplica->id = pCreate->replicas[i].id;
+    pReplica->port = pCreate->replicas[i].port;
+    memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
+    if (pReplica->id == pOption->dnodeId) {
+      pOption->selfIndex = i;
+    }
+  }
+
+  if (pOption->selfIndex == -1) {
+    dError("failed to build mnode options since %s", terrstr());
+    return -1;
+  }
+
+  pMgmt->selfIndex = pOption->selfIndex;
+  pMgmt->replica = pOption->replica;
+  memcpy(&pMgmt->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+  return 0;
+}
+
+static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) {
+  SMnodeOpt option = {0};
+  if (pReq != NULL) {
+    if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
+      return -1;
+    }
+  } else {
+    bool deployed = false;
+    if (mmReadFile(pMgmt, &deployed) != 0) {
+      dError("failed to read file since %s", terrstr());
+      return -1;
+    }
+
+    if (!deployed) {
+      dInfo("mnode start to deploy");
+      mmBuildOptionForDeploy(pMgmt, &option);
+    } else {
+      dInfo("mnode start to open");
+      mmBuildOptionForOpen(pMgmt, &option);
+    }
+  }
+
+  pMgmt->pMnode = mndOpen(pMgmt->path, &option);
+  if (pMgmt->pMnode == NULL) {
+    dError("failed to open mnode since %s", terrstr());
+    return -1;
+  }
+
+  if (mmStartWorker(pMgmt) != 0) {
+    dError("failed to start mnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (mmWriteFile(pMgmt, deployed) != 0) {
+    dError("failed to write mnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void mmCloseImp(SMnodeMgmt *pMgmt) {
+  if (pMgmt->pMnode != NULL) {
+    mmStopWorker(pMgmt);
+    mndClose(pMgmt->pMnode);
+    pMgmt->pMnode = NULL;
+  }
+}
+
+int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) {
+  SMnodeOpt option = {0};
+  if (pReq != NULL) {
+    if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
+      return -1;
+    }
+  }
+
+  return mndAlter(pMgmt->pMnode, &option);
+}
+
+int32_t mmDrop(SMgmtWrapper *pWrapper) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("mnode-mgmt start to drop");
+  bool deployed = false;
+  if (mmWriteFile(pMgmt, deployed) != 0) {
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  }
+
+  mmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("mnode-mgmt is dropped");
+  return 0;
+}
+
+static void mmClose(SMgmtWrapper *pWrapper) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("mnode-mgmt start to cleanup");
+  mmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("mnode-mgmt is cleaned up");
+}
+
+int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) {
+  dInfo("mnode-mgmt start to init");
+  if (walInit() != 0) {
+    dError("failed to init wal since %s", terrstr());
+    return -1;
+  }
+
+  SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = mmOpenImp(pMgmt, pReq);
+  if (code != 0) {
+    dError("failed to init mnode-mgmt since %s", terrstr());
+    mmClose(pWrapper);
+  } else {
+    dInfo("mnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+static int32_t mmOpen(SMgmtWrapper *pWrapper) {
+  return mmOpenFromMsg(pWrapper, NULL);
+}
+
+static int32_t mmStart(SMgmtWrapper *pWrapper) {
+  dDebug("mnode mgmt start to run");
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mndStart(pMgmt->pMnode);
+}
+
+void mmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = mmOpen;
+  mgmtFp.closeFp = mmClose;
+  mgmtFp.startFp = mmStart;
+  mgmtFp.createMsgFp = mmProcessCreateReq;
+  mgmtFp.dropMsgFp = mmProcessDropReq;
+  mgmtFp.requiredFp = mmRequire;
+
+  mmInitMsgHandles(pWrapper);
+  pWrapper->name = "mnode";
+  pWrapper->fp = mgmtFp;
+}
+
+int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+
+  int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey);
+  dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
+  return code;
+}
+
+int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                         SMonGrantInfo *pGrantInfo) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
+}

source/dnode/mgmt/mnode/src/mmMsg.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateMnodeReq createReq = {0};
+  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.replica <= 1 || createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmOpenFromMsg(pWrapper, &createReq);
+  }
+}
+
+int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropMnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmDrop(pWrapper);
+  }
+}
+
+int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pMgmt->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDAlterMnodeReq alterReq = {0};
+  if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (alterReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to alter mnode since %s", terrstr());
+    return -1;
+  } else {
+    return mmAlter(pMgmt, &alterReq);
+  }
+}
+
+void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by DNODE
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg);
+
+  // Requests handled by MNODE
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg);
+
+  // Requests handled by VNODE
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg);
+}

source/dnode/mgmt/mnode/src/mmWorker.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mmInt.h"
+
+static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in mnode queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = -1;
+
+  if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) {
+    pMsg->pNode = pMgmt->pMnode;
+    code = mndProcessMsg(pMsg);
+  } else {
+    code = mmProcessAlterReq(pMgmt, pMsg);
+  }
+
+  if (pRpc->msgType & 1U) {
+    if (pRpc->handle == NULL) return;
+    if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+      if (code != 0) code = terrno;
+      SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
+      dndSendRsp(pMgmt->pWrapper, &rsp);
+    }
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode read worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode write worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) {
+    dError("failed to start mnode sync worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void mmStopWorker(SMnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->readWorker);
+  dndCleanupWorker(&pMgmt->writeWorker);
+  dndCleanupWorker(&pMgmt->syncWorker);
+}
+
+static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) {
+  dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg);
+}
+
+int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg);
+}
+
+int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg);
+}
+
+static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) {
+  SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
+  if (pMsg == NULL) {
+    return -1;
+  }
+
+  dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
+  pMsg->rpcMsg = *pRpc;
+
+  int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
+  if (code != 0) {
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+    rpcFreeCont(pRpc->pCont);
+  }
+
+  return code;
+}
+
+int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc);
+}
+
+int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc);
+}

source/dnode/mgmt/qnode/inc/qm.h

@@ -1,4 +1,3 @@
-
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
@@ -14,28 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DMN_INT_H_
+#ifndef _TD_DND_QNODE_H_
-#define _TD_DMN_INT_H_
+#define _TD_DND_QNODE_H_
 
-#include "tconfig.h"
+#include "dnd.h"
-#include "dnode.h"
-#include "taoserror.h"
-#include "tglobal.h"
-#include "tlog.h"
-#include "version.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-SDnodeObjCfg dmnGetObjCfg();
+void qmGetMgmtFp(SMgmtWrapper *pMgmt);
-
-void dmnDumpCfg();
-void dmnPrintVersion();
-void dmnGenerateGrant();
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DMN_INT_H_*/
+#endif /*_TD_DND_QNODE_H_*/

source/dnode/mgmt/qnode/inc/qmInt.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_QNODE_INT_H_
+#define _TD_DND_QNODE_INT_H_
+
+#include "qm.h"
+#include "qnode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SQnodeMgmt {
+  SQnode       *pQnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SDnodeWorker  queryWorker;
+  SDnodeWorker  fetchWorker;
+} SQnodeMgmt;
+
+// qmInt.c
+int32_t qmOpen(SMgmtWrapper *pWrapper);
+int32_t qmDrop(SMgmtWrapper *pWrapper);
+
+// qmMsg.c
+void    qmInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// qmWorker.c
+int32_t qmStartWorker(SQnodeMgmt *pMgmt);
+void    qmStopWorker(SQnodeMgmt *pMgmt);
+int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_QNODE_INT_H_*/

source/dnode/mgmt/qnode/src/qmInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t qmOpenImp(SQnodeMgmt *pMgmt) {
+  SQnodeOpt option = {0};
+  qmInitOption(pMgmt, &option);
+
+  pMgmt->pQnode = qndOpen(&option);
+  if (pMgmt->pQnode == NULL) {
+    dError("failed to open qnode since %s", terrstr());
+    return -1;
+  }
+
+  if (qmStartWorker(pMgmt) != 0) {
+    dError("failed to start qnode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write qnode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void qmCloseImp(SQnodeMgmt *pMgmt) {
+  if (pMgmt->pQnode != NULL) {
+    qmStopWorker(pMgmt);
+    qndClose(pMgmt->pQnode);
+    pMgmt->pQnode = NULL;
+  }
+}
+
+int32_t qmDrop(SMgmtWrapper *pWrapper) {
+  SQnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("qnode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop qnode since %s", terrstr());
+    return -1;
+  }
+
+  qmCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("qnode-mgmt is dropped");
+  return 0;
+}
+
+static void qmClose(SMgmtWrapper *pWrapper) {
+  SQnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("qnode-mgmt start to cleanup");
+  qmCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("qnode-mgmt is cleaned up");
+}
+
+int32_t qmOpen(SMgmtWrapper *pWrapper) {
+  dInfo("qnode-mgmt start to init");
+  SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = qmOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init qnode-mgmt since %s", terrstr());
+    qmClose(pWrapper);
+  } else {
+    dInfo("qnode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void qmGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = qmOpen;
+  mgmtFp.closeFp = qmClose;
+  mgmtFp.createMsgFp = qmProcessCreateReq;
+  mgmtFp.dropMsgFp = qmProcessDropReq;
+  mgmtFp.requiredFp = qmRequire;
+
+  qmInitMsgHandles(pWrapper);
+  pWrapper->name = "qnode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/qnode/src/qmMsg.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateQnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create qnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return qmOpen(pWrapper);
+  }
+}
+
+int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropQnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop qnode since %s", terrstr());
+    return -1;
+  } else {
+    return qmDrop(pWrapper);
+  }
+}
+
+void qmInitMsgHandles(SMgmtWrapper *pWrapper) {}

source/dnode/mgmt/qnode/src/qmWorker.c

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "qmInt.h"
+
+static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in qnode queue", pMsg);
+  SRpcMsg *pRsp = NULL;
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp);
+
+  if (pRpc->msgType & 1u) {
+    if (pRsp != NULL) {
+      pRsp->ahandle = pRpc->ahandle;
+      dndSendRsp(pMgmt->pWrapper, pRsp);
+      free(pRsp);
+    } else {
+      if (code != 0) code = terrno;
+      SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+      dndSendRsp(pMgmt->pWrapper, &rpcRsp);
+    }
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->queryWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->fetchWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
+  if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) {
+    dError("failed to start qnode query worker since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) {
+    dError("failed to start qnode fetch worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void qmStopWorker(SQnodeMgmt *pMgmt) {
+  dndCleanupWorker(&pMgmt->queryWorker);
+  dndCleanupWorker(&pMgmt->fetchWorker);
+}

source/dnode/mgmt/snode/inc/sm.h

@@ -13,21 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_TRANSPORT_H_
+#ifndef _TD_DND_SNODE_H_
-#define _TD_DND_TRANSPORT_H_
+#define _TD_DND_SNODE_H_
+
+#include "dnd.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dndEnv.h"
 
-int32_t dndInitTrans(SDnode *pDnode);
+void smGetMgmtFp(SMgmtWrapper *pWrapper);
-void    dndCleanupTrans(SDnode *pDnode);
-int32_t dndSendReqToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg);
-int32_t dndSendReqToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_TRANSPORT_H_*/
+#endif /*_TD_DND_SNODE_H_*/

source/dnode/mgmt/snode/inc/smInt.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DND_SNODE_INT_H_
+#define _TD_DND_SNODE_INT_H_
+
+#include "sm.h"
+#include "snode.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SSnodeMgmt {
+  SSnode       *pSnode;
+  SDnode       *pDnode;
+  SMgmtWrapper *pWrapper;
+  const char   *path;
+  SRWLatch      latch;
+  int8_t        uniqueWorkerInUse;
+  SArray       *uniqueWorkers;  // SArray<SDnodeWorker*>
+  SDnodeWorker  sharedWorker;
+} SSnodeMgmt;
+
+// smInt.c
+int32_t smOpen(SMgmtWrapper *pWrapper);
+int32_t smDrop(SMgmtWrapper *pWrapper);
+
+// smMsg.c
+void    smInitMsgHandles(SMgmtWrapper *pWrapper);
+int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
+// smWorker.c
+int32_t smStartWorker(SSnodeMgmt *pMgmt);
+void    smStopWorker(SSnodeMgmt *pMgmt);
+int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DND_SNODE_INT_H_*/

source/dnode/mgmt/snode/src/smInt.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
+
+static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
+  SDnode *pDnode = pMgmt->pDnode;
+  pOption->pWrapper = pMgmt->pWrapper;
+  pOption->sendReqFp = dndSendReqToDnode;
+  pOption->sendMnodeReqFp = dndSendReqToMnode;
+  pOption->sendRspFp = dndSendRsp;
+  pOption->dnodeId = pDnode->dnodeId;
+  pOption->clusterId = pDnode->clusterId;
+}
+
+static int32_t smOpenImp(SSnodeMgmt *pMgmt) {
+  SSnodeOpt option = {0};
+  smInitOption(pMgmt, &option);
+
+  pMgmt->pSnode = sndOpen(pMgmt->path, &option);
+  if (pMgmt->pSnode == NULL) {
+    dError("failed to open snode since %s", terrstr());
+    return -1;
+  }
+
+  if (smStartWorker(pMgmt) != 0) {
+    dError("failed to start snode worker since %s", terrstr());
+    return -1;
+  }
+
+  bool deployed = true;
+  if (dndWriteFile(pMgmt->pWrapper, deployed) != 0) {
+    dError("failed to write snode file since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static void smCloseImp(SSnodeMgmt *pMgmt) {
+  if (pMgmt->pSnode != NULL) {
+    smStopWorker(pMgmt);
+    sndClose(pMgmt->pSnode);
+    pMgmt->pSnode = NULL;
+  }
+}
+
+int32_t smDrop(SMgmtWrapper *pWrapper) {
+  SSnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return 0;
+
+  dInfo("snode-mgmt start to drop");
+  bool deployed = false;
+  if (dndWriteFile(pWrapper, deployed) != 0) {
+    dError("failed to drop snode since %s", terrstr());
+    return -1;
+  }
+
+  smCloseImp(pMgmt);
+  taosRemoveDir(pMgmt->path);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("snode-mgmt is dropped");
+  return 0;
+}
+
+static void smClose(SMgmtWrapper *pWrapper) {
+  SSnodeMgmt *pMgmt = pWrapper->pMgmt;
+  if (pMgmt == NULL) return;
+
+  dInfo("snode-mgmt start to cleanup");
+  smCloseImp(pMgmt);
+  pWrapper->pMgmt = NULL;
+  free(pMgmt);
+  dInfo("snode-mgmt is cleaned up");
+}
+
+int32_t smOpen(SMgmtWrapper *pWrapper) {
+  dInfo("snode-mgmt start to init");
+  SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt));
+  if (pMgmt == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pMgmt->path = pWrapper->path;
+  pMgmt->pDnode = pWrapper->pDnode;
+  pMgmt->pWrapper = pWrapper;
+  pWrapper->pMgmt = pMgmt;
+
+  int32_t code = smOpenImp(pMgmt);
+  if (code != 0) {
+    dError("failed to init snode-mgmt since %s", terrstr());
+    smClose(pWrapper);
+  } else {
+    dInfo("snode-mgmt is initialized");
+  }
+
+  return code;
+}
+
+void smGetMgmtFp(SMgmtWrapper *pWrapper) {
+  SMgmtFp mgmtFp = {0};
+  mgmtFp.openFp = smOpen;
+  mgmtFp.closeFp = smClose;
+  mgmtFp.createMsgFp = smProcessCreateReq;
+  mgmtFp.dropMsgFp = smProcessDropReq;
+  mgmtFp.requiredFp = smRequire;
+
+  smInitMsgHandles(pWrapper);
+  pWrapper->name = "snode";
+  pWrapper->fp = mgmtFp;
+}

source/dnode/mgmt/snode/src/smMsg.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDCreateSnodeReq createReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (createReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to create snode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
+    return -1;
+  } else {
+    return smOpen(pWrapper);
+  }
+}
+
+int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnode  *pDnode = pWrapper->pDnode;
+  SRpcMsg *pReq = &pMsg->rpcMsg;
+
+  SDDropSnodeReq dropReq = {0};
+  if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  if (dropReq.dnodeId != pDnode->dnodeId) {
+    terrno = TSDB_CODE_NODE_INVALID_OPTION;
+    dError("failed to drop snode since %s", terrstr());
+    return -1;
+  } else {
+    return smDrop(pWrapper);
+  }
+}
+
+void smInitMsgHandles(SMgmtWrapper *pWrapper) {
+  // Requests handled by SNODE
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg);
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg);
+}

source/dnode/mgmt/snode/src/smWorker.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "smInt.h"
+
+static void smProcessUniqueQueue(SSnodeMgmt *pMgmt, STaosQall *qall, int32_t numOfMsgs) {
+  for (int32_t i = 0; i < numOfMsgs; i++) {
+    SNodeMsg *pMsg = NULL;
+    taosGetQitem(qall, (void **)&pMsg);
+
+    dTrace("msg:%p, will be processed in snode unique queue", pMsg);
+    sndProcessUMsg(pMgmt->pSnode, &pMsg->rpcMsg);
+
+    dTrace("msg:%p, is freed", pMsg);
+    rpcFreeCont(pMsg->rpcMsg.pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+static void smProcessSharedQueue(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  dTrace("msg:%p, will be processed in snode shared queue", pMsg);
+  sndProcessSMsg(pMgmt->pSnode, &pMsg->rpcMsg);
+
+  dTrace("msg:%p, is freed", pMsg);
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  taosFreeQitem(pMsg);
+}
+
+int32_t smStartWorker(SSnodeMgmt *pMgmt) {
+  pMgmt->uniqueWorkers = taosArrayInit(0, sizeof(void *));
+  if (pMgmt->uniqueWorkers == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
+    SDnodeWorker *pUniqueWorker = malloc(sizeof(SDnodeWorker));
+    if (pUniqueWorker == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    if (dndInitWorker(pMgmt, pUniqueWorker, DND_WORKER_MULTI, "snode-unique", 1, 1, smProcessSharedQueue) != 0) {
+      dError("failed to start snode unique worker since %s", terrstr());
+      return -1;
+    }
+    if (taosArrayPush(pMgmt->uniqueWorkers, &pUniqueWorker) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  if (dndInitWorker(pMgmt, &pMgmt->sharedWorker, DND_WORKER_SINGLE, "snode-shared", SND_SHARED_THREAD_NUM,
+                    SND_SHARED_THREAD_NUM, smProcessSharedQueue)) {
+    dError("failed to start snode shared worker since %s", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+void smStopWorker(SSnodeMgmt *pMgmt) {
+  for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
+    SDnodeWorker *worker = taosArrayGetP(pMgmt->uniqueWorkers, i);
+    dndCleanupWorker(worker);
+  }
+  taosArrayDestroy(pMgmt->uniqueWorkers);
+  dndCleanupWorker(&pMgmt->sharedWorker);
+}
+
+static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
+  SMsgHead *pHead = pMsg->pCont;
+  pHead->streamTaskId = htonl(pHead->streamTaskId);
+  return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM;
+}
+
+static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
+  SStreamExecMsgHead *pHead = pMsg->pCont;
+  pHead->workerType = htonl(pHead->workerType);
+  return pHead->workerType;
+}
+
+int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
+  if (pWorker == NULL) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  int32_t       index = smGetSWIdFromMsg(&pMsg->rpcMsg);
+  SDnodeWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
+  if (pWorker == NULL) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  SDnodeWorker *pWorker = &pMgmt->sharedWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  return dndWriteMsgToWorker(pWorker, pMsg);
+}
+
+int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
+  int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg);
+  if (workerType == SND_WORKER_TYPE__SHARED) {
+    return smProcessSharedMsg(pMgmt, pMsg);
+  } else {
+    return smProcessUniqueMsg(pMgmt, pMsg);
+  }
+}