diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h
index cf5a3c8e05..e70a8539c3 100644
--- a/include/common/taosmsg.h
+++ b/include/common/taosmsg.h
@@ -672,16 +672,16 @@ typedef struct {
} SDnodeCfg;
typedef struct {
- int32_t dnodeId;
+ int32_t id;
int8_t isMnode;
int8_t reserved;
- uint16_t dnodePort;
- char dnodeFqdn[TSDB_FQDN_LEN];
+ uint16_t port;
+ char fqdn[TSDB_FQDN_LEN];
} SDnodeEp;
typedef struct {
- int32_t dnodeNum;
- SDnodeEp dnodeEps[];
+ int32_t num;
+ SDnodeEp eps[];
} SDnodeEps;
typedef struct {
@@ -820,9 +820,9 @@ typedef struct {
} SCreateDnodeMsg, SDropDnodeMsg;
typedef struct {
- int32_t dnodeId;
- int8_t replica;
- int8_t reserved[3];
+ int32_t dnodeId;
+ int8_t replica;
+ int8_t reserved[3];
SReplica replicas[TSDB_MAX_REPLICA];
} SCreateMnodeMsg, SAlterMnodeMsg, SDropMnodeMsg;
diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h
index 5002ee37b0..a1a94bb10b 100644
--- a/include/dnode/mgmt/dnode.h
+++ b/include/dnode/mgmt/dnode.h
@@ -16,6 +16,8 @@
#ifndef _TD_DNODE_H_
#define _TD_DNODE_H_
+#include "tdef.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -23,6 +25,24 @@ extern "C" {
/* ------------------------ TYPES EXPOSED ------------------------ */
typedef struct SDnode SDnode;
+typedef struct {
+ int32_t sver;
+ int32_t numOfCores;
+ float numOfThreadsPerCore;
+ float ratioOfQueryCores;
+ int32_t maxShellConns;
+ int32_t shellActivityTimer;
+ int32_t statusInterval;
+ uint16_t serverPort;
+ char dataDir[PATH_MAX];
+ char localEp[TSDB_EP_LEN];
+ char localFqdn[TSDB_FQDN_LEN];
+ char firstEp[TSDB_EP_LEN];
+ char timezone[TSDB_TIMEZONE_LEN];
+ char locale[TSDB_LOCALE_LEN];
+ char charset[TSDB_LOCALE_LEN];
+} SDnodeOpt;
+
/* ------------------------ SDnode ------------------------ */
/**
* @brief Initialize and start the dnode.
@@ -30,14 +50,14 @@ typedef struct SDnode SDnode;
* @param cfgPath Config file path.
* @return SDnode* The dnode object.
*/
-SDnode *dnodeInit(const char *cfgPath);
+SDnode *dndInit(SDnodeOpt *pOptions);
/**
* @brief Stop and cleanup dnode.
*
- * @param pDnode The dnode object to close.
+ * @param pDnd The dnode object to close.
*/
-void dnodeCleanup(SDnode *pDnode);
+void dndCleanup(SDnode *pDnd);
#ifdef __cplusplus
}
diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h
index 0071296bc1..98aefc6db3 100644
--- a/include/dnode/mnode/mnode.h
+++ b/include/dnode/mnode/mnode.h
@@ -24,10 +24,10 @@ extern "C" {
typedef struct SDnode SDnode;
typedef struct SMnode SMnode;
typedef struct SMnodeMsg SMnodeMsg;
-typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg, bool forShell);
-typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnode, SMnodeMsg *pMsg);
+typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
+typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg);
+typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell);
+typedef int32_t (*PutMsgToMnodeQFp)(SDnode *pDnd, SMnodeMsg *pMsg);
typedef struct SMnodeLoad {
int64_t numOfDnode;
@@ -48,7 +48,7 @@ typedef struct {
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
- struct SServer *pServer;
+ struct SDnode *pDnode;
PutMsgToMnodeQFp putMsgToApplyMsgFp;
SendMsgToDnodeFp sendMsgToDnodeFp;
SendMsgToMnodeFp sendMsgToMnodeFp;
@@ -122,10 +122,17 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg);
/**
* @brief Cleanup mnode msg
*
- * @param pMnode The mnode object
* @param pMsg The request msg
*/
-void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg);
+void mnodeCleanupMsg(SMnodeMsg *pMsg);
+
+/**
+ * @brief Cleanup mnode msg
+ *
+ * @param pMsg The request msg
+ * @param code The error code
+ */
+void mnodeSendRsp(SMnodeMsg *pMsg, int32_t code);
/**
* @brief Process the read request
diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h
index 36f6a3b6fb..586cb49d0f 100644
--- a/include/dnode/vnode/vnode.h
+++ b/include/dnode/vnode/vnode.h
@@ -185,10 +185,10 @@ typedef struct {
} SVnodeMsg;
typedef struct SDnode SDnode;
-typedef void (*SendMsgToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
-typedef void (*SendMsgToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
-typedef void (*SendRedirectMsgFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg, bool forShell);
-typedef int32_t (*PutMsgToVnodeQFp)(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg);
+typedef void (*SendMsgToDnodeFp)(SDnode *pDnd, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
+typedef void (*SendMsgToMnodeFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg);
+typedef void (*SendRedirectMsgFp)(SDnode *pDnd, struct SRpcMsg *rpcMsg, bool forShell);
+typedef int32_t (*PutMsgToVnodeQFp)(SDnode *pDnd, int32_t vgId, SVnodeMsg *pMsg);
typedef struct {
PutMsgToVnodeQFp putMsgToApplyQueueFp;
diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 5460ae5401..2a8a7aad4b 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -51,7 +51,7 @@ typedef struct SRpcMsg {
} SRpcMsg;
typedef struct SRpcInit {
- uint16_t localPort; // local port
+ uint16_t localPort; // local port
char *label; // for debug purpose
int numOfThreads; // number of threads to handle connections
int sessions; // number of sessions allowed
@@ -66,10 +66,12 @@ typedef struct SRpcInit {
char *ckey; // ciphering key
// call back to process incoming msg, code shall be ignored by server app
- void (*cfp)(SRpcMsg *, SEpSet *);
+ void (*cfp)(void *parent, SRpcMsg *, SEpSet *);
- // call back to retrieve the client auth info, for server app only
- int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+ // call back to retrieve the client auth info, for server app only
+ int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+
+ void *parent;
} SRpcInit;
int32_t rpcInit();
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 022c3e9096..304fb56a6a 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -68,12 +68,13 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0106)
#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0107)
#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108)
-#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0109)
-#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x010A)
-#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x010B)
-#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x010C)
-#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x010D)
-#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010E)
+#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109)
+#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110)
+#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111)
+#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112)
+#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0113)
+#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0114)
+#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0115)
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation")
@@ -223,20 +224,20 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) //"Topic already exists)
// dnode
-#define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) //"Message not processed")
-#define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) //"Dnode out of memory")
-#define TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE TAOS_DEF_ERROR_CODE(0, 0x0402) //"Mnode Id not match Dnode")
-#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) //"Mnode already deployed")
-#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) //"Mnode not deployed")
-#define TSDB_CODE_DND_READ_MNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) //"Read mnode.json error")
-#define TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0406) //"Write mnode.json error")
-#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0407) //"No permission for disk files in dnode")
-#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0408) //"Invalid message length")
-#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0409) //"Action in progress")
-#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x040A) //"Too many vnode directories")
-#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x040B) //"Dnode is exiting"
-#define TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x040C) //"Parse vnodes.json error")
-#define TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x040D) //"Parse dnodes.json error")
+#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)
+#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401)
+#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402)
+#define TSDB_CODE_DND_DNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0410)
+#define TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0411)
+#define TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0420)
+#define TSDB_CODE_DND_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0421)
+#define TSDB_CODE_DND_MNODE_ID_INVALID TAOS_DEF_ERROR_CODE(0, 0x0422)
+#define TSDB_CODE_DND_MNODE_ID_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0423)
+#define TSDB_CODE_DND_MNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0424)
+#define TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0425)
+#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0430)
+#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0431)
+#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0432)
// vnode
#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress")
diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c
index 720d1589c2..326dfae4af 100644
--- a/source/dnode/mgmt/daemon/src/daemon.c
+++ b/source/dnode/mgmt/daemon/src/daemon.c
@@ -16,38 +16,173 @@
#define _DEFAULT_SOURCE
#include "dnode.h"
#include "os.h"
+#include "tglobal.h"
#include "ulog.h"
-static bool stop = false;
+static struct {
+ bool stop;
+ bool dumpConfig;
+ bool generateGrant;
+ bool printAuth;
+ bool printVersion;
+ char configDir[PATH_MAX];
+} global = {0};
-static void sigintHandler(int32_t signum, void *info, void *ctx) { stop = true; }
+void dmnSigintHandle(int signum, void *info, void *ctx) { global.stop = true; }
-static void setSignalHandler() {
- taosSetSignal(SIGTERM, sigintHandler);
- taosSetSignal(SIGHUP, sigintHandler);
- taosSetSignal(SIGINT, sigintHandler);
- taosSetSignal(SIGABRT, sigintHandler);
- taosSetSignal(SIGBREAK, sigintHandler);
+void dmnSetSignalHandle() {
+ taosSetSignal(SIGTERM, dmnSigintHandle);
+ taosSetSignal(SIGHUP, dmnSigintHandle);
+ taosSetSignal(SIGINT, dmnSigintHandle);
+ taosSetSignal(SIGABRT, dmnSigintHandle);
+ taosSetSignal(SIGBREAK, dmnSigintHandle);
}
-int main(int argc, char const *argv[]) {
- const char *path = "/etc/taos";
+int dmnParseOpts(int argc, char const *argv[]) {
+ tstrncpy(global.configDir, "/etc/taos", PATH_MAX);
- SDnode *pDnode = dnodeInit(path);
- if (pDnode == NULL) {
- uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
- exit(EXIT_FAILURE);
+ for (int i = 1; i < argc; ++i) {
+ if (strcmp(argv[i], "-c") == 0) {
+ if (i < argc - 1) {
+ if (strlen(argv[++i]) >= PATH_MAX) {
+ printf("config file path overflow");
+ return -1;
+ }
+ tstrncpy(global.configDir, argv[i], PATH_MAX);
+ } else {
+ printf("'-c' requires a parameter, default:%s\n", configDir);
+ return -1;
+ }
+ } else if (strcmp(argv[i], "-C") == 0) {
+ global.dumpConfig = true;
+ } else if (strcmp(argv[i], "-k") == 0) {
+ global.generateGrant = true;
+ } else if (strcmp(argv[i], "-A") == 0) {
+ global.printAuth = true;
+ } else if (strcmp(argv[i], "-V") == 0) {
+ global.printVersion = true;
+ } else {
+ }
}
- uInfo("Started TDengine service successfully.");
-
- setSignalHandler();
- while (!stop) {
- taosMsleep(100);
- }
-
- uInfo("TDengine is shut down!");
- dnodeCleanup(pDnode);
-
return 0;
}
+
+void dmnGenerateGrant() { grantParseParameter(); }
+
+void dmnPrintVersion() {
+#ifdef TD_ENTERPRISE
+ char *versionStr = "enterprise";
+#else
+ char *versionStr = "community";
+#endif
+ printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version);
+ printf("gitinfo: %s\n", gitinfo);
+ printf("gitinfoI: %s\n", gitinfoOfInternal);
+ printf("builuInfo: %s\n", buildinfo);
+}
+
+int dmnReadConfig(const char *path) {
+ taosIgnSIGPIPE();
+ taosBlockSIGPIPE();
+ taosResolveCRC();
+ taosInitGlobalCfg();
+ taosReadGlobalLogCfg();
+
+ if (taosMkDir(tsLogDir) != 0) {
+ printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
+ return -1;
+ }
+
+ char temp[PATH_MAX];
+ snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir);
+ if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
+ printf("failed to init log file\n");
+ return -1;
+ }
+
+ if (taosInitNotes() != 0) {
+ printf("failed to init log file\n");
+ return -1;
+ }
+
+ if (taosReadGlobalCfg() != 0) {
+ uError("failed to read global config");
+ return -1;
+ }
+
+ if (taosCheckGlobalCfg() != 0) {
+ uError("failed to check global config");
+ return -1;
+ }
+
+ taosSetCoreDump(tsEnableCoreFile);
+ return 0;
+}
+
+void dmnDumpConfig() { taosDumpGlobalCfg(); }
+
+void dmnWaitSignal() {
+ dmnSetSignalHandle();
+ while (!global.stop) {
+ taosMsleep(100);
+ }
+}
+
+void dmnInitOption(SDnodeOpt *pOpt) {
+ pOpt->sver = tsVersion;
+ pOpt->numOfCores = tsNumOfCores;
+ pOpt->statusInterval = tsStatusInterval;
+ pOpt->serverPort = tsServerPort;
+ tstrncpy(pOpt->localEp, tsLocalEp, TSDB_EP_LEN);
+ tstrncpy(pOpt->localFqdn, tsLocalEp, TSDB_FQDN_LEN);
+ tstrncpy(pOpt->timezone, tsLocalEp, TSDB_TIMEZONE_LEN);
+ tstrncpy(pOpt->locale, tsLocalEp, TSDB_LOCALE_LEN);
+ tstrncpy(pOpt->charset, tsLocalEp, TSDB_LOCALE_LEN);
+}
+
+int dmnRunDnode() {
+ SDnodeOpt opt = {0};
+ dmnInitOption(&opt);
+
+ SDnode *pDnd = dndInit(&opt);
+ if (pDnd == NULL) {
+ uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
+ return -1;
+ }
+
+ uInfo("Started TDengine service successfully.");
+ dmnWaitSignal();
+ uInfo("TDengine is shut down!");
+
+ dndCleanup(pDnd);
+ taosCloseLog();
+ return 0;
+}
+
+int main(int argc, char const *argv[]) {
+ if (dmnParseOpts(argc, argv) != 0) {
+ return -1;
+ }
+
+ if (global.generateGrant) {
+ dmnGenerateGrant();
+ return 0;
+ }
+
+ if (global.printVersion) {
+ dmnPrintVersion();
+ return 0;
+ }
+
+ if (dmnReadConfig(global.configDir) != 0) {
+ return -1;
+ }
+
+ if (global.dumpConfig) {
+ dmnDumpConfig();
+ return 0;
+ }
+
+ return dmnRunDnode();
+}
diff --git a/source/dnode/mgmt/impl/inc/dnodeVnodes.h b/source/dnode/mgmt/impl/inc/dndDnode.h
similarity index 53%
rename from source/dnode/mgmt/impl/inc/dnodeVnodes.h
rename to source/dnode/mgmt/impl/inc/dndDnode.h
index 31eae049ab..ef16b1c8f0 100644
--- a/source/dnode/mgmt/impl/inc/dnodeVnodes.h
+++ b/source/dnode/mgmt/impl/inc/dndDnode.h
@@ -13,26 +13,27 @@
* along with this program. If not, see .
*/
-#ifndef _TD_DNODE_VNODES_H_
-#define _TD_DNODE_VNODES_H_
+#ifndef _TD_DND_DNODE_H_
+#define _TD_DND_DNODE_H_
#ifdef __cplusplus
extern "C" {
#endif
-#include "dnodeInt.h"
+#include "dndInt.h"
-int32_t dnodeInitVnodes();
-void dnodeCleanupVnodes();
-void dnodeGetVnodeLoads(SVnodeLoads *pVloads);
+int32_t dndInitDnode(SDnode *pDnd);
+void dndCleanupDnode(SDnode *pDnd);
+void dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
+int32_t dndGetDnodeId(SDnode *pDnd);
+int64_t dndGetClusterId(SDnode *pDnd);
+void dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
+void dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet);
+void dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_DNODE_VNODES_H_*/
\ No newline at end of file
+#endif /*_TD_DND_DNODE_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/inc/dnodeInt.h b/source/dnode/mgmt/impl/inc/dndInt.h
similarity index 54%
rename from source/dnode/mgmt/impl/inc/dnodeInt.h
rename to source/dnode/mgmt/impl/inc/dndInt.h
index 8944755268..966781426b 100644
--- a/source/dnode/mgmt/impl/inc/dnodeInt.h
+++ b/source/dnode/mgmt/impl/inc/dndInt.h
@@ -13,20 +13,27 @@
* along with this program. If not, see .
*/
-#ifndef _TD_DNODE_INT_H_
-#define _TD_DNODE_INT_H_
+#ifndef _TD_DND_INT_H_
+#define _TD_DND_INT_H_
#ifdef __cplusplus
extern "C" {
#endif
+
+#include "cJSON.h"
#include "os.h"
#include "taosmsg.h"
-#include "tglobal.h"
#include "thash.h"
+#include "tlockfree.h"
#include "tlog.h"
+#include "tqueue.h"
#include "trpc.h"
#include "tthread.h"
#include "ttime.h"
+#include "tworker.h"
+#include "mnode.h"
+#include "vnode.h"
+#include "dnode.h"
extern int32_t dDebugFlag;
@@ -37,8 +44,8 @@ extern int32_t dDebugFlag;
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("SRV ", dDebugFlag, __VA_ARGS__); }}
-typedef enum { DN_STAT_INIT, DN_STAT_RUNNING, DN_STAT_STOPPED } EStat;
-typedef void (*MsgFp)(SRpcMsg *pMsg, SEpSet *pEpSet);
+typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
+typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
typedef struct {
char *dnode;
@@ -48,50 +55,75 @@ typedef struct {
typedef struct {
int32_t dnodeId;
- int64_t clusterId;
- SDnodeEps *dnodeEps;
- SHashObj *dnodeHash;
- SEpSet mnodeEpSetForShell;
- SEpSet mnodeEpSetForPeer;
- char *file;
uint32_t rebootTime;
- int8_t dropped;
- int8_t threadStop;
+ int32_t dropped;
+ int64_t clusterId;
+ SEpSet shellEpSet;
+ SEpSet peerEpSet;
+ char *file;
+ SHashObj *dnodeHash;
+ SDnodeEps *dnodeEps;
pthread_t *threadId;
pthread_mutex_t mutex;
-} SDnodeDnode;
+} SDnodeMgmt;
typedef struct {
-} SDnodeMnode;
+ int32_t refCount;
+ int8_t deployed;
+ int8_t dropped;
+ SWorkerPool mgmtPool;
+ SWorkerPool readPool;
+ SWorkerPool writePool;
+ SWorkerPool syncPool;
+ taos_queue pReadQ;
+ taos_queue pWriteQ;
+ taos_queue pApplyQ;
+ taos_queue pSyncQ;
+ taos_queue pMgmtQ;
+ char *file;
+ SMnode *pMnode;
+ SRWLatch latch;
+} SMnodeMgmt;
typedef struct {
-} SDnodeVnodes;
+ SHashObj *hash;
+ SWorkerPool mgmtPool;
+ SWorkerPool queryPool;
+ SWorkerPool fetchPool;
+ SMWorkerPool syncPool;
+ SMWorkerPool writePool;
+ taos_queue pMgmtQ;
+ int32_t openVnodes;
+ int32_t totalVnodes;
+ SRWLatch latch;
+} SVnodesMgmt;
typedef struct {
- void *peerRpc;
- void *shellRpc;
- void *clientRpc;
-} SDnodeTrans;
+ void *serverRpc;
+ void *clientRpc;
+ DndMsgFp msgFp[TSDB_MSG_TYPE_MAX];
+} STransMgmt;
typedef struct SDnode {
- EStat stat;
- SDnodeDir dir;
- SDnodeDnode dnode;
- SDnodeVnodes vnodes;
- SDnodeMnode mnode;
- SDnodeTrans trans;
- SStartupMsg startup;
+ EStat stat;
+ SDnodeOpt opt;
+ SDnodeDir dir;
+ SDnodeMgmt d;
+ SMnodeMgmt m;
+ SVnodesMgmt vmgmt;
+ STransMgmt t;
+ SStartupMsg startup;
} SDnode;
-EStat dnodeGetStat(SDnode *pDnode);
-void dnodeSetStat(SDnode *pDnode, EStat stat);
-char *dnodeStatStr(EStat stat);
+EStat dndGetStat(SDnode *pDnode);
+void dndSetStat(SDnode *pDnode, EStat stat);
+char *dndStatStr(EStat stat);
-void dnodeReportStartup(SDnode *pDnode, char *name, char *desc);
-void dnodeGetStartup(SDnode *pDnode, SStartupMsg *pStartup);
+void dndReportStartup(SDnode *pDnode, char *name, char *desc);
+void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_DNODE_INT_H_*/
\ No newline at end of file
+#endif /*_TD_DND_INT_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/inc/dnodeDnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h
similarity index 53%
rename from source/dnode/mgmt/impl/inc/dnodeDnode.h
rename to source/dnode/mgmt/impl/inc/dndMnode.h
index 87dc0fdb9b..67c51e51a8 100644
--- a/source/dnode/mgmt/impl/inc/dnodeDnode.h
+++ b/source/dnode/mgmt/impl/inc/dndMnode.h
@@ -13,27 +13,24 @@
* along with this program. If not, see .
*/
-#ifndef _TD_DNODE_DNODE_H_
-#define _TD_DNODE_DNODE_H_
+#ifndef _TD_DND_MNODE_H_
+#define _TD_DND_MNODE_H_
#ifdef __cplusplus
extern "C" {
#endif
-#include "dnodeInt.h"
+#include "dndInt.h"
-int32_t dnodeInitDnode(SDnode *pDnode);
-void dnodeCleanupDnode(SDnode *pDnode);
-void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-
-int32_t dnodeGetDnodeId();
-int64_t dnodeGetClusterId();
-void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
-void dnodeGetMnodeEpSetForPeer(SEpSet *epSet);
-void dnodeGetMnodeEpSetForShell(SEpSet *epSet);
-void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *rpcMsg, bool forShell);
+int32_t dndInitMnode(SDnode *pDnode);
+void dndCleanupMnode(SDnode *pDnode);
+int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey);
+void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_DNODE_DNODE_H_*/
\ No newline at end of file
+#endif /*_TD_DND_MNODE_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/inc/dnodeTransport.h b/source/dnode/mgmt/impl/inc/dndTransport.h
similarity index 68%
rename from source/dnode/mgmt/impl/inc/dnodeTransport.h
rename to source/dnode/mgmt/impl/inc/dndTransport.h
index 7d3f4be1ff..312da69fa2 100644
--- a/source/dnode/mgmt/impl/inc/dnodeTransport.h
+++ b/source/dnode/mgmt/impl/inc/dndTransport.h
@@ -13,21 +13,21 @@
* along with this program. If not, see .
*/
-#ifndef _TD_DNODE_TRANSPORT_H_
-#define _TD_DNODE_TRANSPORT_H_
+#ifndef _TD_DND_TRANSPORT_H_
+#define _TD_DND_TRANSPORT_H_
#ifdef __cplusplus
extern "C" {
#endif
-#include "dnodeInt.h"
+#include "dndInt.h"
-int32_t dnodeInitTrans();
-void dnodeCleanupTrans();
-void dnodeSendMsgToMnode(SDnode *pDnode, SRpcMsg *rpcMsg);
-void dnodeSendMsgToDnode(SDnode *pDnode, SEpSet *epSet, SRpcMsg *rpcMsg);
+int32_t dndInitTrans(SDnode *pDnode);
+void dndCleanupTrans(SDnode *pDnode);
+void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pRpcMsg);
+void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pRpcMsg);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_DNODE_TRANSPORT_H_*/
+#endif /*_TD_DND_TRANSPORT_H_*/
diff --git a/source/dnode/mgmt/impl/inc/dnodeMnode.h b/source/dnode/mgmt/impl/inc/dndVnodes.h
similarity index 52%
rename from source/dnode/mgmt/impl/inc/dnodeMnode.h
rename to source/dnode/mgmt/impl/inc/dndVnodes.h
index 9f52e586fe..35f99ee73b 100644
--- a/source/dnode/mgmt/impl/inc/dnodeMnode.h
+++ b/source/dnode/mgmt/impl/inc/dndVnodes.h
@@ -13,25 +13,25 @@
* along with this program. If not, see .
*/
-#ifndef _TD_DNODE_MNODE_H_
-#define _TD_DNODE_MNODE_H_
+#ifndef _TD_DND_VNODES_H_
+#define _TD_DND_VNODES_H_
#ifdef __cplusplus
extern "C" {
#endif
-#include "dnodeInt.h"
+#include "dndInt.h"
-int32_t dnodeInitMnode();
-void dnodeCleanupMnode();
-int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-
-void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
-void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet);
+int32_t dndInitVnodes(SDnode *pDnode);
+void dndCleanupVnodes(SDnode *pDnode);
+void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pVloads);
+void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_DNODE_MNODE_H_*/
\ No newline at end of file
+#endif /*_TD_DND_VNODES_H_*/
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dnodeDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c
similarity index 51%
rename from source/dnode/mgmt/impl/src/dnodeDnode.c
rename to source/dnode/mgmt/impl/src/dndDnode.c
index ec60116ce6..378d76e046 100644
--- a/source/dnode/mgmt/impl/src/dnodeDnode.c
+++ b/source/dnode/mgmt/impl/src/dndDnode.c
@@ -14,69 +14,76 @@
*/
#define _DEFAULT_SOURCE
-#include "dnodeDnode.h"
-#include "dnodeTransport.h"
-#include "dnodeVnodes.h"
-#include "cJSON.h"
+#include "dndDnode.h"
+#include "dndTransport.h"
+#include "dndVnodes.h"
-int32_t dnodeGetDnodeId() {
- int32_t dnodeId = 0;
- pthread_mutex_lock(&pDnode->mutex);
- dnodeId = pDnode->dnodeId;
- pthread_mutex_unlock(&pDnode->mutex);
+static inline void dndLockDnode(SDnode *pDnd) { pthread_mutex_lock(&pDnd->d.mutex); }
+
+static inline void dndUnLockDnode(SDnode *pDnd) { pthread_mutex_unlock(&pDnd->d.mutex); }
+
+int32_t dndGetDnodeId(SDnode *pDnd) {
+ dndLockDnode(pDnd);
+ int32_t dnodeId = pDnd->d.dnodeId;
+ dndUnLockDnode(pDnd);
return dnodeId;
}
-int64_t dnodeGetClusterId() {
- int64_t clusterId = 0;
- pthread_mutex_lock(&pDnode->mutex);
- clusterId = pDnode->clusterId;
- pthread_mutex_unlock(&pDnode->mutex);
+int64_t dndGetClusterId(SDnode *pDnd) {
+ dndLockDnode(pDnd);
+ int64_t clusterId = pDnd->d.clusterId;
+ dndUnLockDnode(pDnd);
return clusterId;
}
-void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) {
- pthread_mutex_lock(&pDnode->mutex);
+void dndGetDnodeEp(SDnode *pDnd, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) {
+ dndLockDnode(pDnd);
- SDnodeEp *pEp = taosHashGet(pDnode->dnodeHash, &dnodeId, sizeof(int32_t));
- if (pEp != NULL) {
- if (port) *port = pEp->dnodePort;
- if (fqdn) tstrncpy(fqdn, pEp->dnodeFqdn, TSDB_FQDN_LEN);
- if (ep) snprintf(ep, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort);
+ SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t));
+ if (pDnodeEp != NULL) {
+ if (pPort != NULL) {
+ *pPort = pDnodeEp->port;
+ }
+ if (pFqdn != NULL) {
+ tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN);
+ }
+ if (pEp != NULL) {
+ snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
+ }
}
- pthread_mutex_unlock(&pDnode->mutex);
+ dndUnLockDnode(pDnd);
}
-void dnodeGetMnodeEpSetForPeer(SEpSet *pEpSet) {
- pthread_mutex_lock(&pDnode->mutex);
- *pEpSet = pDnode->mnodeEpSetForPeer;
- pthread_mutex_unlock(&pDnode->mutex);
+void dndGetMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+ dndLockDnode(pDnd);
+ *pEpSet = pDnd->d.peerEpSet;
+ dndUnLockDnode(pDnd);
}
-void dnodeGetMnodeEpSetForShell(SEpSet *pEpSet) {
- pthread_mutex_lock(&pDnode->mutex);
- *pEpSet = pDnode->mnodeEpSetForShell;
- pthread_mutex_unlock(&pDnode->mutex);
+void dndGetShellEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+ dndLockDnode(pDnd);
+ *pEpSet = pDnd->d.shellEpSet;
+ dndUnLockDnode(pDnd);
}
-void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg, bool forShell) {
+void dndSendRedirectMsg(SDnode *pDnd, SRpcMsg *pMsg, bool forShell) {
int32_t msgType = pMsg->msgType;
SEpSet epSet = {0};
if (forShell) {
- dnodeGetMnodeEpSetForShell(&epSet);
+ dndGetShellEpSet(pDnd, &epSet);
} else {
- dnodeGetMnodeEpSetForPeer(&epSet);
+ dndGetMnodeEpSet(pDnd, &epSet);
}
dDebug("RPC %p, msg:%s is redirected, num:%d use:%d", pMsg->handle, taosMsg[msgType], epSet.numOfEps, epSet.inUse);
for (int32_t i = 0; i < epSet.numOfEps; ++i) {
dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]);
- if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) {
- if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) ||
- (epSet.port[i] == tsServerPort && forShell)) {
+ if (strcmp(epSet.fqdn[i], pDnd->opt.localFqdn) == 0) {
+ if ((epSet.port[i] == pDnd->opt.serverPort + TSDB_PORT_DNODEDNODE && !forShell) ||
+ (epSet.port[i] == pDnd->opt.serverPort && forShell)) {
epSet.inUse = (i + 1) % epSet.numOfEps;
dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse);
}
@@ -88,39 +95,37 @@ void dnodeSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg, bool forShell) {
rpcSendRedirectRsp(pMsg->handle, &epSet);
}
-static void dnodeUpdateMnodeEpSet(SDnodeDnode *pDnode, SEpSet *pEpSet) {
- if (pEpSet == NULL || pEpSet->numOfEps <= 0) {
- dError("mnode is changed, but content is invalid, discard it");
- return;
- } else {
- dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
- }
+static void dndUpdateMnodeEpSet(SDnode *pDnd, SEpSet *pEpSet) {
+ dInfo("mnode is changed, num:%d use:%d", pEpSet->numOfEps, pEpSet->inUse);
- pthread_mutex_lock(&pDnode->mutex);
+ dndLockDnode(pDnd);
- pDnode->mnodeEpSetForPeer = *pEpSet;
+ pDnd->d.peerEpSet = *pEpSet;
for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
pEpSet->port[i] -= TSDB_PORT_DNODEDNODE;
dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]);
}
- pDnode->mnodeEpSetForShell = *pEpSet;
+ pDnd->d.shellEpSet = *pEpSet;
- pthread_mutex_unlock(&pDnode->mutex);
+ dndUnLockDnode(pDnd);
}
-static void dnodePrintDnodes() {
- dDebug("print dnode endpoint list, num:%d", pDnode->dnodeEps->dnodeNum);
- for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; i++) {
- SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i];
- dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort, ep->isMnode);
+static void dndPrintDnodes(SDnode *pDnd) {
+ SDnodeMgmt *pDnode = &pDnd->d;
+
+ dDebug("print dnode endpoint list, num:%d", pDnode->dnodeEps->num);
+ for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) {
+ SDnodeEp *pEp = &pDnode->dnodeEps->eps[i];
+ dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode);
}
}
-static void dnodeResetDnodes(SDnodeEps *pEps) {
- assert(pEps != NULL);
- int32_t size = sizeof(SDnodeEps) + pEps->dnodeNum * sizeof(SDnodeEp);
+static void dndResetDnodes(SDnode *pDnd, SDnodeEps *pDnodeEps) {
+ SDnodeMgmt *pDnode = &pDnd->d;
- if (pEps->dnodeNum > pDnode->dnodeEps->dnodeNum) {
+ int32_t size = sizeof(SDnodeEps) + pDnodeEps->num * sizeof(SDnodeEp);
+
+ if (pDnodeEps->num > pDnode->dnodeEps->num) {
SDnodeEps *tmp = calloc(1, size);
if (tmp == NULL) return;
@@ -128,49 +133,51 @@ static void dnodeResetDnodes(SDnodeEps *pEps) {
pDnode->dnodeEps = tmp;
}
- if (pDnode->dnodeEps != pEps) {
- memcpy(pDnode->dnodeEps, pEps, size);
+ if (pDnode->dnodeEps != pDnodeEps) {
+ memcpy(pDnode->dnodeEps, pDnodeEps, size);
}
- pDnode->mnodeEpSetForPeer.inUse = 0;
- pDnode->mnodeEpSetForShell.inUse = 0;
+ pDnode->peerEpSet.inUse = 0;
+ pDnode->shellEpSet.inUse = 0;
int32_t mIndex = 0;
- for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; i++) {
- SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i];
- if (!ep->isMnode) continue;
+ for (int32_t i = 0; i < pDnode->dnodeEps->num; i++) {
+ SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+ if (!pDnodeEp->isMnode) continue;
if (mIndex >= TSDB_MAX_REPLICA) continue;
- strcpy(pDnode->mnodeEpSetForShell.fqdn[mIndex], ep->dnodeFqdn);
- strcpy(pDnode->mnodeEpSetForPeer.fqdn[mIndex], ep->dnodeFqdn);
- pDnode->mnodeEpSetForShell.port[mIndex] = ep->dnodePort;
- pDnode->mnodeEpSetForShell.port[mIndex] = ep->dnodePort + tsDnodeDnodePort;
+ strcpy(pDnode->shellEpSet.fqdn[mIndex], pDnodeEp->fqdn);
+ strcpy(pDnode->peerEpSet.fqdn[mIndex], pDnodeEp->fqdn);
+ pDnode->shellEpSet.port[mIndex] = pDnodeEp->port;
+ pDnode->shellEpSet.port[mIndex] = pDnodeEp->port + TSDB_PORT_DNODEDNODE;
mIndex++;
}
- for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; ++i) {
- SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i];
- taosHashPut(pDnode->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+ for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) {
+ SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+ taosHashPut(pDnode->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
}
- dnodePrintDnodes();
+ dndPrintDnodes(pDnd);
}
-static bool dnodeIsEpChanged(int32_t dnodeId, char *epStr) {
+static bool dndIsEpChanged(SDnode *pDnd, int32_t dnodeId) {
bool changed = false;
- pthread_mutex_lock(&pDnode->mutex);
+ dndLockDnode(pDnd);
- SDnodeEp *pEp = taosHashGet(pDnode->dnodeHash, &dnodeId, sizeof(int32_t));
- if (pEp != NULL) {
- char epSaved[TSDB_EP_LEN + 1];
- snprintf(epSaved, TSDB_EP_LEN, "%s:%u", pEp->dnodeFqdn, pEp->dnodePort);
- changed = strcmp(epStr, epSaved) != 0;
+ SDnodeEp *pDnodeEp = taosHashGet(pDnd->d.dnodeHash, &dnodeId, sizeof(int32_t));
+ if (pDnodeEp != NULL) {
+ char epstr[TSDB_EP_LEN + 1];
+ snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port);
+ changed = strcmp(pDnd->opt.localEp, epstr) != 0;
}
- pthread_mutex_unlock(&pDnode->mutex);
+ dndUnLockDnode(pDnd);
return changed;
}
-static int32_t dnodeReadDnodes() {
+static int32_t dndReadDnodes(SDnode *pDnd) {
+ SDnodeMgmt *pDnode = &pDnd->d;
+
int32_t len = 0;
int32_t maxLen = 30000;
char *content = calloc(1, maxLen + 1);
@@ -234,70 +241,72 @@ static int32_t dnodeReadDnodes() {
dError("failed to calloc dnodeEpList since %s", strerror(errno));
goto PRASE_DNODE_OVER;
}
- pDnode->dnodeEps->dnodeNum = dnodeInfosSize;
+ pDnode->dnodeEps->num = dnodeInfosSize;
for (int32_t i = 0; i < dnodeInfosSize; ++i) {
cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
if (dnodeInfo == NULL) break;
- SDnodeEp *pEp = &pDnode->dnodeEps->dnodeEps[i];
+ SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
if (!dnodeId || dnodeId->type != cJSON_String) {
dError("failed to read %s, dnodeId not found", pDnode->file);
goto PRASE_DNODE_OVER;
}
- pEp->dnodeId = atoi(dnodeId->valuestring);
+ pDnodeEp->id = atoi(dnodeId->valuestring);
cJSON *isMnode = cJSON_GetObjectItem(dnodeInfo, "isMnode");
if (!isMnode || isMnode->type != cJSON_String) {
dError("failed to read %s, isMnode not found", pDnode->file);
goto PRASE_DNODE_OVER;
}
- pEp->isMnode = atoi(isMnode->valuestring);
+ pDnodeEp->isMnode = atoi(isMnode->valuestring);
cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
dError("failed to read %s, dnodeFqdn not found", pDnode->file);
goto PRASE_DNODE_OVER;
}
- tstrncpy(pEp->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+ tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
if (!dnodePort || dnodePort->type != cJSON_String) {
dError("failed to read %s, dnodePort not found", pDnode->file);
goto PRASE_DNODE_OVER;
}
- pEp->dnodePort = atoi(dnodePort->valuestring);
+ pDnodeEp->port = atoi(dnodePort->valuestring);
}
dInfo("succcessed to read file %s", pDnode->file);
- dnodePrintDnodes();
+ dndPrintDnodes(pDnd);
PRASE_DNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
- if (dnodeIsEpChanged(pDnode->dnodeId, tsLocalEp)) {
- dError("localEp %s different with %s and need reconfigured", tsLocalEp, pDnode->file);
+ if (dndIsEpChanged(pDnd, pDnode->dnodeId)) {
+ dError("localEp %s different with %s and need reconfigured", pDnd->opt.localEp, pDnode->file);
return -1;
}
if (pDnode->dnodeEps == NULL) {
pDnode->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
- pDnode->dnodeEps->dnodeNum = 1;
- pDnode->dnodeEps->dnodeEps[0].dnodePort = tsServerPort;
- tstrncpy(pDnode->dnodeEps->dnodeEps[0].dnodeFqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+ pDnode->dnodeEps->num = 1;
+ pDnode->dnodeEps->eps[0].port = pDnd->opt.serverPort;
+ tstrncpy(pDnode->dnodeEps->eps[0].fqdn, pDnd->opt.localFqdn, TSDB_FQDN_LEN);
}
- dnodeResetDnodes(pDnode->dnodeEps);
+ dndResetDnodes(pDnd, pDnode->dnodeEps);
terrno = 0;
return 0;
}
-static int32_t dnodeWriteDnodes() {
+static int32_t dndWriteDnodes(SDnode *pDnd) {
+ SDnodeMgmt *pDnode = &pDnd->d;
+
FILE *fp = fopen(pDnode->file, "w");
if (!fp) {
dError("failed to write %s since %s", pDnode->file, strerror(errno));
@@ -313,13 +322,13 @@ static int32_t dnodeWriteDnodes() {
len += snprintf(content + len, maxLen - len, " \"clusterId\": \"%" PRId64 "\",\n", pDnode->clusterId);
len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\",\n", pDnode->dropped);
len += snprintf(content + len, maxLen - len, " \"dnodeInfos\": [{\n");
- for (int32_t i = 0; i < pDnode->dnodeEps->dnodeNum; ++i) {
- SDnodeEp *ep = &pDnode->dnodeEps->dnodeEps[i];
- len += snprintf(content + len, maxLen - len, " \"dnodeId\": \"%d\",\n", ep->dnodeId);
- len += snprintf(content + len, maxLen - len, " \"isMnode\": \"%d\",\n", ep->isMnode);
- len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
- len += snprintf(content + len, maxLen - len, " \"dnodePort\": \"%u\"\n", ep->dnodePort);
- if (i < pDnode->dnodeEps->dnodeNum - 1) {
+ for (int32_t i = 0; i < pDnode->dnodeEps->num; ++i) {
+ SDnodeEp *pDnodeEp = &pDnode->dnodeEps->eps[i];
+ len += snprintf(content + len, maxLen - len, " \"dnodeId\": \"%d\",\n", pDnodeEp->id);
+ len += snprintf(content + len, maxLen - len, " \"isMnode\": \"%d\",\n", pDnodeEp->isMnode);
+ len += snprintf(content + len, maxLen - len, " \"dnodeFqdn\": \"%s\",\n", pDnodeEp->fqdn);
+ len += snprintf(content + len, maxLen - len, " \"dnodePort\": \"%u\"\n", pDnodeEp->port);
+ if (i < pDnode->dnodeEps->num - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
} else {
len += snprintf(content + len, maxLen - len, " }]\n");
@@ -337,107 +346,114 @@ static int32_t dnodeWriteDnodes() {
return 0;
}
-static void dnodeSendStatusMsg() {
- int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-
+static void dndSendStatusMsg(SDnode *pDnd) {
+ int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
SStatusMsg *pStatus = rpcMallocCont(contLen);
if (pStatus == NULL) {
dError("failed to malloc status message");
return;
}
- pStatus->sversion = htonl(tsVersion);
- pStatus->dnodeId = htonl(dnodeGetDnodeId());
- pStatus->clusterId = htobe64(dnodeGetClusterId());
- pStatus->rebootTime = htonl(pDnode->rebootTime);
- pStatus->numOfCores = htonl(tsNumOfCores);
- tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
-
- pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval);
+ dndLockDnode(pDnd);
+ pStatus->sversion = htonl(pDnd->opt.sver);
+ pStatus->dnodeId = htonl(pDnd->d.dnodeId);
+ pStatus->clusterId = htobe64(pDnd->d.clusterId);
+ pStatus->rebootTime = htonl(pDnd->d.rebootTime);
+ pStatus->numOfCores = htonl(pDnd->opt.numOfCores);
+ tstrncpy(pStatus->dnodeEp, pDnd->opt.localEp, TSDB_EP_LEN);
+ pStatus->clusterCfg.statusInterval = htonl(pDnd->opt.statusInterval);
+ tstrncpy(pStatus->clusterCfg.timezone, pDnd->opt.timezone, TSDB_TIMEZONE_LEN);
+ tstrncpy(pStatus->clusterCfg.locale, pDnd->opt.locale, TSDB_LOCALE_LEN);
+ tstrncpy(pStatus->clusterCfg.charset, pDnd->opt.charset, TSDB_LOCALE_LEN);
pStatus->clusterCfg.checkTime = 0;
- tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, TSDB_TIMEZONE_LEN);
- tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
- tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+ dndUnLockDnode(pDnd);
- dnodeGetVnodeLoads(&pStatus->vnodeLoads);
+ dndGetVnodeLoads(pDnd, &pStatus->vnodeLoads);
contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad);
SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS};
- dnodeSendMsgToMnode(NULL, &rpcMsg);
+ dndSendMsgToMnode(pDnd, &rpcMsg);
}
-static void dnodeUpdateCfg(SDnodeCfg *pCfg) {
- if (pDnode->dnodeId == 0) return;
- if (pDnode->dropped) return;
+static void dndUpdateDnodeCfg(SDnode *pDnd, SDnodeCfg *pCfg) {
+ SDnodeMgmt *pDnode = &pDnd->d;
+ if (pDnode->dnodeId != 0 && pDnode->dropped != pCfg->dropped) return;
- pthread_mutex_lock(&pDnode->mutex);
+ dndLockDnode(pDnd);
pDnode->dnodeId = pCfg->dnodeId;
pDnode->clusterId = pCfg->clusterId;
pDnode->dropped = pCfg->dropped;
- dInfo("dnodeId is set to %d, clusterId is set to %" PRId64, pCfg->dnodeId, pCfg->clusterId);
+ dInfo("set dnodeId:%d clusterId:%" PRId64 " dropped:%d", pCfg->dnodeId, pCfg->clusterId, pCfg->dropped);
- dnodeWriteDnodes();
- pthread_mutex_unlock(&pDnode->mutex);
+ dndWriteDnodes(pDnd);
+ dndUnLockDnode(pDnd);
}
-static void dnodeUpdateDnodeEps(SDnodeEps *pEps) {
- if (pEps == NULL || pEps->dnodeNum <= 0) return;
+static void dndUpdateDnodeEps(SDnode *pDnd, SDnodeEps *pDnodeEps) {
+ if (pDnodeEps == NULL || pDnodeEps->num <= 0) return;
- pthread_mutex_lock(&pDnode->mutex);
+ dndLockDnode(pDnd);
- if (pEps->dnodeNum != pDnode->dnodeEps->dnodeNum) {
- dnodeResetDnodes(pEps);
- dnodeWriteDnodes();
+ if (pDnodeEps->num != pDnd->d.dnodeEps->num) {
+ dndResetDnodes(pDnd, pDnodeEps);
+ dndWriteDnodes(pDnd);
} else {
- int32_t size = pEps->dnodeNum * sizeof(SDnodeEp) + sizeof(SDnodeEps);
- if (memcmp(pDnode->dnodeEps, pEps, size) != 0) {
- dnodeResetDnodes(pEps);
- dnodeWriteDnodes();
+ int32_t size = pDnodeEps->num * sizeof(SDnodeEp) + sizeof(SDnodeEps);
+ if (memcmp(pDnd->d.dnodeEps, pDnodeEps, size) != 0) {
+ dndResetDnodes(pDnd, pDnodeEps);
+ dndWriteDnodes(pDnd);
}
}
- pthread_mutex_unlock(&pDnode->mutex);
+ dndUnLockDnode(pDnd);
}
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
+static void dndProcessStatusRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ if (pEpSet && pEpSet->numOfEps > 0) {
+ dndUpdateMnodeEpSet(pDnd, pEpSet);
+ }
+
if (pMsg->code != TSDB_CODE_SUCCESS) return;
SStatusRsp *pStatusRsp = pMsg->pCont;
-
- SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+ SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
pCfg->dnodeId = htonl(pCfg->dnodeId);
pCfg->clusterId = htobe64(pCfg->clusterId);
- dnodeUpdateCfg(pCfg);
+ dndUpdateDnodeCfg(pDnd, pCfg);
if (pCfg->dropped) return;
- SDnodeEps *pEps = &pStatusRsp->dnodeEps;
- pEps->dnodeNum = htonl(pEps->dnodeNum);
- for (int32_t i = 0; i < pEps->dnodeNum; ++i) {
- pEps->dnodeEps[i].dnodeId = htonl(pEps->dnodeEps[i].dnodeId);
- pEps->dnodeEps[i].dnodePort = htons(pEps->dnodeEps[i].dnodePort);
+ SDnodeEps *pDnodeEps = &pStatusRsp->dnodeEps;
+ pDnodeEps->num = htonl(pDnodeEps->num);
+ for (int32_t i = 0; i < pDnodeEps->num; ++i) {
+ pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id);
+ pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port);
}
- dnodeUpdateDnodeEps(pEps);
+ dndUpdateDnodeEps(pDnd, pDnodeEps);
}
-static void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) {
+static void dndProcessAuthRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
+
+static void dndProcessGrantRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
+
+static void dndProcessConfigDnodeReq(SDnode *pDnd, SRpcMsg *pMsg) {
SCfgDnodeMsg *pCfg = pMsg->pCont;
- int32_t code = taosCfgDynamicOptions(pCfg->config);
+ int32_t code = TSDB_CODE_OPS_NOT_SUPPORT;
SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code};
rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont);
}
-static void dnodeProcessStartupReq(SRpcMsg *pMsg) {
- dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont);
+static void dndProcessStartupReq(SDnode *pDnd, SRpcMsg *pMsg) {
+ dInfo("startup msg is received");
SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg));
- dnodeGetStartup(NULL, pStartup);
+ dndGetStartup(pDnd, pStartup);
dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
@@ -447,108 +463,120 @@ static void dnodeProcessStartupReq(SRpcMsg *pMsg) {
}
static void *dnodeThreadRoutine(void *param) {
- int32_t ms = tsStatusInterval * 1000;
+ SDnode *pDnd = param;
+ int32_t ms = pDnd->opt.statusInterval * 1000;
- while (!pDnode->threadStop) {
- if (dnodeGetStat() != DN_STAT_RUNNING) {
- continue;
- } else {
- dnodeSendStatusMsg();
- }
+ while (true) {
taosMsleep(ms);
+ if (dndGetStat(pDnd) != DND_STAT_RUNNING) {
+ continue;
+ }
+
+ pthread_testcancel();
+ dndSendStatusMsg(pDnd);
}
}
-int32_t dnodeInitDnode(SDnode *pServer) {
- SDnodeDnode *pDnode = &pServer->dnode;
+int32_t dndInitDnode(SDnode *pDnd) {
+ SDnodeMgmt *pDnode = &pDnd->d;
+
+ pDnode->dnodeId = 0;
+ pDnode->rebootTime = taosGetTimestampSec();
+ pDnode->dropped = 0;
+ pDnode->clusterId = 0;
char path[PATH_MAX];
- snprintf(path, PATH_MAX, "%s/dnode.json", pServer->dir.dnode);
+ snprintf(path, PATH_MAX, "%s/dnode.json", pDnd->dir.dnode);
pDnode->file = strdup(path);
if (pDnode->file == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
- pDnode->dnodeId = 0;
- pDnode->clusterId = 0;
- pDnode->dnodeEps = NULL;
-
- pDnode->rebootTime = taosGetTimestampSec();
- pDnode->dropped = 0;
- pthread_mutex_init(&pDnode->mutex, NULL);
- pDnode->threadStop = false;
-
pDnode->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (pDnode->dnodeHash == NULL) {
dError("failed to init dnode hash");
- return TSDB_CODE_DND_OUT_OF_MEMORY;
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
}
- pDnode->threadId = taosCreateThread(dnodeThreadRoutine, NULL);
+ if (dndReadDnodes(pDnd) != 0) {
+ dError("failed to read file:%s since %s", pDnode->file, terrstr());
+ return -1;
+ }
+
+ pthread_mutex_init(&pDnode->mutex, NULL);
+
+ pDnode->threadId = taosCreateThread(dnodeThreadRoutine, pDnd);
if (pDnode->threadId == NULL) {
dError("failed to init dnode thread");
- return TSDB_CODE_DND_OUT_OF_MEMORY;
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
}
- int32_t code = dnodeReadDnodes();
- if (code != 0) {
- dError("failed to read file:%s since %s", pDnode->file, tstrerror(code));
- return code;
- }
-
- dInfo("dnode-dnode is initialized");
+ dInfo("dnd-dnode is initialized");
return 0;
}
-void dnodeCleanupDnode(SDnode *pServer) {
- SDnodeDnode *pDnode = &pServer->dnode;
+void dndCleanupDnode(SDnode *pDnd) {
+ SDnodeMgmt *pDnode = &pDnd->d;
if (pDnode->threadId != NULL) {
- pDnode->threadStop = true;
taosDestoryThread(pDnode->threadId);
pDnode->threadId = NULL;
}
- pthread_mutex_lock(&pDnode->mutex);
+ dndLockDnode(pDnd);
if (pDnode->dnodeEps != NULL) {
free(pDnode->dnodeEps);
pDnode->dnodeEps = NULL;
}
- if (pDnode->dnodeHash) {
+ if (pDnode->dnodeHash != NULL) {
taosHashCleanup(pDnode->dnodeHash);
pDnode->dnodeHash = NULL;
}
- pthread_mutex_unlock(&pDnode->mutex);
- pthread_mutex_destroy(&pDnode->mutex);
-
- dInfo("dnode-dnode is cleaned up");
-}
-
-void dnodeProcessDnodeMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
- int32_t msgType = pMsg->msgType;
-
- if (msgType == TSDB_MSG_TYPE_STATUS_RSP && pEpSet) {
- dnodeUpdateMnodeEpSet(&pDnode->dnode, pEpSet);
+ if (pDnode->file != NULL) {
+ free(pDnode->file);
+ pDnode->file = NULL;
}
- switch (msgType) {
+ dndUnLockDnode(pDnd);
+ pthread_mutex_destroy(&pDnode->mutex);
+
+ dInfo("dnd-dnode is cleaned up");
+}
+
+void dndProcessDnodeReq(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ switch (pMsg->msgType) {
case TSDB_MSG_TYPE_NETWORK_TEST:
- dnodeProcessStartupReq(pMsg);
+ dndProcessStartupReq(pDnd, pMsg);
break;
case TSDB_MSG_TYPE_CONFIG_DNODE_IN:
- dnodeProcessConfigDnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_STATUS_RSP:
- dnodeProcessStatusRsp(pMsg);
+ dndProcessConfigDnodeReq(pDnd, pMsg);
break;
default:
- dError("RPC %p, %s not processed", pMsg->handle, taosMsg[msgType]);
- SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_MSG_NOT_PROCESSED};
+ dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+ SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
rpcSendResponse(&rspMsg);
rpcFreeCont(pMsg->pCont);
}
}
+
+void dndProcessDnodeRsp(SDnode *pDnd, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ switch (pMsg->msgType) {
+ case TSDB_MSG_TYPE_STATUS_RSP:
+ dndProcessStatusRsp(pDnd, pMsg, pEpSet);
+ break;
+ case TSDB_MSG_TYPE_AUTH_RSP:
+ dndProcessAuthRsp(pDnd, pMsg, pEpSet);
+ break;
+ case TSDB_MSG_TYPE_GRANT_RSP:
+ dndProcessGrantRsp(pDnd, pMsg, pEpSet);
+ break;
+ default:
+ dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+ }
+}
diff --git a/source/dnode/mgmt/impl/src/dnodeInt.c b/source/dnode/mgmt/impl/src/dndInt.c
similarity index 56%
rename from source/dnode/mgmt/impl/src/dnodeInt.c
rename to source/dnode/mgmt/impl/src/dndInt.c
index 8641a54def..9e77da61b4 100644
--- a/source/dnode/mgmt/impl/src/dnodeInt.c
+++ b/source/dnode/mgmt/impl/src/dndInt.c
@@ -14,50 +14,47 @@
*/
#define _DEFAULT_SOURCE
-#include "dnodeDnode.h"
-#include "dnodeMnode.h"
-#include "dnodeTransport.h"
-#include "dnodeVnodes.h"
+#include "dndDnode.h"
+#include "dndMnode.h"
+#include "dndTransport.h"
+#include "dndVnodes.h"
#include "sync.h"
#include "tcache.h"
-#include "tconfig.h"
-#include "tnote.h"
-#include "tstep.h"
#include "wal.h"
-EStat dnodeGetStat(SDnode *pDnode) { return pDnode->stat; }
+EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
-void dnodeSetStat(SDnode *pDnode, EStat stat) {
- dDebug("dnode stat set from %s to %s", dnodeStatStr(pDnode->stat), dnodeStatStr(stat));
+void dndSetStat(SDnode *pDnode, EStat stat) {
+ dDebug("dnode stat set from %s to %s", dndStatStr(pDnode->stat), dndStatStr(stat));
pDnode->stat = stat;
}
-char *dnodeStatStr(EStat stat) {
+char *dndStatStr(EStat stat) {
switch (stat) {
- case DN_STAT_INIT:
+ case DND_STAT_INIT:
return "init";
- case DN_STAT_RUNNING:
+ case DND_STAT_RUNNING:
return "running";
- case DN_STAT_STOPPED:
+ case DND_STAT_STOPPED:
return "stopped";
default:
return "unknown";
}
}
-void dnodeReportStartup(SDnode *pDnode, char *name, char *desc) {
+void dndReportStartup(SDnode *pDnode, char *name, char *desc) {
SStartupMsg *pStartup = &pDnode->startup;
tstrncpy(pStartup->name, name, strlen(pStartup->name));
tstrncpy(pStartup->desc, desc, strlen(pStartup->desc));
pStartup->finished = 0;
}
-void dnodeGetStartup(SDnode *pDnode, SStartupMsg *pStartup) {
- memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg);
- pStartup->finished = (dnodeGetStat(pDnode) == DN_STAT_RUNNING);
+void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) {
+ memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg));
+ pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
}
-static int32_t dnodeCheckRunning(char *dataDir) {
+static int32_t dndCheckRunning(char *dataDir) {
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
@@ -79,15 +76,19 @@ static int32_t dnodeCheckRunning(char *dataDir) {
return 0;
}
-static int32_t dnodeInitDisk(SDnode *pDnode, char *dataDir) {
- char path[PATH_MAX];
- snprintf(path, PATH_MAX, "%s/mnode", dataDir);
+static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOptions) {
+ if (dndCheckRunning(pOptions->dataDir) != 0) {
+ return -1;
+ }
+
+ char path[PATH_MAX + 100];
+ snprintf(path, sizeof(path), "%s%smnode", pOptions->dataDir, TD_DIRSEP);
pDnode->dir.mnode = strdup(path);
- sprintf(path, PATH_MAX, "%s/vnode", dataDir);
+ snprintf(path, sizeof(path), "%s%svnode", pOptions->dataDir, TD_DIRSEP);
pDnode->dir.vnodes = strdup(path);
- sprintf(path, PATH_MAX, "%s/dnode", dataDir);
+ snprintf(path, sizeof(path), "%s%sdnode", pOptions->dataDir, TD_DIRSEP);
pDnode->dir.dnode = strdup(path);
if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) {
@@ -114,55 +115,10 @@ static int32_t dnodeInitDisk(SDnode *pDnode, char *dataDir) {
return -1;
}
- if (dnodeCheckRunning(dataDir) != 0) {
- return -1;
- }
-
return 0;
}
-static int32_t dnodeInitEnv(SDnode *pDnode, const char *cfgPath) {
- taosIgnSIGPIPE();
- taosBlockSIGPIPE();
- taosResolveCRC();
- taosInitGlobalCfg();
- taosReadGlobalLogCfg();
- taosSetCoreDump(tsEnableCoreFile);
-
- if (!taosMkDir(tsLogDir)) {
- printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
- return -1;
- }
-
- char temp[TSDB_FILENAME_LEN];
- sprintf(temp, "%s/taosdlog", tsLogDir);
- if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) {
- dError("failed to init log file\n");
- return -1;
- }
-
- if (!taosReadGlobalCfg()) {
- taosPrintGlobalCfg();
- dError("TDengine read global config failed");
- return -1;
- }
-
- taosInitNotes();
-
- if (taosCheckGlobalCfg() != 0) {
- dError("TDengine check global config failed");
- return -1;
- }
-
- if (dnodeInitDisk(pDnode, tsDataDir) != 0) {
- dError("TDengine failed to init directory");
- return -1;
- }
-
- return 0;
-}
-
-static void dnodeCleanupEnv(SDnode *pDnode) {
+static void dndCleanupEnv(SDnode *pDnode) {
if (pDnode->dir.mnode != NULL) {
tfree(pDnode->dir.mnode);
}
@@ -175,11 +131,10 @@ static void dnodeCleanupEnv(SDnode *pDnode) {
tfree(pDnode->dir.dnode);
}
- taosCloseLog();
taosStopCacheRefreshWorker();
}
-SDnode *dnodeInit(const char *cfgPath) {
+SDnode *dndInit(SDnodeOpt *pOptions) {
SDnode *pDnode = calloc(1, sizeof(pDnode));
if (pDnode == NULL) {
dError("failed to create dnode object");
@@ -188,73 +143,73 @@ SDnode *dnodeInit(const char *cfgPath) {
}
dInfo("start to initialize TDengine");
- dnodeSetStat(pDnode, DN_STAT_INIT);
+ dndSetStat(pDnode, DND_STAT_INIT);
- if (dnodeInitEnv(pDnode, cfgPath) != 0) {
+ if (dndInitEnv(pDnode, pOptions) != 0) {
dError("failed to init env");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
if (rpcInit() != 0) {
dError("failed to init rpc env");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
if (walInit() != 0) {
dError("failed to init wal env");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
- if (dnodeInitDnode(pDnode) != 0) {
+ if (dndInitDnode(pDnode) != 0) {
dError("failed to init dnode");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
- if (dnodeInitVnodes(pDnode) != 0) {
+ if (dndInitVnodes(pDnode) != 0) {
dError("failed to init vnodes");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
- if (dnodeInitMnode(pDnode) != 0) {
+ if (dndInitMnode(pDnode) != 0) {
dError("failed to init mnode");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
- if (dnodeInitTrans(pDnode) != 0) {
+ if (dndInitTrans(pDnode) != 0) {
dError("failed to init transport");
- dnodeCleanup(pDnode);
+ dndCleanup(pDnode);
return NULL;
}
- dnodeSetStat(pDnode, DN_STAT_RUNNING);
- dnodeReportStartup(pDnode, "TDengine", "initialized successfully");
+ dndSetStat(pDnode, DND_STAT_RUNNING);
+ dndReportStartup(pDnode, "TDengine", "initialized successfully");
dInfo("TDengine is initialized successfully");
return 0;
}
-void dnodeCleanup(SDnode *pDnode) {
- if (dnodeGetStat(pDnode) == DN_STAT_STOPPED) {
+void dndCleanup(SDnode *pDnode) {
+ if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
dError("dnode is shutting down");
return;
}
dInfo("start to cleanup TDengine");
- dnodeSetStat(pDnode, DN_STAT_STOPPED);
- dnodeCleanupTrans(pDnode);
- dnodeCleanupMnode(pDnode);
- dnodeCleanupVnodes(pDnode);
- dnodeCleanupDnode(pDnode);
+ dndSetStat(pDnode, DND_STAT_STOPPED);
+ dndCleanupTrans(pDnode);
+ dndCleanupMnode(pDnode);
+ dndCleanupVnodes(pDnode);
+ dndCleanupDnode(pDnode);
walCleanUp();
rpcCleanup();
dInfo("TDengine is cleaned up successfully");
- dnodeCleanupEnv(pDnode);
+ dndCleanupEnv(pDnode);
free(pDnode);
}
diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c
new file mode 100644
index 0000000000..5f3e48d8a1
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndMnode.c
@@ -0,0 +1,816 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndMnode.h"
+#include "dndDnode.h"
+#include "dndTransport.h"
+
+static int32_t dndInitMnodeReadWorker(SDnode *pDnode);
+static int32_t dndInitMnodeWriteWorker(SDnode *pDnode);
+static int32_t dndInitMnodeSyncWorker(SDnode *pDnode);
+static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode);
+static void dndCleanupMnodeReadWorker(SDnode *pDnode);
+static void dndCleanupMnodeWriteWorker(SDnode *pDnode);
+static void dndCleanupMnodeSyncWorker(SDnode *pDnode);
+static void dndCleanupMnodeMgmtWorker(SDnode *pDnode);
+static int32_t dndAllocMnodeReadQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode);
+static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode);
+static void dndFreeMnodeReadQueue(SDnode *pDnode);
+static void dndFreeMnodeWriteQueue(SDnode *pDnode);
+static void dndFreeMnodeApplyQueue(SDnode *pDnode);
+static void dndFreeMnodeSyncQueue(SDnode *pDnode);
+static void dndFreeMnodeMgmtQueue(SDnode *pDnode);
+
+static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
+static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg);
+void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg);
+
+static int32_t dndStartMnodeWorker(SDnode *pDnode);
+static void dndStopMnodeWorker(SDnode *pDnode);
+
+static SMnode *dndAcquireMnode(SDnode *pDnode);
+static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode);
+
+static int32_t dndReadMnodeFile(SDnode *pDnode);
+static int32_t dndWriteMnodeFile(SDnode *pDnode);
+
+static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions);
+static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions);
+static int32_t dndDropMnode(SDnode *pDnode);
+
+static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
+
+static SMnode *dndAcquireMnode(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SMnode *pMnode = NULL;
+ int32_t refCount = 0;
+
+ taosRLockLatch(&pMgmt->latch);
+ if (pMgmt->deployed && !pMgmt->dropped) {
+ refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
+ pMnode = pMgmt->pMnode;
+ } else {
+ terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
+ }
+ taosRUnLockLatch(&pMgmt->latch);
+
+ dTrace("acquire mnode, refCount:%d", refCount);
+ return pMnode;
+}
+
+static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ int32_t refCount = 0;
+
+ taosRLockLatch(&pMgmt->latch);
+ if (pMnode != NULL) {
+ refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
+ }
+ taosRUnLockLatch(&pMgmt->latch);
+
+ dTrace("release mnode, refCount:%d", refCount);
+}
+
+static int32_t dndReadMnodeFile(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ int32_t code = TSDB_CODE_DND_MNODE_READ_FILE_ERROR;
+ int32_t len = 0;
+ int32_t maxLen = 300;
+ char *content = calloc(1, maxLen + 1);
+ cJSON *root = NULL;
+
+ FILE *fp = fopen(pMgmt->file, "r");
+ if (fp == NULL) {
+ dDebug("file %s not exist", pMgmt->file);
+ code = 0;
+ goto PRASE_MNODE_OVER;
+ }
+
+ len = (int32_t)fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s since content is null", pMgmt->file);
+ goto PRASE_MNODE_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s since invalid json format", pMgmt->file);
+ goto PRASE_MNODE_OVER;
+ }
+
+ cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+ if (!deployed || deployed->type != cJSON_String) {
+ dError("failed to read %s since deployed not found", pMgmt->file);
+ goto PRASE_MNODE_OVER;
+ }
+ pMgmt->deployed = atoi(deployed->valuestring);
+
+ cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
+ if (!dropped || dropped->type != cJSON_String) {
+ dError("failed to read %s since dropped not found", pMgmt->file);
+ goto PRASE_MNODE_OVER;
+ }
+ pMgmt->dropped = atoi(dropped->valuestring);
+
+ code = 0;
+ dInfo("succcessed to read file %s", pMgmt->file);
+
+PRASE_MNODE_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ terrno = code;
+ return code;
+}
+
+static int32_t dndWriteMnodeFile(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ char file[PATH_MAX + 20] = {0};
+ snprintf(file, sizeof(file), "%s.bak", pMgmt->file);
+
+ FILE *fp = fopen(file, "w");
+ if (fp != NULL) {
+ terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
+ dError("failed to write %s since %s", file, terrstr());
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 300;
+ char *content = calloc(1, maxLen + 1);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"deployed\": \"%d\",\n", pMgmt->deployed);
+ len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pMgmt->dropped);
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ taosFsyncFile(fileno(fp));
+ fclose(fp);
+ free(content);
+
+ if (taosRenameFile(file, pMgmt->file) != 0) {
+ terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
+ dError("failed to rename %s since %s", pMgmt->file, terrstr());
+ return -1;
+ }
+
+ dInfo("successed to write %s", pMgmt->file);
+ return 0;
+}
+
+static int32_t dndStartMnodeWorker(SDnode *pDnode) {
+ if (dndAllocMnodeReadQueue(pDnode) != 0) {
+ dError("failed to alloc mnode read queue since %s", terrstr());
+ return -1;
+ }
+
+ if (dndAllocMnodeWriteQueue(pDnode) != 0) {
+ dError("failed to alloc mnode write queue since %s", terrstr());
+ return -1;
+ }
+
+ if (dndAllocMnodeApplyQueue(pDnode) != 0) {
+ dError("failed to alloc mnode apply queue since %s", terrstr());
+ return -1;
+ }
+
+ if (dndAllocMnodeSyncQueue(pDnode) != 0) {
+ dError("failed to alloc mnode sync queue since %s", terrstr());
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndStopMnodeWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ taosWLockLatch(&pMgmt->latch);
+ pMgmt->deployed = 0;
+ pMgmt->pMnode = NULL;
+ taosWUnLockLatch(&pMgmt->latch);
+
+ while (pMgmt->refCount > 1) taosMsleep(10);
+ while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pMgmt->pApplyQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10);
+
+ dndFreeMnodeReadQueue(pDnode);
+ dndFreeMnodeWriteQueue(pDnode);
+ dndFreeMnodeApplyQueue(pDnode);
+ dndFreeMnodeSyncQueue(pDnode);
+
+ dndCleanupMnodeReadWorker(pDnode);
+ dndCleanupMnodeWriteWorker(pDnode);
+ dndCleanupMnodeSyncWorker(pDnode);
+}
+
+static bool dndNeedDeployMnode(SDnode *pDnode) {
+ if (dndGetDnodeId(pDnode) > 0) {
+ return false;
+ }
+
+ if (dndGetClusterId(pDnode) > 0) {
+ return false;
+ }
+ if (strcmp(pDnode->opt.localEp, pDnode->opt.firstEp) != 0) {
+ return false;
+ }
+
+ return true;
+}
+
+static void dndInitMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions) {
+ pOptions->pDnode = pDnode;
+ pOptions->sendMsgToDnodeFp = dndSendMsgToDnode;
+ pOptions->sendMsgToMnodeFp = dndSendMsgToMnode;
+ pOptions->sendRedirectMsgFp = dndSendRedirectMsg;
+ pOptions->putMsgToApplyMsgFp = dndPutMsgIntoMnodeApplyQueue;
+}
+
+static int32_t dndBuildMnodeOptions(SDnode *pDnode, SMnodeOptions *pOptions, SCreateMnodeMsg *pMsg) {
+ dndInitMnodeOptions(pDnode, pOptions);
+
+ if (pMsg == NULL) {
+ pOptions->dnodeId = 1;
+ pOptions->clusterId = 1234;
+ pOptions->replica = 1;
+ pOptions->selfIndex = 0;
+ SReplica *pReplica = &pOptions->replicas[0];
+ pReplica->id = 1;
+ pReplica->port = pDnode->opt.serverPort;
+ tstrncpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN);
+ } else {
+ pOptions->dnodeId = dndGetDnodeId(pDnode);
+ pOptions->clusterId = dndGetClusterId(pDnode);
+ pOptions->selfIndex = -1;
+ pOptions->replica = pMsg->replica;
+ for (int32_t index = 0; index < pMsg->replica; ++index) {
+ SReplica *pReplica = &pOptions->replicas[index];
+ pReplica->id = pMsg->replicas[index].id;
+ pReplica->port = pMsg->replicas[index].port;
+ tstrncpy(pReplica->fqdn, pMsg->replicas[index].fqdn, TSDB_FQDN_LEN);
+ if (pReplica->id == pOptions->dnodeId) {
+ pOptions->selfIndex = index;
+ }
+ }
+ }
+
+ if (pOptions->selfIndex == -1) {
+ terrno = TSDB_CODE_DND_MNODE_ID_NOT_FOUND;
+ dError("failed to build mnode options since %s", terrstr());
+ return -1;
+ }
+
+ return 0;
+}
+
+static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOptions *pOptions) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ int32_t code = dndStartMnodeWorker(pDnode);
+ if (code != 0) {
+ dError("failed to start mnode worker since %s", terrstr());
+ return code;
+ }
+
+ SMnode *pMnode = mnodeOpen(pDnode->dir.mnode, pOptions);
+ if (pMnode == NULL) {
+ dError("failed to open mnode since %s", terrstr());
+ code = terrno;
+ dndStopMnodeWorker(pDnode);
+ terrno = code;
+ return code;
+ }
+
+ if (dndWriteMnodeFile(pDnode) != 0) {
+ dError("failed to write mnode file since %s", terrstr());
+ code = terrno;
+ dndStopMnodeWorker(pDnode);
+ mnodeClose(pMnode);
+ mnodeDestroy(pDnode->dir.mnode);
+ terrno = code;
+ return code;
+ }
+
+ taosWLockLatch(&pMgmt->latch);
+ pMgmt->pMnode = pMnode;
+ pMgmt->deployed = 1;
+ taosWUnLockLatch(&pMgmt->latch);
+
+ return 0;
+}
+
+static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOptions *pOptions) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL) {
+ dError("failed to alter mnode since %s", terrstr());
+ return -1;
+ }
+
+ if (mnodeAlter(pMnode, pOptions) != 0) {
+ dError("failed to alter mnode since %s", terrstr());
+ dndReleaseMnode(pDnode, pMnode);
+ return -1;
+ }
+
+ dndReleaseMnode(pDnode, pMnode);
+ return 0;
+}
+
+static int32_t dndDropMnode(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL) {
+ dError("failed to drop mnode since %s", terrstr());
+ return -1;
+ }
+
+ taosRLockLatch(&pMgmt->latch);
+ pMgmt->dropped = 1;
+ taosRUnLockLatch(&pMgmt->latch);
+
+ if (dndWriteMnodeFile(pDnode) != 0) {
+ taosRLockLatch(&pMgmt->latch);
+ pMgmt->dropped = 0;
+ taosRUnLockLatch(&pMgmt->latch);
+
+ dndReleaseMnode(pDnode, pMnode);
+ dError("failed to drop mnode since %s", terrstr());
+ return -1;
+ }
+
+ dndStopMnodeWorker(pDnode);
+ dndWriteMnodeFile(pDnode);
+ mnodeClose(pMnode);
+ mnodeDestroy(pDnode->dir.mnode);
+
+ return 0;
+}
+
+static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) {
+ SCreateMnodeMsg *pMsg = pRpcMsg->pCont;
+ pMsg->dnodeId = htonl(pMsg->dnodeId);
+ for (int32_t i = 0; i < pMsg->replica; ++i) {
+ pMsg->replicas[i].id = htonl(pMsg->replicas[i].id);
+ pMsg->replicas[i].port = htons(pMsg->replicas[i].port);
+ }
+
+ return pMsg;
+}
+
+static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+ SCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+ if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+ terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+ return -1;
+ } else {
+ SMnodeOptions option = {0};
+ if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) {
+ return -1;
+ }
+ return dndOpenMnode(pDnode, &option);
+ }
+}
+
+static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+ SAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+ if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+ terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+ return -1;
+ } else {
+ SMnodeOptions option = {0};
+ if (dndBuildMnodeOptions(pDnode, &option, pMsg) != 0) {
+ return -1;
+ }
+ return dndAlterMnode(pDnode, &option);
+ }
+}
+
+static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
+ SDropMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
+
+ if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
+ terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
+ return -1;
+ } else {
+ return dndDropMnode(pDnode);
+ }
+}
+
+static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+ int32_t code = 0;
+
+ switch (pMsg->msgType) {
+ case TSDB_MSG_TYPE_CREATE_MNODE_IN:
+ code = dndProcessCreateMnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_ALTER_MNODE_IN:
+ code = dndProcessAlterMnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_DROP_MNODE_IN:
+ code = dndProcessDropMnodeReq(pDnode, pMsg);
+ break;
+ default:
+ code = TSDB_CODE_MSG_NOT_PROCESSED;
+ break;
+ }
+
+ SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ taosFreeQitem(pMsg);
+}
+
+static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode != NULL) {
+ mnodeProcessReadMsg(pMnode, pMsg);
+ dndReleaseMnode(pDnode, pMnode);
+ } else {
+ mnodeSendRsp(pMsg, terrno);
+ }
+
+ mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode != NULL) {
+ mnodeProcessWriteMsg(pMnode, pMsg);
+ dndReleaseMnode(pDnode, pMnode);
+ } else {
+ mnodeSendRsp(pMsg, terrno);
+ }
+
+ mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode != NULL) {
+ mnodeProcessApplyMsg(pMnode, pMsg);
+ dndReleaseMnode(pDnode, pMnode);
+ } else {
+ mnodeSendRsp(pMsg, terrno);
+ }
+
+ mnodeCleanupMsg(pMsg);
+}
+
+static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode != NULL) {
+ mnodeProcessSyncMsg(pMnode, pMsg);
+ dndReleaseMnode(pDnode, pMnode);
+ } else {
+ mnodeSendRsp(pMsg, terrno);
+ }
+
+ mnodeCleanupMsg(pMsg);
+}
+
+static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
+ assert(pQueue);
+
+ SMnodeMsg *pMsg = mnodeInitMsg(pMnode, pRpcMsg);
+ if (pMsg == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if (taosWriteQitem(pQueue, pMsg) != 0) {
+ mnodeCleanupMsg(pMsg);
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+
+ SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
+ if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) {
+ SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pRpcMsg->pCont);
+ taosFreeQitem(pMsg);
+ }
+}
+
+void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ }
+
+ dndReleaseMnode(pDnode, pMnode);
+}
+
+void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ }
+
+ dndReleaseMnode(pDnode, pMnode);
+}
+
+void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ }
+
+ dndReleaseMnode(pDnode, pMnode);
+}
+
+static int32_t dndPutMsgIntoMnodeApplyQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL) {
+ return -1;
+ }
+
+ int32_t code = taosWriteQitem(pMgmt->pApplyQ, pMsg);
+ dndReleaseMnode(pDnode, pMnode);
+ return code;
+}
+
+static int32_t dndAllocMnodeMgmtQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ pMgmt->pMgmtQ = tWorkerAllocQueue(&pMgmt->mgmtPool, NULL, (FProcessItem)dndProcessMnodeMgmtQueue);
+ if (pMgmt->pMgmtQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeMnodeMgmtQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ);
+ pMgmt->pMgmtQ = NULL;
+}
+
+static int32_t dndInitMnodeMgmtWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SWorkerPool *pPool = &pMgmt->mgmtPool;
+ pPool->name = "mnode-mgmt";
+ pPool->min = 1;
+ pPool->max = 1;
+ if (tWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupMnodeMgmtWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ ;
+ tWorkerCleanup(&pMgmt->mgmtPool);
+}
+
+static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, NULL, (FProcessItem)dndProcessMnodeReadQueue);
+ if (pMgmt->pReadQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeMnodeReadQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ);
+ pMgmt->pReadQ = NULL;
+}
+
+static int32_t dndInitMnodeReadWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SWorkerPool *pPool = &pMgmt->readPool;
+ pPool->name = "mnode-read";
+ pPool->min = 0;
+ pPool->max = 1;
+ if (tWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupMnodeReadWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerCleanup(&pMgmt->readPool);
+}
+
+static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeWriteQueue);
+ if (pMgmt->pWriteQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeMnodeWriteQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ);
+ pMgmt->pWriteQ = NULL;
+}
+
+static int32_t dndAllocMnodeApplyQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ pMgmt->pApplyQ = tWorkerAllocQueue(&pMgmt->writePool, NULL, (FProcessItem)dndProcessMnodeApplyQueue);
+ if (pMgmt->pApplyQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeMnodeApplyQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pApplyQ);
+ pMgmt->pApplyQ = NULL;
+}
+
+static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SWorkerPool *pPool = &pMgmt->writePool;
+ pPool->name = "mnode-write";
+ pPool->min = 0;
+ pPool->max = 1;
+ if (tWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupMnodeWriteWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerCleanup(&pMgmt->writePool);
+}
+
+static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, NULL, (FProcessItem)dndProcessMnodeSyncQueue);
+ if (pMgmt->pSyncQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeMnodeSyncQueue(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ);
+ pMgmt->pSyncQ = NULL;
+}
+
+static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ SWorkerPool *pPool = &pMgmt->syncPool;
+ pPool->name = "mnode-sync";
+ pPool->min = 0;
+ pPool->max = 1;
+ return tWorkerInit(pPool);
+}
+
+static void dndCleanupMnodeSyncWorker(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ tWorkerCleanup(&pMgmt->syncPool);
+}
+
+int32_t dndInitMnode(SDnode *pDnode) {
+ dInfo("dnode-mnode start to init");
+ SMnodeMgmt *pMgmt = &pDnode->m;
+ taosInitRWLatch(&pMgmt->latch);
+
+ if (dndInitMnodeMgmtWorker(pDnode) != 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ char path[PATH_MAX];
+ snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode);
+ pMgmt->file = strdup(path);
+ if (pMgmt->file == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ if (dndReadMnodeFile(pDnode) != 0) {
+ return -1;
+ }
+
+ if (pMgmt->dropped) {
+ dInfo("mnode has been deployed and needs to be deleted");
+ mnodeDestroy(pDnode->dir.mnode);
+ return 0;
+ }
+
+ if (!pMgmt->deployed) {
+ bool needDeploy = dndNeedDeployMnode(pDnode);
+ if (!needDeploy) {
+ dDebug("mnode does not need to be deployed");
+ return 0;
+ }
+
+ dInfo("start to deploy mnode");
+ } else {
+ dInfo("start to open mnode");
+ }
+
+ SMnodeOptions option = {0};
+ dndInitMnodeOptions(pDnode, &option);
+ return dndOpenMnode(pDnode, &option);
+}
+
+void dndCleanupMnode(SDnode *pDnode) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ dInfo("dnode-mnode start to clean up");
+ dndStopMnodeWorker(pDnode);
+ dndCleanupMnodeMgmtWorker(pDnode);
+ tfree(pMgmt->file);
+ dInfo("dnode-mnode is cleaned up");
+}
+
+int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+ SMnodeMgmt *pMgmt = &pDnode->m;
+
+ SMnode *pMnode = dndAcquireMnode(pDnode);
+ if (pMnode == NULL) {
+ terrno = TSDB_CODE_APP_NOT_READY;
+ dTrace("failed to get user auth since %s", terrstr());
+ return -1;
+ }
+
+ int32_t code = mnodeRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
+ dndReleaseMnode(pDnode, pMnode);
+ return code;
+}
diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c
new file mode 100644
index 0000000000..679e3ef5f9
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndTransport.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/* this file is mainly responsible for the communication between DNODEs. Each
+ * dnode works as both server and client. Dnode may send status, grant, config
+ * messages to mnode, mnode may send create/alter/drop table/vnode messages
+ * to dnode. All theses messages are handled from here
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndTransport.h"
+#include "dndDnode.h"
+#include "dndMnode.h"
+#include "dndVnodes.h"
+
+static void dndInitMsgFp(STransMgmt *pMgmt) {
+ // msg from client to dnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_SUBMIT] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_QUERY] = dndProcessVnodeQueryMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_FETCH] = dndProcessVnodeFetchMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessVnodeQueryMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg;
+
+ // msg from client to mnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_USER] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_USER] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_USER] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_DB] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_DB] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg;
+
+ // message from client to dnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq;
+
+ // message from mnode to vnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+
+ // message from mnode to dnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dndProcessVnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dndProcessMnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dndProcessMnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dndProcessMnodeMgmtMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dndProcessDnodeReq;
+ pMgmt->msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dndProcessMnodeWriteMsg;
+
+ // message from dnode to mnode
+ pMgmt->msgFp[TSDB_MSG_TYPE_AUTH] = dndProcessMnodeReadMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dndProcessDnodeRsp;
+ pMgmt->msgFp[TSDB_MSG_TYPE_GRANT] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dndProcessDnodeRsp;
+ pMgmt->msgFp[TSDB_MSG_TYPE_STATUS] = dndProcessMnodeWriteMsg;
+ pMgmt->msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dndProcessDnodeRsp;
+}
+
+static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SDnode *pDnode = parent;
+ STransMgmt *pMgmt = &pDnode->t;
+
+ int32_t msgType = pMsg->msgType;
+
+ if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
+ if (pMsg == NULL || pMsg->pCont == NULL) return;
+ dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]);
+ rpcFreeCont(pMsg->pCont);
+ return;
+ }
+
+ DndMsgFp fp = pMgmt->msgFp[msgType];
+ if (fp != NULL) {
+ dTrace("RPC %p, rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code));
+ (*fp)(pDnode, pMsg, pEpSet);
+ } else {
+ dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
+ rpcFreeCont(pMsg->pCont);
+ }
+}
+
+static int32_t dndInitClient(SDnode *pDnode) {
+ STransMgmt *pMgmt = &pDnode->t;
+
+ SRpcInit rpcInit;
+ memset(&rpcInit, 0, sizeof(rpcInit));
+ rpcInit.label = "DND-C";
+ rpcInit.numOfThreads = 1;
+ rpcInit.cfp = dndProcessResponse;
+ rpcInit.sessions = TSDB_MAX_VNODES << 4;
+ rpcInit.connType = TAOS_CONN_CLIENT;
+ rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000;
+ rpcInit.user = "-internal";
+ rpcInit.ckey = "-key";
+ rpcInit.secret = "-secret";
+
+ pMgmt->clientRpc = rpcOpen(&rpcInit);
+ if (pMgmt->clientRpc == NULL) {
+ dError("failed to init rpc client");
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupClient(SDnode *pDnode) {
+ STransMgmt *pMgmt = &pDnode->t;
+ if (pMgmt->clientRpc) {
+ rpcClose(pMgmt->clientRpc);
+ pMgmt->clientRpc = NULL;
+ dInfo("dnode peer rpc client is closed");
+ }
+}
+
+static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SDnode *pDnode = param;
+ STransMgmt *pMgmt = &pDnode->t;
+
+ int32_t msgType = pMsg->msgType;
+ if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
+ dndProcessDnodeReq(pDnode, pMsg, pEpSet);
+ return;
+ }
+
+ if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
+ dError("RPC %p, req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]);
+ SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_EXITING};
+ rpcSendResponse(&rspMsg);
+ rpcFreeCont(pMsg->pCont);
+ return;
+ } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) {
+ dError("RPC %p, req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
+ SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY};
+ rpcSendResponse(&rspMsg);
+ rpcFreeCont(pMsg->pCont);
+ return;
+ }
+
+ if (pMsg->pCont == NULL) {
+ SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN};
+ rpcSendResponse(&rspMsg);
+ return;
+ }
+
+ DndMsgFp fp = pMgmt->msgFp[msgType];
+ if (fp != NULL) {
+ dTrace("RPC %p, req:%s will be processed", pMsg->handle, taosMsg[msgType]);
+ (*fp)(pDnode, pMsg, pEpSet);
+ } else {
+ dError("RPC %p, req:%s is not processed", pMsg->handle, taosMsg[msgType]);
+ SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
+ rpcSendResponse(&rspMsg);
+ rpcFreeCont(pMsg->pCont);
+ }
+}
+
+static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
+ STransMgmt *pMgmt = &pDnode->t;
+
+ SEpSet epSet = {0};
+ dndGetMnodeEpSet(pDnode, &epSet);
+ rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
+}
+
+static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
+ SDnode *pDnode = parent;
+
+ if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) != 0) {
+ if (terrno != TSDB_CODE_APP_NOT_READY) {
+ dTrace("failed to get user auth from mnode since %s", terrstr());
+ return -1;
+ }
+ }
+
+ dDebug("user:%s, send auth msg to mnodes", user);
+
+ SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
+ tstrncpy(pMsg->user, user, TSDB_USER_LEN);
+
+ SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH};
+ SRpcMsg rpcRsp = {0};
+ dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
+
+ if (rpcRsp.code != 0) {
+ terrno = rpcRsp.code;
+ dError("user:%s, failed to get user auth from mnodes since %s", user, terrstr());
+ } else {
+ SAuthRsp *pRsp = rpcRsp.pCont;
+ memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
+ memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
+ *spi = pRsp->spi;
+ *encrypt = pRsp->encrypt;
+ dDebug("user:%s, success to get user auth from mnodes", user);
+ }
+
+ rpcFreeCont(rpcRsp.pCont);
+ return rpcRsp.code;
+}
+
+static int32_t dndInitServer(SDnode *pDnode) {
+ STransMgmt *pMgmt = &pDnode->t;
+ dndInitMsgFp(pMgmt);
+
+ int32_t numOfThreads = (int32_t)((pDnode->opt.numOfCores * pDnode->opt.numOfThreadsPerCore) / 2.0);
+ if (numOfThreads < 1) {
+ numOfThreads = 1;
+ }
+
+ SRpcInit rpcInit;
+ memset(&rpcInit, 0, sizeof(rpcInit));
+ rpcInit.localPort = pDnode->opt.serverPort;
+ rpcInit.label = "DND-S";
+ rpcInit.numOfThreads = numOfThreads;
+ rpcInit.cfp = dndProcessRequest;
+ rpcInit.sessions = pDnode->opt.maxShellConns;
+ rpcInit.connType = TAOS_CONN_SERVER;
+ rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000;
+ rpcInit.afp = dndRetrieveUserAuthInfo;
+
+ pMgmt->serverRpc = rpcOpen(&rpcInit);
+ if (pMgmt->serverRpc == NULL) {
+ dError("failed to init rpc server");
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupServer(SDnode *pDnode) {
+ STransMgmt *pMgmt = &pDnode->t;
+ if (pMgmt->serverRpc) {
+ rpcClose(pMgmt->serverRpc);
+ pMgmt->serverRpc = NULL;
+ }
+}
+
+int32_t dndInitTrans(SDnode *pDnode) {
+ if (dndInitClient(pDnode) != 0) {
+ return -1;
+ }
+
+ if (dndInitServer(pDnode) != 0) {
+ return -1;
+ }
+
+ dInfo("dnode-transport is initialized");
+ return 0;
+}
+
+void dndCleanupTrans(SDnode *pDnode) {
+ dndCleanupServer(pDnode);
+ dndCleanupClient(pDnode);
+ dInfo("dnode-transport is cleaned up");
+}
+
+void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pMsg) {
+ STransMgmt *pMgmt = &pDnode->t;
+ rpcSendRequest(pMgmt->clientRpc, pEpSet, pMsg, NULL);
+}
+
+void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) {
+ SEpSet epSet = {0};
+ dndGetMnodeEpSet(pDnode, &epSet);
+ dndSendMsgToDnode(pDnode, &epSet, pMsg);
+}
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c
new file mode 100644
index 0000000000..d5e94106a7
--- /dev/null
+++ b/source/dnode/mgmt/impl/src/dndVnodes.c
@@ -0,0 +1,1119 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndVnodes.h"
+#include "dndTransport.h"
+
+typedef struct {
+ int32_t vgId;
+ int32_t refCount;
+ int8_t dropped;
+ int8_t accessState;
+ SVnode *pImpl;
+ taos_queue pWriteQ;
+ taos_queue pSyncQ;
+ taos_queue pApplyQ;
+ taos_queue pQueryQ;
+ taos_queue pFetchQ;
+} SVnodeObj;
+
+typedef struct {
+ int32_t vnodeNum;
+ int32_t opened;
+ int32_t failed;
+ int32_t threadIndex;
+ pthread_t *pThreadId;
+ SVnodeObj *pVnodes;
+ SDnode *pDnode;
+} SVnodeThread;
+
+static int32_t dndInitVnodeReadWorker(SDnode *pDnode);
+static int32_t dndInitVnodeWriteWorker(SDnode *pDnode);
+static int32_t dndInitVnodeSyncWorker(SDnode *pDnode);
+static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode);
+static void dndCleanupVnodeReadWorker(SDnode *pDnode);
+static void dndCleanupVnodeWriteWorker(SDnode *pDnode);
+static void dndCleanupVnodeSyncWorker(SDnode *pDnode);
+static void dndCleanupVnodeMgmtWorker(SDnode *pDnode);
+static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode);
+static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode);
+
+static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
+static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
+static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
+static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg);
+void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
+static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg);
+
+static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId);
+static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl);
+static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode);
+static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes);
+static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes);
+static int32_t dndWriteVnodesToFile(SDnode *pDnode);
+
+static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg);
+static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode);
+static int32_t dndOpenVnodes(SDnode *pDnode);
+static void dndCloseVnodes(SDnode *pDnode);
+
+static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
+
+static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ SVnodeObj *pVnode = NULL;
+ int32_t refCount = 0;
+
+ taosRLockLatch(&pMgmt->latch);
+ taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
+ if (pVnode == NULL) {
+ terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+ } else {
+ refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+ }
+ taosRUnLockLatch(&pMgmt->latch);
+
+ dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+ return pVnode;
+}
+
+static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ int32_t refCount = 0;
+
+ taosRLockLatch(&pMgmt->latch);
+ if (pVnode != NULL) {
+ refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+ }
+ taosRUnLockLatch(&pMgmt->latch);
+
+ if (pVnode != NULL) {
+ dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
+ }
+}
+
+static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, SVnode *pImpl) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
+ if (pVnode == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ pVnode->vgId = vgId;
+ pVnode->refCount = 0;
+ pVnode->dropped = 0;
+ pVnode->accessState = TSDB_VN_ALL_ACCCESS;
+ pVnode->pImpl = pImpl;
+
+ if (dndAllocVnodeQueryQueue(pDnode, pVnode) != 0) {
+ return -1;
+ }
+
+ if (dndAllocVnodeFetchQueue(pDnode, pVnode) != 0) {
+ return -1;
+ }
+
+ if (dndAllocVnodeWriteQueue(pDnode, pVnode) != 0) {
+ return -1;
+ }
+
+ if (dndAllocVnodeApplyQueue(pDnode, pVnode) != 0) {
+ return -1;
+ }
+
+ if (dndAllocVnodeSyncQueue(pDnode, pVnode) != 0) {
+ return -1;
+ }
+
+ taosWLockLatch(&pMgmt->latch);
+ int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+ taosWUnLockLatch(&pMgmt->latch);
+
+ if (code != 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ }
+ return code;
+}
+
+static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ taosWLockLatch(&pMgmt->latch);
+ taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
+ taosWUnLockLatch(&pMgmt->latch);
+
+ dndReleaseVnode(pDnode, pVnode);
+ while (pVnode->refCount > 0) taosMsleep(10);
+ while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
+ while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+
+ dndFreeVnodeQueryQueue(pDnode, pVnode);
+ dndFreeVnodeFetchQueue(pDnode, pVnode);
+ dndFreeVnodeWriteQueue(pDnode, pVnode);
+ dndFreeVnodeApplyQueue(pDnode, pVnode);
+ dndFreeVnodeSyncQueue(pDnode, pVnode);
+}
+
+static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ taosRLockLatch(&pMgmt->latch);
+
+ int32_t num = 0;
+ int32_t size = taosHashGetSize(pMgmt->hash);
+ SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *));
+
+ void *pIter = taosHashIterate(pMgmt->hash, NULL);
+ while (pIter) {
+ SVnodeObj **ppVnode = pIter;
+ SVnodeObj *pVnode = *ppVnode;
+ if (pVnode) {
+ num++;
+ if (num < size) {
+ int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+ dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+ pVnodes[num] = (*ppVnode);
+ }
+ }
+ pIter = taosHashIterate(pMgmt->hash, pIter);
+ }
+
+ taosRUnLockLatch(&pMgmt->latch);
+ *numOfVnodes = num;
+
+ return pVnodes;
+}
+
+static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) {
+ int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR;
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char *content = calloc(1, maxLen + 1);
+ cJSON *root = NULL;
+ FILE *fp = NULL;
+ char file[PATH_MAX + 20] = {0};
+ SVnodeObj *pVnodes = NULL;
+
+ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
+
+ fp = fopen(file, "r");
+ if (!fp) {
+ dDebug("file %s not exist", file);
+ code = 0;
+ goto PRASE_VNODE_OVER;
+ }
+
+ len = (int32_t)fread(content, 1, maxLen, fp);
+ if (len <= 0) {
+ dError("failed to read %s since content is null", file);
+ goto PRASE_VNODE_OVER;
+ }
+
+ content[len] = 0;
+ root = cJSON_Parse(content);
+ if (root == NULL) {
+ dError("failed to read %s since invalid json format", file);
+ goto PRASE_VNODE_OVER;
+ }
+
+ cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes");
+ if (!vnodes || vnodes->type != cJSON_Array) {
+ dError("failed to read %s since vnodes not found", file);
+ goto PRASE_VNODE_OVER;
+ }
+
+ int32_t vnodesNum = cJSON_GetArraySize(vnodes);
+ if (vnodesNum <= 0) {
+ dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum);
+ goto PRASE_VNODE_OVER;
+ }
+
+ pVnodes = calloc(vnodesNum, sizeof(SVnodeObj));
+ if (pVnodes == NULL) {
+ dError("failed to read %s since out of memory", file);
+ goto PRASE_VNODE_OVER;
+ }
+
+ for (int32_t i = 0; i < vnodesNum; ++i) {
+ cJSON *vnode = cJSON_GetArrayItem(vnodes, i);
+ SVnodeObj *pVnode = &pVnodes[i];
+
+ cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
+ if (!vgId || vgId->type != cJSON_String) {
+ dError("failed to read %s since vgId not found", file);
+ goto PRASE_VNODE_OVER;
+ }
+ pVnode->vgId = atoi(vgId->valuestring);
+
+ cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
+ if (!dropped || dropped->type != cJSON_String) {
+ dError("failed to read %s since dropped not found", file);
+ goto PRASE_VNODE_OVER;
+ }
+ pVnode->dropped = atoi(vnode->valuestring);
+ }
+
+ code = 0;
+ dInfo("succcessed to read file %s", file);
+
+PRASE_VNODE_OVER:
+ if (content != NULL) free(content);
+ if (root != NULL) cJSON_Delete(root);
+ if (fp != NULL) fclose(fp);
+
+ return code;
+}
+
+static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
+ char file[PATH_MAX + 20] = {0};
+ char realfile[PATH_MAX + 20] = {0};
+ snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
+ snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
+
+ FILE *fp = fopen(file, "w");
+ if (fp != NULL) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ dError("failed to write %s since %s", file, terrstr());
+ return -1;
+ }
+
+ int32_t len = 0;
+ int32_t maxLen = 30000;
+ char *content = calloc(1, maxLen + 1);
+ int32_t numOfVnodes = 0;
+ SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
+
+ len += snprintf(content + len, maxLen - len, "{\n");
+ len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n");
+ for (int32_t i = 0; i < numOfVnodes; ++i) {
+ SVnodeObj *pVnode = pVnodes[i];
+ len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId);
+ len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped);
+ if (i < numOfVnodes - 1) {
+ len += snprintf(content + len, maxLen - len, " },{\n");
+ } else {
+ len += snprintf(content + len, maxLen - len, " }]\n");
+ }
+ }
+ len += snprintf(content + len, maxLen - len, "}\n");
+
+ fwrite(content, 1, len, fp);
+ taosFsyncFile(fileno(fp));
+ fclose(fp);
+ free(content);
+ terrno = 0;
+
+ for (int32_t i = 0; i < numOfVnodes; ++i) {
+ SVnodeObj *pVnode = pVnodes[i];
+ dndReleaseVnode(pDnode, pVnode);
+ }
+
+ if (pVnodes != NULL) {
+ free(pVnodes);
+ }
+
+ dInfo("successed to write %s", file);
+ return taosRenameFile(file, realfile);
+}
+
+static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) {
+ char path[PATH_MAX + 20] = {0};
+ snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId);
+ SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
+
+ if (pImpl == NULL) {
+ return -1;
+ }
+
+ int32_t code = dndCreateVnodeWrapper(pDnode, vgId, pImpl);
+ if (code != 0) {
+ vnodeDrop(pImpl);
+ terrno = code;
+ return code;
+ }
+
+ code = dndWriteVnodesToFile(pDnode);
+ if (code != 0) {
+ vnodeDrop(pImpl);
+ terrno = code;
+ return code;
+ }
+
+ return 0;
+}
+
+static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) {
+ pVnode->dropped = 1;
+ if (dndWriteVnodesToFile(pDnode) != 0) {
+ pVnode->dropped = 0;
+ return -1;
+ }
+
+ dndDropVnodeWrapper(pDnode, pVnode);
+ vnodeDrop(pVnode->pImpl);
+ dndWriteVnodesToFile(pDnode);
+ return 0;
+}
+
+static void *dnodeOpenVnodeFunc(void *param) {
+ SVnodeThread *pThread = param;
+ SDnode *pDnode = pThread->pDnode;
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+ dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+ setThreadName("open-vnodes");
+
+ for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+ SVnodeObj *pVnode = &pThread->pVnodes[v];
+
+ char stepDesc[TSDB_STEP_DESC_LEN] = {0};
+ snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId,
+ pMgmt->openVnodes, pMgmt->totalVnodes);
+ dndReportStartup(pDnode, "open-vnodes", stepDesc);
+
+ char path[PATH_MAX + 20] = {0};
+ snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId);
+ SVnode *pImpl = vnodeOpen(path, NULL);
+ if (pImpl == NULL) {
+ dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
+ pThread->failed++;
+ } else {
+ dndCreateVnodeWrapper(pDnode, pVnode->vgId, pImpl);
+ dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex);
+ pThread->opened++;
+ }
+
+ atomic_add_fetch_32(&pMgmt->openVnodes, 1);
+ }
+
+ dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+ pThread->failed);
+ return NULL;
+}
+
+static int32_t dndOpenVnodes(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ taosInitRWLatch(&pMgmt->latch);
+
+ pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+ if (pMgmt->hash == NULL) {
+ dError("failed to init vnode hash");
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ SVnodeObj *pVnodes = NULL;
+ int32_t numOfVnodes = 0;
+ if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) {
+ dInfo("failed to get vnode list from disk since %s", terrstr());
+ return -1;
+ }
+
+ pMgmt->totalVnodes = numOfVnodes;
+
+ int32_t threadNum = tsNumOfCores;
+ int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+
+ SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
+ for (int32_t t = 0; t < threadNum; ++t) {
+ threads[t].threadIndex = t;
+ threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj));
+ }
+
+ for (int32_t v = 0; v < numOfVnodes; ++v) {
+ int32_t t = v % threadNum;
+ SVnodeThread *pThread = &threads[t];
+ pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v];
+ }
+
+ dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+ for (int32_t t = 0; t < threadNum; ++t) {
+ SVnodeThread *pThread = &threads[t];
+ if (pThread->vnodeNum == 0) continue;
+
+ pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread);
+ if (pThread->pThreadId == NULL) {
+ dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+ }
+ }
+
+ for (int32_t t = 0; t < threadNum; ++t) {
+ SVnodeThread *pThread = &threads[t];
+ taosDestoryThread(pThread->pThreadId);
+ pThread->pThreadId = NULL;
+ free(pThread->pVnodes);
+ }
+ free(threads);
+
+ if (pMgmt->openVnodes != pMgmt->totalVnodes) {
+ dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes);
+ return -1;
+ } else {
+ dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes);
+ return 0;
+ }
+}
+
+static void dndCloseVnodes(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+ int32_t numOfVnodes = 0;
+ SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
+
+ for (int32_t i = 0; i < numOfVnodes; ++i) {
+ dndDropVnodeWrapper(pDnode, pVnodes[i]);
+ }
+
+ if (pVnodes != NULL) {
+ free(pVnodes);
+ }
+
+ if (pMgmt->hash != NULL) {
+ taosHashCleanup(pMgmt->hash);
+ pMgmt->hash = NULL;
+ }
+
+ dInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
+ SCreateVnodeMsg *pCreate = rpcMsg->pCont;
+ *vgId = htonl(pCreate->vgId);
+
+ tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
+ pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
+ pCfg->totalBlocks = htonl(pCreate->totalBlocks);
+ pCfg->daysPerFile = htonl(pCreate->daysPerFile);
+ pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
+ pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
+ pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
+ pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
+ pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
+ pCfg->precision = pCreate->precision;
+ pCfg->compression = pCreate->compression;
+ pCfg->cacheLastRow = pCreate->cacheLastRow;
+ pCfg->update = pCreate->update;
+ pCfg->quorum = pCreate->quorum;
+ pCfg->replica = pCreate->replica;
+ pCfg->walLevel = pCreate->walLevel;
+ pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
+
+ for (int32_t i = 0; i < pCfg->replica; ++i) {
+ pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
+ tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
+ }
+
+ return 0;
+}
+
+static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
+ SDropVnodeMsg *pDrop = rpcMsg->pCont;
+ pDrop->vgId = htonl(pDrop->vgId);
+ return pDrop;
+}
+
+static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
+ SAuthVnodeMsg *pAuth = rpcMsg->pCont;
+ pAuth->vgId = htonl(pAuth->vgId);
+ return pAuth;
+}
+
+static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SVnodeCfg vnodeCfg = {0};
+ int32_t vgId = 0;
+
+ dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
+ dDebug("vgId:%d, create vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode != NULL) {
+ dDebug("vgId:%d, already exist, return success", vgId);
+ dndReleaseVnode(pDnode, pVnode);
+ return 0;
+ }
+
+ if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) {
+ dError("vgId:%d, failed to create vnode since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ return 0;
+}
+
+static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SVnodeCfg vnodeCfg = {0};
+ int32_t vgId = 0;
+
+ dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
+ dDebug("vgId:%d, alter vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) {
+ dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
+ dndReleaseVnode(pDnode, pVnode);
+ return terrno;
+ }
+
+ dndReleaseVnode(pDnode, pVnode);
+ return 0;
+}
+
+static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg);
+
+ int32_t vgId = pDrop->vgId;
+ dDebug("vgId:%d, drop vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ if (dndDropVnode(pDnode, pVnode) != 0) {
+ dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr());
+ dndReleaseVnode(pDnode, pVnode);
+ return terrno;
+ }
+
+ return 0;
+}
+
+static int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
+
+ int32_t code = 0;
+ int32_t vgId = pAuth->vgId;
+ dDebug("vgId:%d, auth vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ dDebug("vgId:%d, failed to auth since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ pVnode->accessState = pAuth->accessState;
+ dndReleaseVnode(pDnode, pVnode);
+ return 0;
+}
+
+static int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
+
+ int32_t vgId = pAuth->vgId;
+ dDebug("vgId:%d, auth vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ dDebug("vgId:%d, failed to auth since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ if (vnodeSync(pVnode->pImpl) != 0) {
+ dError("vgId:%d, failed to auth vnode since %s", vgId, terrstr());
+ dndReleaseVnode(pDnode, pVnode);
+ return terrno;
+ }
+
+ dndReleaseVnode(pDnode, pVnode);
+ return 0;
+}
+
+static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
+ SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg);
+
+ int32_t vgId = pCompact->vgId;
+ dDebug("vgId:%d, compact vnode req is received", vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ dDebug("vgId:%d, failed to compact since %s", vgId, terrstr());
+ return terrno;
+ }
+
+ if (vnodeCompact(pVnode->pImpl) != 0) {
+ dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr());
+ dndReleaseVnode(pDnode, pVnode);
+ return terrno;
+ }
+
+ dndReleaseVnode(pDnode, pVnode);
+ return 0;
+}
+
+static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
+ int32_t code = 0;
+
+ switch (pMsg->msgType) {
+ case TSDB_MSG_TYPE_CREATE_VNODE_IN:
+ code = dndProcessCreateVnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_ALTER_VNODE_IN:
+ code = dndProcessAlterVnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_DROP_VNODE_IN:
+ code = dndProcessDropVnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_AUTH_VNODE_IN:
+ code = dndProcessAuthVnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_SYNC_VNODE_IN:
+ code = dndProcessSyncVnodeReq(pDnode, pMsg);
+ break;
+ case TSDB_MSG_TYPE_COMPACT_VNODE_IN:
+ code = dndProcessCompactVnodeReq(pDnode, pMsg);
+ break;
+ default:
+ code = TSDB_CODE_MSG_NOT_PROCESSED;
+ break;
+ }
+
+ if (code != 0) {
+ SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ taosFreeQitem(pMsg);
+ }
+}
+
+static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
+ vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY);
+}
+
+static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
+ vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH);
+}
+
+static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+ SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
+ SRpcMsg *pRpcMsg = NULL;
+
+ for (int32_t i = 0; i < numOfMsgs; ++i) {
+ taosGetQitem(qall, (void **)&pRpcMsg);
+ vnodeAppendMsg(pMsg, pRpcMsg);
+ taosFreeQitem(pRpcMsg);
+ }
+
+ vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE);
+}
+
+static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+ SVnodeMsg *pMsg = NULL;
+ for (int32_t i = 0; i < numOfMsgs; ++i) {
+ taosGetQitem(qall, (void **)&pMsg);
+ vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY);
+ }
+}
+
+static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
+ SVnodeMsg *pMsg = NULL;
+ for (int32_t i = 0; i < numOfMsgs; ++i) {
+ taosGetQitem(qall, (void **)&pMsg);
+ vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC);
+ }
+}
+
+static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
+ int32_t code = 0;
+
+ if (pQueue == NULL) {
+ code = TSDB_CODE_MSG_NOT_PROCESSED;
+ } else {
+ SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
+ if (pMsg == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ } else {
+ *pMsg = *pRpcMsg;
+ if (taosWriteQitem(pQueue, pMsg) != 0) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ }
+ }
+ }
+
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pRpcMsg->pCont);
+ }
+}
+
+static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
+ int32_t code = 0;
+
+ if (pQueue == NULL) {
+ code = TSDB_CODE_MSG_NOT_PROCESSED;
+ } else {
+ SVnodeMsg *pMsg = vnodeInitMsg(1);
+ if (pMsg == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ } else {
+ if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) {
+ code = terrno;
+ } else {
+ if (taosWriteQitem(pQueue, pMsg) != 0) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ }
+ }
+ }
+ }
+
+ if (code != TSDB_CODE_SUCCESS) {
+ SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pRpcMsg->pCont);
+ }
+}
+
+static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) {
+ SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
+ pHead->vgId = htonl(pHead->vgId);
+
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId);
+ if (pVnode == NULL) {
+ SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
+ rpcSendResponse(&rsp);
+ rpcFreeCont(pMsg->pCont);
+ }
+
+ return pVnode;
+}
+
+void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ dndWriteRpcMsgToVnodeQueue(pMgmt->pMgmtQ, pMsg);
+}
+
+void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+ if (pVnode != NULL) {
+ dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg);
+ dndReleaseVnode(pDnode, pVnode);
+ }
+}
+
+void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+ if (pVnode != NULL) {
+ dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
+ dndReleaseVnode(pDnode, pVnode);
+ }
+}
+
+void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+ if (pVnode != NULL) {
+ dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
+ dndReleaseVnode(pDnode, pVnode);
+ }
+}
+
+void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+ SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
+ if (pVnode != NULL) {
+ dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
+ dndReleaseVnode(pDnode, pVnode);
+ }
+}
+
+static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) {
+ SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
+ if (pVnode == NULL) {
+ return -1;
+ }
+
+ int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg);
+ dndReleaseVnode(pDnode, pVnode);
+ return code;
+}
+
+static int32_t dndInitVnodeMgmtWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ SWorkerPool *pPool = &pMgmt->mgmtPool;
+ pPool->name = "vnode-mgmt";
+ pPool->min = 1;
+ pPool->max = 1;
+ if (tWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ pMgmt->pMgmtQ = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)dndProcessVnodeMgmtQueue);
+ if (pMgmt->pMgmtQ == NULL) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupVnodeMgmtWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ);
+ tWorkerCleanup(&pMgmt->mgmtPool);
+ pMgmt->pMgmtQ = NULL;
+}
+
+static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue);
+ if (pVnode->pQueryQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
+ pVnode->pQueryQ = NULL;
+}
+
+static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue);
+ if (pVnode->pFetchQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ return 0;
+}
+
+static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
+ pVnode->pFetchQ = NULL;
+}
+
+static int32_t dndInitVnodeReadWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+ int32_t maxFetchThreads = 4;
+ float threadsForQuery = MAX(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores, 1);
+
+ SWorkerPool *pPool = &pMgmt->queryPool;
+ pPool->name = "vnode-query";
+ pPool->min = (int32_t)threadsForQuery;
+ pPool->max = pPool->min;
+ if (tWorkerInit(pPool) != 0) {
+ return TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
+
+ pPool = &pMgmt->fetchPool;
+ pPool->name = "vnode-fetch";
+ pPool->min = MIN(maxFetchThreads, pDnode->opt.numOfCores);
+ pPool->max = pPool->min;
+ if (tWorkerInit(pPool) != 0) {
+ TSDB_CODE_VND_OUT_OF_MEMORY;
+ }
+
+ return 0;
+}
+
+static void dndCleanupVnodeReadWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tWorkerCleanup(&pMgmt->fetchPool);
+ tWorkerCleanup(&pMgmt->queryPool);
+}
+
+static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue);
+ if (pVnode->pWriteQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
+ pVnode->pWriteQ = NULL;
+}
+
+static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue);
+ if (pVnode->pApplyQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
+ pVnode->pApplyQ = NULL;
+}
+
+static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ SMWorkerPool *pPool = &pMgmt->writePool;
+ pPool->name = "vnode-write";
+ pPool->max = tsNumOfCores;
+ if (tMWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupVnodeWriteWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tMWorkerCleanup(&pMgmt->writePool);
+}
+
+static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue);
+ if (pVnode->pSyncQ == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ);
+ pVnode->pSyncQ = NULL;
+}
+
+static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) {
+ int32_t maxThreads = tsNumOfCores / 2;
+ if (maxThreads < 1) maxThreads = 1;
+
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ SMWorkerPool *pPool = &pMgmt->writePool;
+ pPool->name = "vnode-sync";
+ pPool->max = maxThreads;
+ if (tMWorkerInit(pPool) != 0) {
+ terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void dndCleanupVnodeSyncWorker(SDnode *pDnode) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+ tMWorkerCleanup(&pMgmt->syncPool);
+}
+
+int32_t dndInitVnodes(SDnode *pDnode) {
+ dInfo("dnode-vnodes start to init");
+
+ if (dndInitVnodeReadWorker(pDnode) != 0) {
+ dError("failed to init vnodes read worker since %s", terrstr());
+ return -1;
+ }
+
+ if (dndInitVnodeWriteWorker(pDnode) != 0) {
+ dError("failed to init vnodes write worker since %s", terrstr());
+ return -1;
+ }
+
+ if (dndInitVnodeSyncWorker(pDnode) != 0) {
+ dError("failed to init vnodes sync worker since %s", terrstr());
+ return -1;
+ }
+
+ if (dndInitVnodeMgmtWorker(pDnode) != 0) {
+ dError("failed to init vnodes mgmt worker since %s", terrstr());
+ return -1;
+ }
+
+ if (dndOpenVnodes(pDnode) != 0) {
+ dError("failed to open vnodes since %s", terrstr());
+ return -1;
+ }
+
+ dInfo("dnode-vnodes is initialized");
+ return 0;
+}
+
+void dndCleanupVnodes(SDnode *pDnode) {
+ dInfo("dnode-vnodes start to clean up");
+ dndCloseVnodes(pDnode);
+ dndCleanupVnodeReadWorker(pDnode);
+ dndCleanupVnodeWriteWorker(pDnode);
+ dndCleanupVnodeSyncWorker(pDnode);
+ dndCleanupVnodeMgmtWorker(pDnode);
+ dInfo("dnode-vnodes is cleaned up");
+}
+
+void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) {
+ SVnodesMgmt *pMgmt = &pDnode->vmgmt;
+
+ taosRLockLatch(&pMgmt->latch);
+ pLoads->num = taosHashGetSize(pMgmt->hash);
+
+ int32_t v = 0;
+ void *pIter = taosHashIterate(pMgmt->hash, NULL);
+ while (pIter) {
+ SVnodeObj **ppVnode = pIter;
+ if (ppVnode == NULL || *ppVnode == NULL) continue;
+
+ SVnodeObj *pVnode = *ppVnode;
+ SVnodeLoad *pLoad = &pLoads->data[v++];
+
+ vnodeGetLoad(pVnode->pImpl, pLoad);
+ pLoad->vgId = htonl(pLoad->vgId);
+ pLoad->totalStorage = htobe64(pLoad->totalStorage);
+ pLoad->compStorage = htobe64(pLoad->compStorage);
+ pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
+ pLoad->tablesNum = htobe64(pLoad->tablesNum);
+
+ pIter = taosHashIterate(pMgmt->hash, pIter);
+ }
+
+ taosRUnLockLatch(&pMgmt->latch);
+}
diff --git a/source/dnode/mgmt/impl/src/dnodeMnode.c b/source/dnode/mgmt/impl/src/dnodeMnode.c
deleted file mode 100644
index 06b28aeea9..0000000000
--- a/source/dnode/mgmt/impl/src/dnodeMnode.c
+++ /dev/null
@@ -1,608 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeMnode.h"
-#include "cJSON.h"
-#include "dnodeDnode.h"
-#include "dnodeTransport.h"
-#include "mnode.h"
-#include "tlockfree.h"
-#include "tqueue.h"
-#include "tstep.h"
-#include "tworker.h"
-
-static struct {
- int32_t refCount;
- int8_t deployed;
- int8_t dropped;
- SWorkerPool mgmtPool;
- SWorkerPool readPool;
- SWorkerPool writePool;
- SWorkerPool syncPool;
- taos_queue pReadQ;
- taos_queue pWriteQ;
- taos_queue pApplyQ;
- taos_queue pSyncQ;
- taos_queue pMgmtQ;
- SSteps *pSteps;
- SMnode *pMnode;
- SRWLatch latch;
-} tsMnode = {0};
-
-static int32_t dnodeAllocMnodeReadQueue();
-static void dnodeFreeMnodeReadQueue();
-static int32_t dnodeAllocMnodeWriteQueue();
-static void dnodeFreeMnodeWriteQueue();
-static int32_t dnodeAllocMnodeApplyQueue();
-static void dnodeFreeMnodeApplyQueue();
-static int32_t dnodeAllocMnodeSyncQueue();
-static void dnodeFreeMnodeSyncQueue();
-
-static SMnode *dnodeAcquireMnode() {
- SMnode *pMnode = NULL;
- taosRLockLatch(&tsMnode.latch);
-
- if (tsMnode.deployed) {
- atomic_add_fetch_32(&tsMnode.refCount, 1);
- pMnode = tsMnode.pMnode;
- }
-
- taosRUnLockLatch(&tsMnode.latch);
- return pMnode;
-}
-
-static void dnodeReleaseMnode(SMnode *pMnode) {
- taosRLockLatch(&tsMnode.latch);
- atomic_sub_fetch_32(&tsMnode.refCount, 1);
- taosRUnLockLatch(&tsMnode.latch);
-}
-
-static int32_t dnodeReadMnodeFile() {
- int32_t code = TSDB_CODE_DND_READ_MNODE_FILE_ERROR;
- int32_t len = 0;
- int32_t maxLen = 300;
- char *content = calloc(1, maxLen + 1);
- cJSON *root = NULL;
- FILE *fp = NULL;
- char file[PATH_MAX + 20] = {0};
-
- snprintf(file, sizeof(file), "%s/mnode.json", tsDnodeDir);
- fp = fopen(file, "r");
- if (!fp) {
- dDebug("file %s not exist", file);
- code = 0;
- goto PRASE_MNODE_OVER;
- }
-
- len = (int32_t)fread(content, 1, maxLen, fp);
- if (len <= 0) {
- dError("failed to read %s since content is null", file);
- goto PRASE_MNODE_OVER;
- }
-
- content[len] = 0;
- root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read %s since invalid json format", file);
- goto PRASE_MNODE_OVER;
- }
-
- cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
- if (!deployed || deployed->type != cJSON_String) {
- dError("failed to read %s since deployed not found", file);
- goto PRASE_MNODE_OVER;
- }
- tsMnode.deployed = atoi(deployed->valuestring);
-
- cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
- if (!dropped || dropped->type != cJSON_String) {
- dError("failed to read %s since dropped not found", file);
- goto PRASE_MNODE_OVER;
- }
- tsMnode.dropped = atoi(dropped->valuestring);
-
- code = 0;
- dInfo("succcessed to read file %s", file);
-
-PRASE_MNODE_OVER:
- if (content != NULL) free(content);
- if (root != NULL) cJSON_Delete(root);
- if (fp != NULL) fclose(fp);
-
- return code;
-}
-
-static int32_t dnodeWriteMnodeFile() {
- char file[PATH_MAX + 20] = {0};
- char realfile[PATH_MAX + 20] = {0};
- snprintf(file, sizeof(file), "%s/mnode.json.bak", tsDnodeDir);
- snprintf(realfile, sizeof(realfile), "%s/mnode.json", tsDnodeDir);
-
- FILE *fp = fopen(file, "w");
- if (!fp) {
- dError("failed to write %s since %s", file, strerror(errno));
- return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR;
- }
-
- int32_t len = 0;
- int32_t maxLen = 300;
- char *content = calloc(1, maxLen + 1);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"deployed\": \"%d\",\n", tsMnode.deployed);
- len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", tsMnode.dropped);
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- taosFsyncFile(fileno(fp));
- fclose(fp);
- free(content);
-
- int32_t code = taosRenameFile(file, realfile);
- if (code != 0) {
- dError("failed to rename %s since %s", file, tstrerror(code));
- return TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR;
- }
-
- dInfo("successed to write %s", realfile);
- return 0;
-}
-
-static int32_t dnodeStartMnode() {
- int32_t code = dnodeAllocMnodeReadQueue();
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocMnodeWriteQueue();
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocMnodeApplyQueue();
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocMnodeSyncQueue();
- if (code != 0) {
- return code;
- }
-
- taosWLockLatch(&tsMnode.latch);
- tsMnode.deployed = 1;
- taosWUnLockLatch(&tsMnode.latch);
-
- return mnodeStart(NULL);
-}
-
-static void dnodeStopMnode() {
- taosWLockLatch(&tsMnode.latch);
- tsMnode.deployed = 0;
- taosWUnLockLatch(&tsMnode.latch);
-
- dnodeReleaseMnode();
-
- while (tsMnode.refCount > 0) taosMsleep(10);
- while (!taosQueueEmpty(tsMnode.pReadQ)) taosMsleep(10);
- while (!taosQueueEmpty(tsMnode.pApplyQ)) taosMsleep(10);
- while (!taosQueueEmpty(tsMnode.pWriteQ)) taosMsleep(10);
- while (!taosQueueEmpty(tsMnode.pSyncQ)) taosMsleep(10);
-
- dnodeFreeMnodeReadQueue();
- dnodeFreeMnodeWriteQueue();
- dnodeFreeMnodeApplyQueue();
- dnodeFreeMnodeSyncQueue();
-}
-
-static int32_t dnodeUnDeployMnode() {
- tsMnode.dropped = 1;
- int32_t code = dnodeWriteMnodeFile();
- if (code != 0) {
- tsMnode.dropped = 0;
- dError("failed to undeploy mnode since %s", tstrerror(code));
- return code;
- }
-
- dnodeStopMnode();
- mnodeUnDeploy();
- dnodeWriteMnodeFile();
-
- return code;
-}
-
-static int32_t dnodeDeployMnode(SMnodeCfg *pCfg) {
- int32_t code = mnodeDeploy(pCfg);
- if (code != 0) {
- dError("failed to deploy mnode since %s", tstrerror(code));
- return code;
- }
-
- code = dnodeStartMnode();
- if (code != 0) {
- dnodeUnDeployMnode();
- dError("failed to deploy mnode since %s", tstrerror(code));
- return code;
- }
-
- code = dnodeWriteMnodeFile();
- if (code != 0) {
- dnodeUnDeployMnode();
- dError("failed to deploy mnode since %s", tstrerror(code));
- return code;
- }
-
- dInfo("deploy mnode success");
- return code;
-}
-
-static int32_t dnodeAlterMnode(SMnodeCfg *pCfg) {
- int32_t code = dnodeAcquireMnode();
- if (code == 0) {
- code = mnodeAlter(pCfg);
- dnodeReleaseMnode();
- }
- return code;
-}
-
-static SCreateMnodeMsg *dnodeParseCreateMnodeMsg(SRpcMsg *pRpcMsg) {
- SCreateMnodeMsg *pMsg = pRpcMsg->pCont;
- pMsg->dnodeId = htonl(pMsg->dnodeId);
- for (int32_t i = 0; i < pMsg->replica; ++i) {
- pMsg->replicas[i].port = htons(pMsg->replicas[i].port);
- }
- return pMsg;
-}
-
-static int32_t dnodeProcessCreateMnodeReq(SRpcMsg *pRpcMsg) {
- SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont);
-
- if (pMsg->dnodeId != dnodeGetDnodeId()) {
- return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
- } else {
- SMnodeCfg cfg = {0};
- cfg.replica = pMsg->replica;
- memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA));
- return dnodeDeployMnode(&cfg);
- }
-}
-
-static int32_t dnodeProcessAlterMnodeReq(SRpcMsg *pRpcMsg) {
- SAlterMnodeMsg *pMsg = (SAlterMnodeMsg *)dnodeParseCreateMnodeMsg(pRpcMsg->pCont);
- if (pMsg->dnodeId != dnodeGetDnodeId()) {
- return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
- } else {
- SMnodeCfg cfg = {0};
- cfg.replica = pMsg->replica;
- memcpy(cfg.replicas, pMsg->replicas, sizeof(SReplica) * sizeof(TSDB_MAX_REPLICA));
- return dnodeAlterMnode(&cfg);
- }
-}
-
-static int32_t dnodeProcessDropMnodeReq(SRpcMsg *pMsg) {
- SAlterMnodeMsg *pCfg = pMsg->pCont;
- pCfg->dnodeId = htonl(pCfg->dnodeId);
-
- if (pCfg->dnodeId != dnodeGetDnodeId()) {
- return TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE;
- } else {
- return dnodeUnDeployMnode();
- }
-}
-
-static void dnodeProcessMnodeMgmtQueue(void *unused, SRpcMsg *pMsg) {
- int32_t code = 0;
-
- switch (pMsg->msgType) {
- case TSDB_MSG_TYPE_CREATE_MNODE_IN:
- code = dnodeProcessCreateMnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_ALTER_MNODE_IN:
- code = dnodeProcessAlterMnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_DROP_MNODE_IN:
- code = dnodeProcessDropMnodeReq(pMsg);
- break;
- default:
- code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- break;
- }
-
- SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
- rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
- taosFreeQitem(pMsg);
-}
-
-static void dnodeProcessMnodeReadQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_READ); }
-
-static void dnodeProcessMnodeWriteQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_WRITE); }
-
-static void dnodeProcessMnodeApplyQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_APPLY); }
-
-static void dnodeProcessMnodeSyncQueue(void *unused, SMnodeMsg *pMsg) { mnodeProcessMsg(pMsg, MN_MSG_TYPE_SYNC); }
-
-static int32_t dnodeWriteMnodeMsgToQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
- int32_t code = 0;
- SMnodeMsg *pMsg = NULL;
-
- if (pQueue == NULL) {
- code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- } else {
- pMsg = mnodeInitMsg(pRpcMsg);
- if (pMsg == NULL) {
- code = terrno;
- }
- }
-
- if (code == 0) {
- code = taosWriteQitem(pQueue, pMsg);
- }
-
- if (code != TSDB_CODE_SUCCESS) {
- SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
- rpcSendResponse(&rsp);
- rpcFreeCont(pRpcMsg->pCont);
- }
-}
-
-void dnodeProcessMnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteMnodeMsgToQueue(tsMnode.pMgmtQ, pMsg); }
-
-void dnodeProcessMnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- if (dnodeAcquireMnode() == 0) {
- dnodeWriteMnodeMsgToQueue(tsMnode.pWriteQ, pMsg);
- dnodeReleaseMnode();
- } else {
- dnodeSendRedirectMsg(NULL, pMsg, 0);
- }
-}
-
-void dnodeProcessMnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- int32_t code = dnodeAcquireMnode();
- if (code == 0) {
- dnodeWriteMnodeMsgToQueue(tsMnode.pSyncQ, pMsg);
- dnodeReleaseMnode();
- } else {
- SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
- rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
- }
-}
-
-void dnodeProcessMnodeReadMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- if (dnodeAcquireMnode() == 0) {
- dnodeWriteMnodeMsgToQueue(tsMnode.pReadQ, pMsg);
- dnodeReleaseMnode();
- } else {
- dnodeSendRedirectMsg(NULL, pMsg, 0);
- }
-}
-
-static int32_t dnodePutMsgIntoMnodeApplyQueue(SMnodeMsg *pMsg) {
- int32_t code = dnodeAcquireMnode();
- if (code != 0) return code;
-
- code = taosWriteQitem(tsMnode.pApplyQ, pMsg);
- dnodeReleaseMnode();
- return code;
-}
-
-static int32_t dnodeAllocMnodeMgmtQueue() {
- tsMnode.pMgmtQ = tWorkerAllocQueue(&tsMnode.mgmtPool, NULL, (FProcessItem)dnodeProcessMnodeMgmtQueue);
- if (tsMnode.pMgmtQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeMnodeMgmtQueue() {
- tWorkerFreeQueue(&tsMnode.mgmtPool, tsMnode.pMgmtQ);
- tsMnode.pMgmtQ = NULL;
-}
-
-static int32_t dnodeInitMnodeMgmtWorker() {
- SWorkerPool *pPool = &tsMnode.mgmtPool;
- pPool->name = "mnode-mgmt";
- pPool->min = 1;
- pPool->max = 1;
- return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeMgmtWorker() { tWorkerCleanup(&tsMnode.mgmtPool); }
-
-static int32_t dnodeAllocMnodeReadQueue() {
- tsMnode.pReadQ = tWorkerAllocQueue(&tsMnode.readPool, NULL, (FProcessItem)dnodeProcessMnodeReadQueue);
- if (tsMnode.pReadQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeMnodeReadQueue() {
- tWorkerFreeQueue(&tsMnode.readPool, tsMnode.pReadQ);
- tsMnode.pReadQ = NULL;
-}
-
-static int32_t dnodeInitMnodeReadWorker() {
- SWorkerPool *pPool = &tsMnode.readPool;
- pPool->name = "mnode-read";
- pPool->min = 0;
- pPool->max = 1;
- return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeReadWorker() { tWorkerCleanup(&tsMnode.readPool); }
-
-static int32_t dnodeAllocMnodeWriteQueue() {
- tsMnode.pWriteQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeWriteQueue);
- if (tsMnode.pWriteQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeMnodeWriteQueue() {
- tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pWriteQ);
- tsMnode.pWriteQ = NULL;
-}
-
-static int32_t dnodeAllocMnodeApplyQueue() {
- tsMnode.pApplyQ = tWorkerAllocQueue(&tsMnode.writePool, NULL, (FProcessItem)dnodeProcessMnodeApplyQueue);
- if (tsMnode.pApplyQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeMnodeApplyQueue() {
- tWorkerFreeQueue(&tsMnode.writePool, tsMnode.pApplyQ);
- tsMnode.pApplyQ = NULL;
-}
-
-static int32_t dnodeInitMnodeWriteWorker() {
- SWorkerPool *pPool = &tsMnode.writePool;
- pPool->name = "mnode-write";
- pPool->min = 0;
- pPool->max = 1;
- return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeWriteWorker() { tWorkerCleanup(&tsMnode.writePool); }
-
-static int32_t dnodeAllocMnodeSyncQueue() {
- tsMnode.pSyncQ = tWorkerAllocQueue(&tsMnode.syncPool, NULL, (FProcessItem)dnodeProcessMnodeSyncQueue);
- if (tsMnode.pSyncQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeMnodeSyncQueue() {
- tWorkerFreeQueue(&tsMnode.syncPool, tsMnode.pSyncQ);
- tsMnode.pSyncQ = NULL;
-}
-
-static int32_t dnodeInitMnodeSyncWorker() {
- SWorkerPool *pPool = &tsMnode.syncPool;
- pPool->name = "mnode-sync";
- pPool->min = 0;
- pPool->max = 1;
- return tWorkerInit(pPool);
-}
-
-static void dnodeCleanupMnodeSyncWorker() { tWorkerCleanup(&tsMnode.syncPool); }
-
-static int32_t dnodeInitMnodeModule() {
- taosInitRWLatch(&tsMnode.latch);
-
- SMnodeOptions para;
- para.dnodeId = dnodeGetDnodeId();
- para.clusterId = dnodeGetClusterId();
- para.sendMsgToDnodeFp = dnodeSendMsgToDnode;
- para.sendMsgToMnodeFp = dnodeSendMsgToMnode;
- para.sendRedirectMsgFp = dnodeSendRedirectMsg;
-
- tsMnode.pMnode = mnodeCreate(para);
- if (tsMnode.pMnode != NULL) {
- return -1;
- }
- return 0;
-}
-
-static void dnodeCleanupMnodeModule() { mnodeDrop(NULL); }
-
-static bool dnodeNeedDeployMnode() {
- if (dnodeGetDnodeId() > 0) return false;
- if (dnodeGetClusterId() > 0) return false;
- if (strcmp(tsFirst, tsLocalEp) != 0) return false;
- return true;
-}
-
-static int32_t dnodeOpenMnode() {
- int32_t code = dnodeReadMnodeFile();
- if (code != 0) {
- dError("failed to read open mnode since %s", tstrerror(code));
- return code;
- }
-
- if (tsMnode.dropped) {
- dInfo("mnode already dropped, undeploy it");
- return dnodeUnDeployMnode();
- }
-
- if (!tsMnode.deployed) {
- bool needDeploy = dnodeNeedDeployMnode();
- if (!needDeploy) return 0;
-
- dInfo("start to deploy mnode");
- SMnodeCfg cfg = {.replica = 1};
- cfg.replicas[0].port = tsServerPort;
- tstrncpy(cfg.replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
- code = dnodeDeployMnode(&cfg);
- } else {
- dInfo("start to open mnode");
- return dnodeStartMnode();
- }
-}
-
-static void dnodeCloseMnode() {
- if (dnodeAcquireMnode() == 0) {
- dnodeStopMnode();
- }
-}
-
-int32_t dnodeInitMnode() {
- dInfo("dnode-mnode start to init");
-
- SSteps *pSteps = taosStepInit(6, dnodeReportStartup);
- taosStepAdd(pSteps, "dnode-mnode-env", dnodeInitMnodeModule, dnodeCleanupMnodeModule);
- taosStepAdd(pSteps, "dnode-mnode-mgmt", dnodeInitMnodeMgmtWorker, dnodeCleanupMnodeMgmtWorker);
- taosStepAdd(pSteps, "dnode-mnode-read", dnodeInitMnodeReadWorker, dnodeCleanupMnodeReadWorker);
- taosStepAdd(pSteps, "dnode-mnode-write", dnodeInitMnodeWriteWorker, dnodeCleanupMnodeWriteWorker);
- taosStepAdd(pSteps, "dnode-mnode-sync", dnodeInitMnodeSyncWorker, dnodeCleanupMnodeSyncWorker);
- taosStepAdd(pSteps, "dnode-mnode", dnodeOpenMnode, dnodeCloseMnode);
-
- tsMnode.pSteps = pSteps;
- int32_t code = taosStepExec(pSteps);
-
- if (code != 0) {
- dError("dnode-mnode init failed since %s", tstrerror(code));
- } else {
- dInfo("dnode-mnode is initialized");
- }
-}
-
-void dnodeCleanupMnode() {
- if (tsMnode.pSteps == NULL) {
- dInfo("dnode-mnode start to clean up");
- taosStepCleanup(tsMnode.pSteps);
- tsMnode.pSteps = NULL;
- dInfo("dnode-mnode is cleaned up");
- }
-}
-
-int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
- SMnode *pMnode = dnodeAcquireMnode();
- if (pMnode == NULL) {
- dTrace("failed to get user auth since mnode not deployed");
- terrno = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
- return -1;
- }
-
- int32_t code = mnodeRetriveAuth(pMnode, user, spi, encrypt, secret, ckey);
- dnodeReleaseMnode(pMnode);
- return code;
-}
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dnodeTransport.c b/source/dnode/mgmt/impl/src/dnodeTransport.c
deleted file mode 100644
index c1e8955625..0000000000
--- a/source/dnode/mgmt/impl/src/dnodeTransport.c
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-/* this file is mainly responsible for the communication between DNODEs. Each
- * dnode works as both server and client. Dnode may send status, grant, config
- * messages to mnode, mnode may send create/alter/drop table/vnode messages
- * to dnode. All theses messages are handled from here
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeTransport.h"
-#include "dnodeDnode.h"
-#include "dnodeMnode.h"
-#include "dnodeVnodes.h"
-
-static struct {
- void *peerRpc;
- void *shellRpc;
- void *clientRpc;
- MsgFp msgFp[TSDB_MSG_TYPE_MAX];
-} tsTrans;
-
-static void dnodeInitMsgFp() {
- // msg from client to dnode
- tsTrans.msgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_QUERY] = dnodeProcessVnodeQueryMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_FETCH] = dnodeProcessVnodeFetchMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TABLE] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_TABLE_META] = dnodeProcessVnodeQueryMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_TABLES_META] = dnodeProcessVnodeQueryMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dnodeProcessVnodeQueryMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dnodeProcessVnodeQueryMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_ACK] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_MQ_RESET] = dnodeProcessVnodeWriteMsg;
-
- // msg from client to mnode
- tsTrans.msgFp[TSDB_MSG_TYPE_CONNECT] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_ACCT] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_ACCT] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_ACCT] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_USER] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_USER] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_USER] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DNODE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DNODE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_DB] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_DB] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_USE_DB] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_DB] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_DB] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_KILL_CONN] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SHOW] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dnodeProcessMnodeWriteMsg;
-
- // message from client to dnode
- tsTrans.msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeProcessDnodeMsg;
-
- // message from mnode to vnode
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessVnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dnodeProcessMnodeWriteMsg;
-
- // message from mnode to dnode
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN] = dnodeProcessVnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dnodeProcessMnodeMgmtMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN] = dnodeProcessDnodeMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP] = dnodeProcessMnodeWriteMsg;
-
- // message from dnode to mnode
- tsTrans.msgFp[TSDB_MSG_TYPE_AUTH] = dnodeProcessMnodeReadMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_AUTH_RSP] = dnodeProcessDnodeMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_GRANT] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_GRANT_RSP] = dnodeProcessDnodeMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_STATUS] = dnodeProcessMnodeWriteMsg;
- tsTrans.msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dnodeProcessDnodeMsg;
-}
-
-static void dnodeProcessPeerReq(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SRpcMsg rspMsg = {.handle = pMsg->handle};
- int32_t msgType = pMsg->msgType;
-
- if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
- dnodeProcessDnodeMsg(pMsg, pEpSet);
- return;
- }
-
- if (dnodeGetStat() != DN_STAT_RUNNING) {
- rspMsg.code = TSDB_CODE_APP_NOT_READY;
- rpcSendResponse(&rspMsg);
- rpcFreeCont(pMsg->pCont);
- dTrace("RPC %p, peer req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
- return;
- }
-
- if (pMsg->pCont == NULL) {
- rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN;
- rpcSendResponse(&rspMsg);
- return;
- }
-
- MsgFp fp = tsTrans.msgFp[msgType];
- if (fp != NULL) {
- dTrace("RPC %p, peer req:%s will be processed", pMsg->handle, taosMsg[msgType]);
- (*fp)(pMsg, pEpSet);
- } else {
- dError("RPC %p, peer req:%s not processed", pMsg->handle, taosMsg[msgType]);
- rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- rpcSendResponse(&rspMsg);
- rpcFreeCont(pMsg->pCont);
- }
-}
-
-static int32_t dnodeInitPeerServer() {
- SRpcInit rpcInit;
- memset(&rpcInit, 0, sizeof(rpcInit));
- rpcInit.localPort = tsDnodeDnodePort;
- rpcInit.label = "DND-S";
- rpcInit.numOfThreads = 1;
- rpcInit.cfp = dnodeProcessPeerReq;
- rpcInit.sessions = TSDB_MAX_VNODES << 4;
- rpcInit.connType = TAOS_CONN_SERVER;
- rpcInit.idleTime = tsShellActivityTimer * 1000;
-
- tsTrans.peerRpc = rpcOpen(&rpcInit);
- if (tsTrans.peerRpc == NULL) {
- dError("failed to init peer rpc server");
- return -1;
- }
-
- dInfo("dnode peer rpc server is initialized");
- return 0;
-}
-
-static void dnodeCleanupPeerServer() {
- if (tsTrans.peerRpc) {
- rpcClose(tsTrans.peerRpc);
- tsTrans.peerRpc = NULL;
- dInfo("dnode peer server is closed");
- }
-}
-
-static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SEpSet *pEpSet) {
- int32_t msgType = pMsg->msgType;
-
- if (dnodeGetStat() == DN_STAT_STOPPED) {
- if (pMsg == NULL || pMsg->pCont == NULL) return;
- dTrace("RPC %p, peer rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]);
- rpcFreeCont(pMsg->pCont);
- return;
- }
-
- MsgFp fp = tsTrans.msgFp[msgType];
- if (fp != NULL) {
- dTrace("RPC %p, peer rsp:%s will be processed, code:%s", pMsg->handle, taosMsg[msgType], tstrerror(pMsg->code));
- (*fp)(pMsg, pEpSet);
- } else {
- dDebug("RPC %p, peer rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
- }
-
- rpcFreeCont(pMsg->pCont);
-}
-
-static int32_t dnodeInitClient() {
- char secret[TSDB_KEY_LEN] = "secret";
-
- SRpcInit rpcInit;
- memset(&rpcInit, 0, sizeof(rpcInit));
- rpcInit.label = "DND-C";
- rpcInit.numOfThreads = 1;
- rpcInit.cfp = dnodeProcessPeerRsp;
- rpcInit.sessions = TSDB_MAX_VNODES << 4;
- rpcInit.connType = TAOS_CONN_CLIENT;
- rpcInit.idleTime = tsShellActivityTimer * 1000;
- rpcInit.user = "t";
- rpcInit.ckey = "key";
- rpcInit.secret = secret;
-
- tsTrans.clientRpc = rpcOpen(&rpcInit);
- if (tsTrans.clientRpc == NULL) {
- dError("failed to init peer rpc client");
- return -1;
- }
-
- dInfo("dnode peer rpc client is initialized");
- return 0;
-}
-
-static void dnodeCleanupClient() {
- if (tsTrans.clientRpc) {
- rpcClose(tsTrans.clientRpc);
- tsTrans.clientRpc = NULL;
- dInfo("dnode peer rpc client is closed");
- }
-}
-
-static void dnodeProcessShellReq(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SRpcMsg rspMsg = {.handle = pMsg->handle};
- int32_t msgType = pMsg->msgType;
-
- if (dnodeGetStat() == DN_STAT_STOPPED) {
- dError("RPC %p, shell req:%s is ignored since dnode exiting", pMsg->handle, taosMsg[msgType]);
- rspMsg.code = TSDB_CODE_DND_EXITING;
- rpcSendResponse(&rspMsg);
- rpcFreeCont(pMsg->pCont);
- return;
- } else if (dnodeGetStat() != DN_STAT_RUNNING) {
- dError("RPC %p, shell req:%s is ignored since dnode not running", pMsg->handle, taosMsg[msgType]);
- rspMsg.code = TSDB_CODE_APP_NOT_READY;
- rpcSendResponse(&rspMsg);
- rpcFreeCont(pMsg->pCont);
- return;
- }
-
- if (pMsg->pCont == NULL) {
- rspMsg.code = TSDB_CODE_DND_INVALID_MSG_LEN;
- rpcSendResponse(&rspMsg);
- return;
- }
-
- MsgFp fp = tsTrans.msgFp[msgType];
- if (fp != NULL) {
- dTrace("RPC %p, shell req:%s will be processed", pMsg->handle, taosMsg[msgType]);
- (*fp)(pMsg, pEpSet);
- } else {
- dError("RPC %p, shell req:%s is not processed", pMsg->handle, taosMsg[msgType]);
- rspMsg.code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- rpcSendResponse(&rspMsg);
- rpcFreeCont(pMsg->pCont);
- }
-}
-
-static void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
- SEpSet epSet = {0};
- dnodeGetMnodeEpSetForPeer(&epSet);
- rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp);
-}
-
-static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
- int32_t code = dnodeGetUserAuthFromMnode(user, spi, encrypt, secret, ckey);
- if (code != TSDB_CODE_APP_NOT_READY) return code;
-
- SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg));
- tstrncpy(pMsg->user, user, sizeof(pMsg->user));
-
- dDebug("user:%s, send auth msg to mnodes", user);
- SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH};
- SRpcMsg rpcRsp = {0};
- dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp);
-
- if (rpcRsp.code != 0) {
- dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code));
- } else {
- dDebug("user:%s, auth msg received from mnodes", user);
- SAuthRsp *pRsp = rpcRsp.pCont;
- memcpy(secret, pRsp->secret, TSDB_KEY_LEN);
- memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN);
- *spi = pRsp->spi;
- *encrypt = pRsp->encrypt;
- }
-
- rpcFreeCont(rpcRsp.pCont);
- return rpcRsp.code;
-}
-
-static int32_t dnodeInitShellServer() {
- dnodeInitMsgFp();
-
- int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
- if (numOfThreads < 1) {
- numOfThreads = 1;
- }
-
- SRpcInit rpcInit;
- memset(&rpcInit, 0, sizeof(rpcInit));
- rpcInit.localPort = tsDnodeShellPort;
- rpcInit.label = "SHELL";
- rpcInit.numOfThreads = numOfThreads;
- rpcInit.cfp = dnodeProcessShellReq;
- rpcInit.sessions = tsMaxShellConns;
- rpcInit.connType = TAOS_CONN_SERVER;
- rpcInit.idleTime = tsShellActivityTimer * 1000;
- rpcInit.afp = dnodeRetrieveUserAuthInfo;
-
- tsTrans.shellRpc = rpcOpen(&rpcInit);
- if (tsTrans.shellRpc == NULL) {
- dError("failed to init shell rpc server");
- return -1;
- }
-
-#if 1
- SRpcMsg rpcMsg = {0};
- rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER;
- rpcMsg.contLen = sizeof(SCreateUserMsg);
- rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
- SCreateUserMsg *pMsg = (SCreateUserMsg*)rpcMsg.pCont;
- strcpy(pMsg->user, "u1");
- strcpy(pMsg->pass, "up1");
- dnodeProcessShellReq(&rpcMsg, NULL);
-
-#endif
- dInfo("dnode shell rpc server is initialized");
- return 0;
-}
-
-static void dnodeCleanupShellServer() {
- if (tsTrans.shellRpc) {
- rpcClose(tsTrans.shellRpc);
- tsTrans.shellRpc = NULL;
- }
-}
-
-int32_t dnodeInitTrans() {
- if (dnodeInitClient() != 0) {
- return -1;
- }
-
- if (dnodeInitPeerServer() != 0) {
- return -1;
- }
-
- if (dnodeInitShellServer() != 0) {
- return -1;
- }
-
- return 0;
-}
-
-void dnodeCleanupTrans() {
- dnodeCleanupShellServer();
- dnodeCleanupPeerServer();
- dnodeCleanupClient();
-}
-
-void dnodeSendMsgToDnode(SDnode *pDnode, SEpSet *epSet, SRpcMsg *rpcMsg) {
- #if 0
- rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL);
- #endif
- }
-
-void dnodeSendMsgToMnode(SDnode *pDnode, SRpcMsg *rpcMsg) {
- SEpSet epSet = {0};
- dnodeGetMnodeEpSetForPeer(&epSet);
- dnodeSendMsgToDnode(NULL, &epSet, rpcMsg);
-}
\ No newline at end of file
diff --git a/source/dnode/mgmt/impl/src/dnodeVnodes.c b/source/dnode/mgmt/impl/src/dnodeVnodes.c
deleted file mode 100644
index 7eaa82ba93..0000000000
--- a/source/dnode/mgmt/impl/src/dnodeVnodes.c
+++ /dev/null
@@ -1,1029 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#define _DEFAULT_SOURCE
-#include "dnodeVnodes.h"
-#include "dnodeTransport.h"
-#include "cJSON.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tqueue.h"
-#include "tstep.h"
-#include "tthread.h"
-#include "tworker.h"
-#include "vnode.h"
-
-typedef struct {
- int32_t vgId;
- int32_t refCount;
- int8_t dropped;
- int8_t accessState;
- SVnode *pImpl;
- taos_queue pWriteQ;
- taos_queue pSyncQ;
- taos_queue pApplyQ;
- taos_queue pQueryQ;
- taos_queue pFetchQ;
-} SVnodeObj;
-
-typedef struct {
- pthread_t *threadId;
- int32_t threadIndex;
- int32_t failed;
- int32_t opened;
- int32_t vnodeNum;
- SVnodeObj *pVnodes;
-} SVThread;
-
-static struct {
- SHashObj *hash;
- SWorkerPool mgmtPool;
- SWorkerPool queryPool;
- SWorkerPool fetchPool;
- SMWorkerPool syncPool;
- SMWorkerPool writePool;
- taos_queue pMgmtQ;
- SSteps *pSteps;
- int32_t openVnodes;
- int32_t totalVnodes;
- SRWLatch latch;
-} tsVnodes;
-
-static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode);
-static void dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode);
-static void dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode);
-static void dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode);
-static void dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode);
-static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode);
-static void dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode);
-
-static SVnodeObj *dnodeAcquireVnode(int32_t vgId) {
- SVnodeObj *pVnode = NULL;
- int32_t refCount = 0;
-
- taosRLockLatch(&tsVnodes.latch);
- taosHashGetClone(tsVnodes.hash, &vgId, sizeof(int32_t), (void *)&pVnode);
- if (pVnode == NULL) {
- terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
- } else {
- refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
- }
- taosRUnLockLatch(&tsVnodes.latch);
-
- dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount);
- return pVnode;
-}
-
-static void dnodeReleaseVnode(SVnodeObj *pVnode) {
- int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
- dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
-}
-
-static int32_t dnodeCreateVnodeWrapper(int32_t vgId, SVnode *pImpl) {
- SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
- if (pVnode == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
-
- pVnode->vgId = vgId;
- pVnode->refCount = 0;
- pVnode->dropped = 0;
- pVnode->accessState = TSDB_VN_ALL_ACCCESS;
- pVnode->pImpl = pImpl;
-
- int32_t code = dnodeAllocVnodeQueryQueue(pVnode);
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocVnodeFetchQueue(pVnode);
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocVnodeWriteQueue(pVnode);
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocVnodeApplyQueue(pVnode);
- if (code != 0) {
- return code;
- }
-
- code = dnodeAllocVnodeSyncQueue(pVnode);
- if (code != 0) {
- return code;
- }
-
- taosWLockLatch(&tsVnodes.latch);
- code = taosHashPut(tsVnodes.hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
- taosWUnLockLatch(&tsVnodes.latch);
-
- return code;
-}
-
-static void dnodeDropVnodeWrapper(SVnodeObj *pVnode) {
- taosWLockLatch(&tsVnodes.latch);
- taosHashRemove(tsVnodes.hash, &pVnode->vgId, sizeof(int32_t));
- taosWUnLockLatch(&tsVnodes.latch);
-
- // wait all queue empty
- dnodeReleaseVnode(pVnode);
- while (pVnode->refCount > 0) taosMsleep(10);
- while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
- while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
- while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
- while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
- while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
-
- dnodeFreeVnodeQueryQueue(pVnode);
- dnodeFreeVnodeFetchQueue(pVnode);
- dnodeFreeVnodeWriteQueue(pVnode);
- dnodeFreeVnodeApplyQueue(pVnode);
- dnodeFreeVnodeSyncQueue(pVnode);
-}
-
-static SVnodeObj **dnodeGetVnodesFromHash(int32_t *numOfVnodes) {
- taosRLockLatch(&tsVnodes.latch);
-
- int32_t num = 0;
- int32_t size = taosHashGetSize(tsVnodes.hash);
- SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *));
-
- void *pIter = taosHashIterate(tsVnodes.hash, NULL);
- while (pIter) {
- SVnodeObj **ppVnode = pIter;
- SVnodeObj *pVnode = *ppVnode;
- if (pVnode) {
- num++;
- if (num < size) {
- int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
- dTrace("vgId:%d, accquire vnode, refCount:%d", pVnode->vgId, refCount);
- pVnodes[num] = (*ppVnode);
- }
- }
- pIter = taosHashIterate(tsVnodes.hash, pIter);
- }
-
- taosRUnLockLatch(&tsVnodes.latch);
- *numOfVnodes = num;
-
- return pVnodes;
-}
-
-static int32_t dnodeGetVnodesFromFile(SVnodeObj **ppVnodes, int32_t *numOfVnodes) {
- int32_t code = TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR;
- int32_t len = 0;
- int32_t maxLen = 30000;
- char *content = calloc(1, maxLen + 1);
- cJSON *root = NULL;
- FILE *fp = NULL;
- char file[PATH_MAX + 20] = {0};
- SVnodeObj *pVnodes = NULL;
-
- snprintf(file, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir);
-
- fp = fopen(file, "r");
- if (!fp) {
- dDebug("file %s not exist", file);
- code = 0;
- goto PRASE_VNODE_OVER;
- }
-
- len = (int32_t)fread(content, 1, maxLen, fp);
- if (len <= 0) {
- dError("failed to read %s since content is null", file);
- goto PRASE_VNODE_OVER;
- }
-
- content[len] = 0;
- root = cJSON_Parse(content);
- if (root == NULL) {
- dError("failed to read %s since invalid json format", file);
- goto PRASE_VNODE_OVER;
- }
-
- cJSON *vnodes = cJSON_GetObjectItem(root, "vnodes");
- if (!vnodes || vnodes->type != cJSON_Array) {
- dError("failed to read %s since vnodes not found", file);
- goto PRASE_VNODE_OVER;
- }
-
- int32_t vnodesNum = cJSON_GetArraySize(vnodes);
- if (vnodesNum <= 0) {
- dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum);
- goto PRASE_VNODE_OVER;
- }
-
- pVnodes = calloc(vnodesNum, sizeof(SVnodeObj));
- if (pVnodes == NULL) {
- dError("failed to read %s since out of memory", file);
- goto PRASE_VNODE_OVER;
- }
-
- for (int32_t i = 0; i < vnodesNum; ++i) {
- cJSON *vnode = cJSON_GetArrayItem(vnodes, i);
- SVnodeObj *pVnode = &pVnodes[i];
-
- cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
- if (!vgId || vgId->type != cJSON_String) {
- dError("failed to read %s since vgId not found", file);
- goto PRASE_VNODE_OVER;
- }
- pVnode->vgId = atoi(vgId->valuestring);
-
- cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
- if (!dropped || dropped->type != cJSON_String) {
- dError("failed to read %s since dropped not found", file);
- goto PRASE_VNODE_OVER;
- }
- pVnode->dropped = atoi(vnode->valuestring);
- }
-
- code = 0;
- dInfo("succcessed to read file %s", file);
-
-PRASE_VNODE_OVER:
- if (content != NULL) free(content);
- if (root != NULL) cJSON_Delete(root);
- if (fp != NULL) fclose(fp);
-
- return code;
-}
-
-static int32_t dnodeWriteVnodesToFile() {
- char file[PATH_MAX + 20] = {0};
- char realfile[PATH_MAX + 20] = {0};
- snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", tsVnodeDir);
- snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", tsVnodeDir);
-
- FILE *fp = fopen(file, "w");
- if (!fp) {
- dError("failed to write %s since %s", file, strerror(errno));
- return -1;
- }
-
- int32_t len = 0;
- int32_t maxLen = 30000;
- char *content = calloc(1, maxLen + 1);
- int32_t numOfVnodes = 0;
- SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes);
-
- len += snprintf(content + len, maxLen - len, "{\n");
- len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n");
- for (int32_t i = 0; i < numOfVnodes; ++i) {
- SVnodeObj *pVnode = pVnodes[i];
- len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId);
- len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped);
- if (i < numOfVnodes - 1) {
- len += snprintf(content + len, maxLen - len, " },{\n");
- } else {
- len += snprintf(content + len, maxLen - len, " }]\n");
- }
- }
- len += snprintf(content + len, maxLen - len, "}\n");
-
- fwrite(content, 1, len, fp);
- taosFsyncFile(fileno(fp));
- fclose(fp);
- free(content);
- terrno = 0;
-
- for (int32_t i = 0; i < numOfVnodes; ++i) {
- SVnodeObj *pVnode = pVnodes[i];
- dnodeReleaseVnode(pVnode);
- }
-
- if (pVnodes != NULL) {
- free(pVnodes);
- }
-
- dInfo("successed to write %s", file);
- return taosRenameFile(file, realfile);
-}
-
-static int32_t dnodeCreateVnode(int32_t vgId, SVnodeCfg *pCfg) {
- int32_t code = 0;
-
- char path[PATH_MAX + 20] = {0};
- snprintf(path, sizeof(path),"%s/vnode%d", tsVnodeDir, vgId);
- SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
-
- if (pImpl == NULL) {
- code = terrno;
- return code;
- }
-
- code = dnodeCreateVnodeWrapper(vgId, pImpl);
- if (code != 0) {
- vnodeDrop(pImpl);
- return code;
- }
-
- code = dnodeWriteVnodesToFile();
- if (code != 0) {
- vnodeDrop(pImpl);
- return code;
- }
-
- return code;
-}
-
-static int32_t dnodeDropVnode(SVnodeObj *pVnode) {
- pVnode->dropped = 1;
-
- int32_t code = dnodeWriteVnodesToFile();
- if (code != 0) {
- pVnode->dropped = 0;
- return code;
- }
-
- dnodeDropVnodeWrapper(pVnode);
- vnodeDrop(pVnode->pImpl);
- dnodeWriteVnodesToFile();
- return 0;
-}
-
-static void *dnodeOpenVnodeFunc(void *param) {
- SVThread *pThread = param;
-
- dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
- setThreadName("open-vnodes");
-
- for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
- SVnodeObj *pVnode = &pThread->pVnodes[v];
-
- char stepDesc[TSDB_STEP_DESC_LEN] = {0};
- snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId,
- tsVnodes.openVnodes, tsVnodes.totalVnodes);
- dnodeReportStartup("open-vnodes", stepDesc);
-
- char path[PATH_MAX + 20] = {0};
- snprintf(path, sizeof(path),"%s/vnode%d", tsVnodeDir, pVnode->vgId);
- SVnode *pImpl = vnodeOpen(path, NULL);
- if (pImpl == NULL) {
- dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
- pThread->failed++;
- } else {
- dnodeCreateVnodeWrapper(pVnode->vgId, pImpl);
- dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex);
- pThread->opened++;
- }
-
- atomic_add_fetch_32(&tsVnodes.openVnodes, 1);
- }
-
- dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
- pThread->failed);
- return NULL;
-}
-
-static int32_t dnodeOpenVnodes() {
- taosInitRWLatch(&tsVnodes.latch);
-
- tsVnodes.hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
- if (tsVnodes.hash == NULL) {
- dError("failed to init vnode hash");
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- SVnodeObj *pVnodes = NULL;
- int32_t numOfVnodes = 0;
- int32_t code = dnodeGetVnodesFromFile(&pVnodes, &numOfVnodes);
- if (code != TSDB_CODE_SUCCESS) {
- dInfo("failed to get vnode list from disk since %s", tstrerror(code));
- return code;
- }
-
- tsVnodes.totalVnodes = numOfVnodes;
-
- int32_t threadNum = tsNumOfCores;
- int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
-
- SVThread *threads = calloc(threadNum, sizeof(SVThread));
- for (int32_t t = 0; t < threadNum; ++t) {
- threads[t].threadIndex = t;
- threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj));
- }
-
- for (int32_t v = 0; v < numOfVnodes; ++v) {
- int32_t t = v % threadNum;
- SVThread *pThread = &threads[t];
- pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v];
- }
-
- dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
-
- for (int32_t t = 0; t < threadNum; ++t) {
- SVThread *pThread = &threads[t];
- if (pThread->vnodeNum == 0) continue;
-
- pThread->threadId = taosCreateThread(dnodeOpenVnodeFunc, pThread);
- if (pThread->threadId == NULL) {
- dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
- }
- }
-
- for (int32_t t = 0; t < threadNum; ++t) {
- SVThread *pThread = &threads[t];
- taosDestoryThread(pThread->threadId);
- pThread->threadId = NULL;
- free(pThread->pVnodes);
- }
- free(threads);
-
- if (tsVnodes.openVnodes != tsVnodes.totalVnodes) {
- dError("there are total vnodes:%d, opened:%d", tsVnodes.totalVnodes, tsVnodes.openVnodes);
- return -1;
- } else {
- dInfo("total vnodes:%d open successfully", tsVnodes.totalVnodes);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeCloseVnodes() {
- int32_t numOfVnodes = 0;
- SVnodeObj **pVnodes = dnodeGetVnodesFromHash(&numOfVnodes);
-
- for (int32_t i = 0; i < numOfVnodes; ++i) {
- dnodeDropVnodeWrapper(pVnodes[i]);
- }
- if (pVnodes != NULL) {
- free(pVnodes);
- }
-
- if (tsVnodes.hash != NULL) {
- taosHashCleanup(tsVnodes.hash);
- tsVnodes.hash = NULL;
- }
-
- dInfo("total vnodes:%d are all closed", numOfVnodes);
-}
-
-static int32_t dnodeParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
- SCreateVnodeMsg *pCreate = rpcMsg->pCont;
- *vgId = htonl(pCreate->vgId);
-
- tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
- pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
- pCfg->totalBlocks = htonl(pCreate->totalBlocks);
- pCfg->daysPerFile = htonl(pCreate->daysPerFile);
- pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
- pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
- pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
- pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
- pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
- pCfg->precision = pCreate->precision;
- pCfg->compression = pCreate->compression;
- pCfg->cacheLastRow = pCreate->cacheLastRow;
- pCfg->update = pCreate->update;
- pCfg->quorum = pCreate->quorum;
- pCfg->replica = pCreate->replica;
- pCfg->walLevel = pCreate->walLevel;
- pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
-
- for (int32_t i = 0; i < pCfg->replica; ++i) {
- pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
- tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
- }
-
- return 0;
-}
-
-static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
- SDropVnodeMsg *pDrop = rpcMsg->pCont;
- pDrop->vgId = htonl(pDrop->vgId);
- return pDrop;
-}
-
-static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
- SAuthVnodeMsg *pAuth = rpcMsg->pCont;
- pAuth->vgId = htonl(pAuth->vgId);
- return pAuth;
-}
-
-static int32_t vnodeProcessCreateVnodeReq(SRpcMsg *rpcMsg) {
- SVnodeCfg vnodeCfg = {0};
- int32_t vgId = 0;
-
- dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
- dDebug("vgId:%d, create vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode != NULL) {
- dDebug("vgId:%d, already exist, return success", vgId);
- dnodeReleaseVnode(pVnode);
- return 0;
- }
-
- int32_t code = dnodeCreateVnode(vgId, &vnodeCfg);
- if (code != 0) {
- dError("vgId:%d, failed to create vnode since %s", vgId, tstrerror(code));
- }
-
- return code;
-}
-
-static int32_t vnodeProcessAlterVnodeReq(SRpcMsg *rpcMsg) {
- SVnodeCfg vnodeCfg = {0};
- int32_t vgId = 0;
- int32_t code = 0;
-
- dnodeParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
- dDebug("vgId:%d, alter vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- code = terrno;
- dDebug("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code));
- return code;
- }
-
- code = vnodeAlter(pVnode->pImpl, &vnodeCfg);
- if (code != 0) {
- dError("vgId:%d, failed to alter vnode since %s", vgId, tstrerror(code));
- }
-
- dnodeReleaseVnode(pVnode);
- return code;
-}
-
-static int32_t vnodeProcessDropVnodeReq(SRpcMsg *rpcMsg) {
- SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg);
-
- int32_t code = 0;
- int32_t vgId = pDrop->vgId;
- dDebug("vgId:%d, drop vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- code = terrno;
- dDebug("vgId:%d, failed to drop since %s", vgId, tstrerror(code));
- return code;
- }
-
- code = dnodeDropVnode(pVnode);
- if (code != 0) {
- dnodeReleaseVnode(pVnode);
- dError("vgId:%d, failed to drop vnode since %s", vgId, tstrerror(code));
- }
-
- return code;
-}
-
-static int32_t vnodeProcessAuthVnodeReq(SRpcMsg *rpcMsg) {
- SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
-
- int32_t code = 0;
- int32_t vgId = pAuth->vgId;
- dDebug("vgId:%d, auth vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- code = terrno;
- dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code));
- return code;
- }
-
- pVnode->accessState = pAuth->accessState;
- dnodeReleaseVnode(pVnode);
- return code;
-}
-
-static int32_t vnodeProcessSyncVnodeReq(SRpcMsg *rpcMsg) {
- SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg);
-
- int32_t code = 0;
- int32_t vgId = pAuth->vgId;
- dDebug("vgId:%d, auth vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- code = terrno;
- dDebug("vgId:%d, failed to auth since %s", vgId, tstrerror(code));
- return code;
- }
-
- code = vnodeSync(pVnode->pImpl);
- if (code != 0) {
- dError("vgId:%d, failed to auth vnode since %s", vgId, tstrerror(code));
- }
-
- dnodeReleaseVnode(pVnode);
- return code;
-}
-
-static int32_t vnodeProcessCompactVnodeReq(SRpcMsg *rpcMsg) {
- SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg);
-
- int32_t code = 0;
- int32_t vgId = pCompact->vgId;
- dDebug("vgId:%d, compact vnode req is received", vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- code = terrno;
- dDebug("vgId:%d, failed to compact since %s", vgId, tstrerror(code));
- return code;
- }
-
- code = vnodeCompact(pVnode->pImpl);
- if (code != 0) {
- dError("vgId:%d, failed to compact vnode since %s", vgId, tstrerror(code));
- }
-
- dnodeReleaseVnode(pVnode);
- return code;
-}
-
-static void dnodeProcessVnodeMgmtQueue(void *unused, SRpcMsg *pMsg) {
- int32_t code = 0;
-
- switch (pMsg->msgType) {
- case TSDB_MSG_TYPE_CREATE_VNODE_IN:
- code = vnodeProcessCreateVnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_ALTER_VNODE_IN:
- code = vnodeProcessAlterVnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_DROP_VNODE_IN:
- code = vnodeProcessDropVnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_AUTH_VNODE_IN:
- code = vnodeProcessAuthVnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_SYNC_VNODE_IN:
- code = vnodeProcessSyncVnodeReq(pMsg);
- break;
- case TSDB_MSG_TYPE_COMPACT_VNODE_IN:
- code = vnodeProcessCompactVnodeReq(pMsg);
- break;
- default:
- code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- break;
- }
-
- SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
- rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
- taosFreeQitem(pMsg);
-}
-
-static void dnodeProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
- vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY);
-}
-
-static void dnodeProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
- vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH);
-}
-
-static void dnodeProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
- SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
- SRpcMsg *pRpcMsg = NULL;
-
- for (int32_t i = 0; i < numOfMsgs; ++i) {
- taosGetQitem(qall, (void **)&pRpcMsg);
- vnodeAppendMsg(pMsg, pRpcMsg);
- taosFreeQitem(pRpcMsg);
- }
-
- vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE);
-}
-
-static void dnodeProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
- SVnodeMsg *pMsg = NULL;
- for (int32_t i = 0; i < numOfMsgs; ++i) {
- taosGetQitem(qall, (void **)&pMsg);
- vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY);
- }
-}
-
-static void dnodeProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
- SVnodeMsg *pMsg = NULL;
- for (int32_t i = 0; i < numOfMsgs; ++i) {
- taosGetQitem(qall, (void **)&pMsg);
- vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC);
- }
-}
-
-static int32_t dnodeWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
- int32_t code = 0;
-
- if (pQueue == NULL) {
- code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- } else {
- SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
- if (pMsg == NULL) {
- code = TSDB_CODE_DND_OUT_OF_MEMORY;
- } else {
- *pMsg = *pRpcMsg;
- code = taosWriteQitem(pQueue, pMsg);
- }
- }
-
- if (code != TSDB_CODE_SUCCESS) {
- SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
- rpcSendResponse(&rsp);
- rpcFreeCont(pRpcMsg->pCont);
- }
-}
-
-static int32_t dnodeWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
- int32_t code = 0;
-
- if (pQueue == NULL) {
- code = TSDB_CODE_DND_MSG_NOT_PROCESSED;
- } else {
- SVnodeMsg *pMsg = vnodeInitMsg(1);
- if (pMsg == NULL) {
- code = TSDB_CODE_DND_OUT_OF_MEMORY;
- } else {
- vnodeAppendMsg(pMsg, pRpcMsg);
- code = taosWriteQitem(pQueue, pMsg);
- }
- }
-
- if (code != TSDB_CODE_SUCCESS) {
- SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
- rpcSendResponse(&rsp);
- rpcFreeCont(pRpcMsg->pCont);
- }
-}
-
-static SVnodeObj *dnodeAcquireVnodeFromMsg(SRpcMsg *pMsg) {
- SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
- pHead->vgId = htonl(pHead->vgId);
-
- SVnodeObj *pVnode = dnodeAcquireVnode(pHead->vgId);
- if (pVnode == NULL) {
- SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
- rpcSendResponse(&rsp);
- rpcFreeCont(pMsg->pCont);
- }
-
- return pVnode;
-}
-
-void dnodeProcessVnodeMgmtMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { dnodeWriteRpcMsgToVnodeQueue(tsVnodes.pMgmtQ, pMsg); }
-
-void dnodeProcessVnodeWriteMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
- if (pVnode != NULL) {
- dnodeWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg);
- dnodeReleaseVnode(pVnode);
- }
-}
-
-void dnodeProcessVnodeSyncMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
- if (pVnode != NULL) {
- dnodeWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
- dnodeReleaseVnode(pVnode);
- }
-}
-
-void dnodeProcessVnodeQueryMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
- if (pVnode != NULL) {
- dnodeWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
- dnodeReleaseVnode(pVnode);
- }
-}
-
-void dnodeProcessVnodeFetchMsg(SRpcMsg *pMsg, SEpSet *pEpSet) {
- SVnodeObj *pVnode = dnodeAcquireVnodeFromMsg(pMsg);
- if (pVnode != NULL) {
- dnodeWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
- dnodeReleaseVnode(pVnode);
- }
-}
-
-static int32_t dnodePutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) {
- SVnodeObj *pVnode = dnodeAcquireVnode(vgId);
- if (pVnode == NULL) {
- return terrno;
- }
-
- int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg);
- dnodeReleaseVnode(pVnode);
- return code;
-}
-
-static int32_t dnodeInitVnodeMgmtWorker() {
- SWorkerPool *pPool = &tsVnodes.mgmtPool;
- pPool->name = "vnode-mgmt";
- pPool->min = 1;
- pPool->max = 1;
- if (tWorkerInit(pPool) != 0) {
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- tsVnodes.pMgmtQ = tWorkerAllocQueue(pPool, NULL, (FProcessItem)dnodeProcessVnodeMgmtQueue);
- if (tsVnodes.pMgmtQ == NULL) {
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- return 0;
-}
-
-static void dnodeCleanupVnodeMgmtWorker() {
- tWorkerFreeQueue(&tsVnodes.mgmtPool, tsVnodes.pMgmtQ);
- tWorkerCleanup(&tsVnodes.mgmtPool);
- tsVnodes.pMgmtQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeQueryQueue(SVnodeObj *pVnode) {
- pVnode->pQueryQ = tWorkerAllocQueue(&tsVnodes.queryPool, pVnode, (FProcessItem)dnodeProcessVnodeQueryQueue);
- if (pVnode->pQueryQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeVnodeQueryQueue(SVnodeObj *pVnode) {
- tWorkerFreeQueue(&tsVnodes.queryPool, pVnode->pQueryQ);
- pVnode->pQueryQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeFetchQueue(SVnodeObj *pVnode) {
- pVnode->pFetchQ = tWorkerAllocQueue(&tsVnodes.fetchPool, pVnode, (FProcessItem)dnodeProcessVnodeFetchQueue);
- if (pVnode->pFetchQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeVnodeFetchQueue(SVnodeObj *pVnode) {
- tWorkerFreeQueue(&tsVnodes.fetchPool, pVnode->pFetchQ);
- pVnode->pFetchQ = NULL;
-}
-
-static int32_t dnodeInitVnodeReadWorker() {
- int32_t maxFetchThreads = 4;
- float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
-
- SWorkerPool *pPool = &tsVnodes.queryPool;
- pPool->name = "vnode-query";
- pPool->min = (int32_t)threadsForQuery;
- pPool->max = pPool->min;
- if (tWorkerInit(pPool) != 0) {
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- pPool = &tsVnodes.fetchPool;
- pPool->name = "vnode-fetch";
- pPool->min = MIN(maxFetchThreads, tsNumOfCores);
- pPool->max = pPool->min;
- if (tWorkerInit(pPool) != 0) {
- TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- return 0;
-}
-
-static void dnodeCleanupVnodeReadWorker() {
- tWorkerCleanup(&tsVnodes.fetchPool);
- tWorkerCleanup(&tsVnodes.queryPool);
-}
-
-static int32_t dnodeAllocVnodeWriteQueue(SVnodeObj *pVnode) {
- pVnode->pWriteQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeWriteQueue);
- if (pVnode->pWriteQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeVnodeWriteQueue(SVnodeObj *pVnode) {
- tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pWriteQ);
- pVnode->pWriteQ = NULL;
-}
-
-static int32_t dnodeAllocVnodeApplyQueue(SVnodeObj *pVnode) {
- pVnode->pApplyQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeApplyQueue);
- if (pVnode->pApplyQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeVnodeApplyQueue(SVnodeObj *pVnode) {
- tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pApplyQ);
- pVnode->pApplyQ = NULL;
-}
-
-static int32_t dnodeInitVnodeWriteWorker() {
- SMWorkerPool *pPool = &tsVnodes.writePool;
- pPool->name = "vnode-write";
- pPool->max = tsNumOfCores;
- if (tMWorkerInit(pPool) != 0) {
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- return 0;
-}
-
-static void dnodeCleanupVnodeWriteWorker() { tMWorkerCleanup(&tsVnodes.writePool); }
-
-static int32_t dnodeAllocVnodeSyncQueue(SVnodeObj *pVnode) {
- pVnode->pSyncQ = tMWorkerAllocQueue(&tsVnodes.writePool, pVnode, (FProcessItems)dnodeProcessVnodeSyncQueue);
- if (pVnode->pSyncQ == NULL) {
- return TSDB_CODE_DND_OUT_OF_MEMORY;
- }
- return 0;
-}
-
-static void dnodeFreeVnodeSyncQueue(SVnodeObj *pVnode) {
- tMWorkerFreeQueue(&tsVnodes.writePool, pVnode->pSyncQ);
- pVnode->pSyncQ = NULL;
-}
-
-static int32_t dnodeInitVnodeSyncWorker() {
- int32_t maxThreads = tsNumOfCores / 2;
- if (maxThreads < 1) maxThreads = 1;
-
- SMWorkerPool *pPool = &tsVnodes.writePool;
- pPool->name = "vnode-sync";
- pPool->max = maxThreads;
- if (tMWorkerInit(pPool) != 0) {
- return TSDB_CODE_VND_OUT_OF_MEMORY;
- }
-
- return 0;
-}
-
-static void dnodeCleanupVnodeSyncWorker() { tMWorkerCleanup(&tsVnodes.syncPool); }
-
-static int32_t dnodeInitVnodeModule() {
- SVnodePara para;
- para.sendMsgToDnodeFp = dnodeSendMsgToDnode;
- para.sendMsgToMnodeFp = dnodeSendMsgToMnode;
- para.putMsgToApplyQueueFp = dnodePutMsgIntoVnodeApplyQueue;
-
- return vnodeInit(para);
-}
-
-int32_t dnodeInitVnodes() {
- dInfo("dnode-vnodes start to init");
-
- SSteps *pSteps = taosStepInit(6, dnodeReportStartup);
- taosStepAdd(pSteps, "dnode-vnode-env", dnodeInitVnodeModule, vnodeCleanup);
- taosStepAdd(pSteps, "dnode-vnode-mgmt", dnodeInitVnodeMgmtWorker, dnodeCleanupVnodeMgmtWorker);
- taosStepAdd(pSteps, "dnode-vnode-read", dnodeInitVnodeReadWorker, dnodeCleanupVnodeReadWorker);
- taosStepAdd(pSteps, "dnode-vnode-write", dnodeInitVnodeWriteWorker, dnodeCleanupVnodeWriteWorker);
- taosStepAdd(pSteps, "dnode-vnode-sync", dnodeInitVnodeSyncWorker, dnodeCleanupVnodeSyncWorker);
- taosStepAdd(pSteps, "dnode-vnodes", dnodeOpenVnodes, dnodeCleanupVnodes);
-
- tsVnodes.pSteps = pSteps;
- return taosStepExec(pSteps);
-}
-
-void dnodeCleanupVnodes() {
- if (tsVnodes.pSteps != NULL) {
- dInfo("dnode-vnodes start to clean up");
- taosStepCleanup(tsVnodes.pSteps);
- tsVnodes.pSteps = NULL;
- dInfo("dnode-vnodes is cleaned up");
- }
-}
-
-void dnodeGetVnodeLoads(SVnodeLoads *pLoads) {
- pLoads->num = taosHashGetSize(tsVnodes.hash);
-
- int32_t v = 0;
- void *pIter = taosHashIterate(tsVnodes.hash, NULL);
- while (pIter) {
- SVnodeObj **ppVnode = pIter;
- if (ppVnode == NULL) continue;
-
- SVnodeObj *pVnode = *ppVnode;
- if (pVnode == NULL) continue;
-
- SVnodeLoad *pLoad = &pLoads->data[v++];
- vnodeGetLoad(pVnode->pImpl, pLoad);
- pLoad->vgId = htonl(pLoad->vgId);
- pLoad->totalStorage = htobe64(pLoad->totalStorage);
- pLoad->compStorage = htobe64(pLoad->compStorage);
- pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
- pLoad->tablesNum = htobe64(pLoad->tablesNum);
-
- pIter = taosHashIterate(tsVnodes.hash, pIter);
- }
-}
diff --git a/source/dnode/mnode/impl/inc/mnodeDef.h b/source/dnode/mnode/impl/inc/mnodeDef.h
index ccdba13006..4b4c4abdb3 100644
--- a/source/dnode/mnode/impl/inc/mnodeDef.h
+++ b/source/dnode/mnode/impl/inc/mnodeDef.h
@@ -131,7 +131,7 @@ typedef struct SMnodeObj {
int64_t roleTime;
int64_t createdTime;
int64_t updateTime;
- SDnodeObj *pDnode;
+ SDnodeObj *pDnd;
} SMnodeObj;
typedef struct {
@@ -215,7 +215,7 @@ typedef struct SDbObj {
typedef struct {
int32_t dnodeId;
int8_t role;
- SDnodeObj *pDnode;
+ SDnodeObj *pDnd;
} SVnodeGid;
typedef struct SVgObj {
diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c
index bc4718ee5b..877ec1431c 100644
--- a/source/dnode/mnode/impl/src/mnode.c
+++ b/source/dnode/mnode/impl/src/mnode.c
@@ -83,7 +83,7 @@ static int32_t mnodeSetOptions(SMnode *pMnode, const SMnodeOptions *pOptions) {
pMnode->replica = pOptions->replica;
pMnode->selfIndex = pOptions->selfIndex;
memcpy(&pMnode->replicas, pOptions->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
- pMnode->pServer = pOptions->pServer;
+ pMnode->pServer = pOptions->pDnode;
pMnode->putMsgToApplyMsgFp = pOptions->putMsgToApplyMsgFp;
pMnode->sendMsgToDnodeFp = pOptions->sendMsgToDnodeFp;
pMnode->sendMsgToMnodeFp = pOptions->sendMsgToMnodeFp;
@@ -187,7 +187,7 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
}
if (rpcGetConnInfo(pRpcMsg->handle, &pMsg->conn) != 0) {
- mnodeCleanupMsg(pMnode, pMsg);
+ mnodeCleanupMsg(pMsg);
mError("can not get user from conn:%p", pMsg->rpcMsg.handle);
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
return NULL;
@@ -199,7 +199,7 @@ SMnodeMsg *mnodeInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
return pMsg;
}
-void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
+void mnodeCleanupMsg(SMnodeMsg *pMsg) {
if (pMsg->pUser != NULL) {
sdbRelease(pMsg->pUser);
}
@@ -210,7 +210,7 @@ void mnodeCleanupMsg(SMnode *pMnode, SMnodeMsg *pMsg) {
static void mnodeProcessRpcMsg(SMnodeMsg *pMsg) {
if (!mnodeIsMaster()) {
mnodeSendRedirectMsg(NULL, &pMsg->rpcMsg, true);
- mnodeCleanupMsg(NULL, pMsg);
+ mnodeCleanupMsg(pMsg);
return;
}
diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c
index 934a8dd6ab..a2041c76fc 100644
--- a/source/libs/transport/src/rpcMain.c
+++ b/source/libs/transport/src/rpcMain.c
@@ -54,10 +54,11 @@ typedef struct {
char secret[TSDB_KEY_LEN]; // secret for the link
char ckey[TSDB_KEY_LEN]; // ciphering key
- void (*cfp)(SRpcMsg *, SEpSet *);
- int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey);
+ void (*cfp)(void *parent, SRpcMsg *, SEpSet *);
+ int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey);
int32_t refCount;
+ void *parent;
void *idPool; // handle to ID pool
void *tmrCtrl; // handle to timer
SHashObj *hash; // handle returned by hash utility
@@ -260,6 +261,7 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc->spi = pInit->spi;
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
+ pRpc->parent = pInit->parent;
pRpc->refCount = 1;
atomic_add_fetch_32(&tsRpcNum, 1);
@@ -744,7 +746,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
if (pConn->user[0] == 0) {
terrno = TSDB_CODE_RPC_AUTH_REQUIRED;
} else {
- terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey);
+ terrno = (*pRpc->afp)(pRpc->parent, pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey);
}
if (terrno != 0) {
@@ -1024,8 +1026,8 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) {
SRpcMsg *pRpcMsg = (SRpcMsg *)(param);
SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle);
SRpcInfo *pRpc = pConn->pRpc;
- (*(pRpc->cfp))(pRpcMsg, NULL);
- free(pRpcMsg);
+ (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL);
+ free(pRpcMsg);
}
static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
SRpcInfo *pRpc = pConn->pRpc;
@@ -1137,9 +1139,9 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
// for asynchronous API
SEpSet *pEpSet = NULL;
if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect)
- pEpSet = &pContext->epSet;
+ pEpSet = &pContext->epSet;
- (*pRpc->cfp)(pMsg, pEpSet);
+ (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet);
}
// free the request message
@@ -1155,15 +1157,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = pHead->content;
rpcMsg.msgType = pHead->msgType;
- rpcMsg.code = pHead->code;
-
- if ( rpcIsReq(pHead->msgType) ) {
+ rpcMsg.code = pHead->code;
+
+ if (rpcIsReq(pHead->msgType)) {
rpcMsg.ahandle = pConn->ahandle;
rpcMsg.handle = pConn;
rpcAddRef(pRpc); // add the refCount for requests
// notify the server app
- (*(pRpc->cfp))(&rpcMsg, NULL);
+ (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
} else {
// it's a response
rpcMsg.handle = pContext;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 0b10274c00..6d4f4eb6a7 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -78,6 +78,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error")
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message")
+TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed")
@@ -235,20 +236,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS, "Invalid topic partito
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists")
// dnode
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, "Message not processed")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, "Dnode out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_MATCH_DNODE, "Mnode Id not match Dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_READ_FILE_ERROR, "Read dnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_WRITE_FILE_ERROR, "Write dnode.json error")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED, "Mnode already deployed")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_NOT_DEPLOYED, "Mnode not deployed")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_READ_MNODE_FILE_ERROR, "Read mnode.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_WRITE_MNODE_FILE_ERROR, "Write mnode.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for disk files in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, "Too many vnode directories")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_VNODE_FILE_ERROR, "Parse vnodes.json error")
-TAOS_DEFINE_ERROR(TSDB_CODE_DND_PARSE_DNODE_FILE_ERROR, "Parse dnodes.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_INVALID, "Mnode Id invalid")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ID_NOT_FOUND, "Mnode Id not found")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_READ_FILE_ERROR, "Read mnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR, "Write mnode.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnode directories")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error")
+TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error")
// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress")