Merge branch '3.0' into feature/3.0_liaohj

CMakeLists.txt

@@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 ## cJson
 cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 
+## leveldb
+option(BUILD_WITH_LEVELDB "If build with leveldb" OFF)
+if(${BUILD_WITH_LEVELDB})
+    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_LEVELDB})
+
+## rocksdb
+option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON)
+if(${BUILD_WITH_ROCKSDB})
+    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_ROCKSDB})
+
 ## download dependencies
 configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

cmake/EnableCMP0048.txt.in

@@ -0,0 +1 @@
+cmake_policy(SET CMP0048 NEW)

cmake/leveldb_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# leveldb
+ExternalProject_Add(leveldb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
+        GIT_TAG master
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/rocksdb_CMakeLists.txt.in

@@ -0,0 +1,11 @@
+
+# rocksdb
+ExternalProject_Add(rocksdb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
+        GIT_TAG v6.23.3
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

deps/CMakeLists.txt

@@ -1,23 +1,57 @@
+# googletest
 if(${BUILD_TEST})
     add_subdirectory(googletest)
 endif(${BUILD_TEST})
 
+# cJson
+# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
+option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
+option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
+option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
 add_subdirectory(cJson)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
     PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson>
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
+# lz4
 add_subdirectory(lz4/build/cmake)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
 )
 
+# zlib
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
 add_subdirectory(zlib)
 target_include_directories(
     zlib
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
+
+# leveldb
+if(${BUILD_WITH_LEVELDB})
+option(LEVELDB_BUILD_TESTS "" OFF)
+    add_subdirectory(leveldb)
+    target_include_directories(
+        leveldb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
+    )
+endif(${BUILD_WITH_LEVELDB})
+
+# rocksdb
+if(${BUILD_WITH_ROCKSDB})
+    option(WITH_TESTS "" OFF)
+    option(WITH_BENCHMARK_TOOLS "" OFF)
+    option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
+    add_subdirectory(rocksdb)
+    target_include_directories(
+        rocksdb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
+    )
+endif(${BUILD_WITH_ROCKSDB})

include/common/taosmsg.h

@@ -990,6 +990,22 @@ typedef struct {
   /* data */
 } SAlterTableRsp;
 
+typedef struct {
+  /* data */
+} SDropStableReq;
+
+typedef struct {
+  /* data */
+} SDropStableRsp;
+
+typedef struct {
+  /* data */
+} SUpdateTagValReq;
+
+typedef struct {
+  /* data */
+} SUpdateTagValRsp;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tcompare.h

@@ -13,24 +13,24 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_READ_H
-#define TDENGINE_VNODE_READ_H
+#ifndef _TD_TCOMPARE_H_
+#define _TD_TCOMPARE_H_
+
+#include "compare.h"
+#include "ttypes.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "vnodeInt.h"
 
-int32_t vnodeInitRead(void);
-void    vnodeCleanupRead(void);
-
-int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
-void    vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead);
-int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
-void    vnodeWaitReadCompleted(SVnodeObj *pVnode);
+int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
+int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
+int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_TCOMPARE_H_*/

include/common/tglobal.h

@@ -57,6 +57,7 @@ extern int32_t  tsCompressMsgSize;
 extern int32_t  tsCompressColData;
 extern int32_t  tsMaxNumOfDistinctResults;
 extern char     tsTempDir[];
+extern int64_t  tsMaxVnodeQueuedBytes;
 
 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer size in MB for each data node during query processing

include/common/trow.h

@@ -16,10 +16,55 @@
 #ifndef _TD_COMMON_ROW_H_
 #define _TD_COMMON_ROW_H_
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// types
+typedef void *                  SRow;
+typedef struct SRowBatch        SRowBatch;
+typedef struct SRowBuilder      SRowBuilder;
+typedef struct SRowBatchIter    SRowBatchIter;
+typedef struct SRowBatchBuilder SRowBatchBuilder;
+
+// SRow
+#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t))
+#define rowType(r) (*(uint8_t *)(r))                                // row type
+#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t)))  // row length
+#define rowSVer(r) \
+  (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t)))  // row schema version, only for SDataRow
+#define rowNCols(r) rowSVer(r)                                         // only for SKVRow
+#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t)))  // row version
+#define rowCopy(dest, r) memcpy((dest), r, rowLen(r))
+
+static FORCE_INLINE SRow rowDup(SRow row) {
+  SRow r = malloc(rowLen(row));
+  if (r == NULL) {
+    return NULL;
+  }
+
+  rowCopy(r, row);
+
+  return r;
+}
+
+// SRowBatch
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate();
+void         rowBuilderDestroy(SRowBuilder *);
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *);
+void           rowBatchIterDestroy(SRowBatchIter *);
+const SRow     rowBatchIterNext(SRowBatchIter *);
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);
+
 #ifdef __cplusplus
 }
 #endif

include/common/ttypes.h

@@ -6,10 +6,10 @@ extern "C" {
 #endif
 
 #include "taosdef.h"
+#include "types.h"
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
 typedef int32_t  VarDataOffsetT;
-typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
 typedef uint16_t TDRowLenT;    // not including overhead: 0 ~ 65535
 typedef uint32_t TDRowTLenT;   // total length, including overhead
 
@@ -30,11 +30,7 @@ typedef struct {
 } SNCharNullT;
 #pragma pack(pop)
 
-#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
-
-#define varDataLen(v)       ((VarDataLenT *)(v))[0]
 #define varDataTLen(v)      (sizeof(VarDataLenT) + varDataLen(v))
-#define varDataVal(v)       ((void *)((char *)v + VARSTR_HEADER_SIZE))
 #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
 #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))

include/libs/raft/raft.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
  * it under the terms of the GNU Affero General Public License, version 3
@@ -13,15 +13,149 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_RAFT_H_
-#define _TD_RAFT_H_
+#ifndef TD_RAFT_H
+#define TD_RAFT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <stdio.h>
+#include "taosdef.h"
+
+typedef unsigned int RaftId;
+typedef unsigned int RaftGroupId;
+
+// buffer holding data
+typedef struct RaftBuffer {
+  void* data;
+  size_t len;
+} RaftBuffer;
+
+// a single server information in a cluster
+typedef struct RaftServer {
+  RaftId    id;
+  char      fqdn[TSDB_FQDN_LEN];
+  uint16_t  port;
+} RaftServer;
+
+// all servers in a cluster
+typedef struct RaftConfiguration {
+  RaftServer *servers;
+  int nServer;
+} RaftConfiguration;
+
+// raft lib module
+struct Raft;
+typedef struct Raft Raft;
+
+struct RaftNode;
+typedef struct RaftNode RaftNode;
+
+// raft state machine
+struct RaftFSM;
+typedef struct RaftFSM {
+  // statemachine user data
+  void *data;
+
+  // apply buffer data, bufs will be free by raft module
+  int (*apply)(struct RaftFSM *fsm, const RaftBuffer *bufs[], int nBufs);
+
+  // configuration commit callback 
+  int (*onConfigurationCommit)(const RaftConfiguration* cluster);
+
+  // fsm return snapshot in ppBuf, bufs will be free by raft module
+  // TODO: getSnapshot SHOULD be async?
+  int (*getSnapshot)(struct RaftFSM *fsm, RaftBuffer **ppBuf);
+
+  // fsm restore with pBuf data
+  int (*restore)(struct RaftFSM *fsm, RaftBuffer *pBuf);
+
+  // fsm send data in buf to server,buf will be free by raft module
+  int (*send)(struct RaftFSM* fsm, const RaftServer* server, const RaftBuffer *buf);
+} RaftFSM;
+
+typedef struct RaftNodeOptions {
+  // user define state machine
+  RaftFSM* pFSM;
+
+  // election timeout(in ms)
+  // by default: 1000
+  int electionTimeoutMS;
+
+  // heart timeout(in ms)
+  // by default: 100
+  int heartbeatTimeoutMS;
+
+  // install snapshot timeout(in ms)
+  int installSnapshotTimeoutMS;
+
+  /** 
+   * number of log entries before starting a new snapshot. 
+   * by default: 1024
+   */
+  int snapshotThreshold;
+
+  /**
+   * Number of log entries to keep in the log after a snapshot has
+   * been taken. 
+   * by default: 128.
+   */
+  int snapshotTrailing;
+
+  /**
+   * Enable or disable pre-vote support.
+   * by default: false
+   */
+  bool preVote;
+
+} RaftNodeOptions;
+
+// create raft lib
+int RaftCreate(Raft** ppRaft);
+
+int RaftDestroy(Raft* pRaft);
+
+// start a raft node with options,node id,group id
+int RaftStart(Raft* pRaft, 
+              RaftId selfId,
+              RaftGroupId selfGroupId,
+              const RaftConfiguration* cluster,
+              const RaftNodeOptions* options,              
+              RaftNode **ppNode);
+
+// stop a raft node
+int RaftStop(RaftNode* pNode);
+
+// client apply a cmd in buf
+typedef void (*RaftApplyFp)(const RaftBuffer *pBuf, int result);
+
+int RaftApply(RaftNode *pNode,
+              const RaftBuffer *pBuf,
+              RaftApplyFp applyCb);
+
+// recv data from other servers in cluster,buf will be free in raft
+int RaftRecv(RaftNode *pNode, const RaftBuffer* pBuf);
+
+// change cluster servers API
+typedef void (*RaftChangeFp)(const RaftServer* pServer, int result);
+
+int RaftAddServer(RaftNode *pNode,
+                  const RaftServer* pServer,
+                  RaftChangeFp changeCb);
+
+int RaftRemoveServer(RaftNode *pNode, 
+                    const RaftServer* pServer,
+                    RaftChangeFp changeCb);
+
+// transfer leader to id
+typedef void (*RaftTransferFp)(RaftId id, int result);
+int RaftTransfer(RaftNode *pNode,
+                 RaftId id,
+                 RaftTransferFp transferCb);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_RAFT_H_*/
+#endif  /* TD_RAFT_H */ 

include/libs/tkv/tkv.h

@@ -20,52 +20,25 @@
 extern "C" {
 #endif
 
-typedef struct tkv_db_s tkv_db_t;
+// Types exported
+typedef struct STkvDb      STkvDb;
+typedef struct STkvOptions STkvOptions;
+typedef struct STkvCache   STkvCache;
 
-typedef struct {
-  /* data */
-} tkv_key_t;
+// DB operations
+STkvDb *tkvOpen(const STkvOptions *options, const char *path);
+void    tkvClose(STkvDb *db);
+void    tkvPut(STkvDb *db, void * /*TODO*/);
 
-typedef struct {
-  bool    pinned;
-  int64_t ref;  // TODO: use util library
-                // TODO: add a RW latch here
-  uint64_t offset;
-  void *   pObj;
-} tkv_obj_t;
+// DB options
+STkvOptions *tkvOptionsCreate();
+void         tkvOptionsDestroy(STkvOptions *);
+void         tkvOptionsSetCache(STkvOptions *, STkvCache *);
 
-typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *);
-typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *);
-typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj);
-typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj);
-typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
-typedef void (*tkv_obj_destroy_fn_t)(void *);
-
-typedef struct {
-  uint64_t             memLimit;
-  tkv_get_key_fn_t     getKey;
-  tkv_obj_encode_fn_t  encode;
-  tkv_obj_decode_fn_t  decode;
-  tkv_obj_comp_fn_t    compare;
-  tkv_obj_destroy_fn_t destroy;
-} tkv_db_option_t;
-
-tkv_db_t *       tkvOpenDB(char *dir, tkv_db_option_t *);
-int              tkvCloseDB(tkv_db_t *);
-int              tkvPut(tkv_db_t *, tkv_obj_t *);
-int              tkvPutBatch(tkv_db_t *, tkv_obj_t **, int);  // TODO: use array here
-const tkv_obj_t *tkvGet(tkv_key_t *);
-int              tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **);  // TODO: use array here
-int              tkvDrop(tkv_db_t *, tkv_key_t *);
-int              tkvDropBatch(tkv_db_t *, tkv_key_t **, int);  // TODO: use array here
-int              tkvCommit(tkv_db_t *, void * /*TODO*/);
-
-typedef struct {
-} tkv_db_iter_t;
-
-tkv_db_iter_t *  tkvIterNew(tkv_db_t *);
-void             tkvIterFree(tkv_db_iter_t *);
-const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
+// DB cache
+typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
+STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
+void       tkvCacheDestroy(STkvCache *);
 
 #ifdef __cplusplus
 }

include/libs/wal/wal.h

@@ -60,8 +60,8 @@ void     walClose(twalh);
 //write
 int64_t  walWrite(twalh, int8_t msgType, void* body, uint32_t bodyLen);
 void     walFsync(twalh, bool forceHint);
-//int32_t  walCommit(twalh, uint64_t ver);
-//int32_t  walRollback(twalh, uint64_t ver);
+//int32_t  walCommit(twalh, int64_t ver);
+//int32_t  walRollback(twalh, int64_t ver);
 
 //read
 int32_t  walRead(twalh, SWalHead **, int64_t ver);

include/server/mnode/mnode.h

@@ -118,7 +118,7 @@ typedef struct {
 int32_t mnodeGetStatistics(SMnodeStat *stat);
 
 /**
- * Get the auth information.
+ * Get the auth information of Mnode.
  *
  * @param user, username.
  * @param spi,  security parameter index.

include/server/vnode/meta/meta.h

@@ -18,16 +18,42 @@
 
 #include "taosmsg.h"
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SMeta SMeta;
+typedef uint64_t tuid_t;
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq);
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq);
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq);
-int metaCommit(SMeta *pMeta);
+// Types exported
+typedef struct SMeta             SMeta;
+typedef struct SMetaOptions      SMetaOptions;
+typedef struct SMetaQueryHandle  SMetaQueryHandle;
+typedef struct SMetaQueryOptions SMetaQueryOptions;
+
+// SMeta operations
+int    metaCreate(const char *path);
+int    metaDestroy(const char *path);
+SMeta *metaOpen(SMetaOptions *);
+void   metaClose(SMeta *);
+int    metaCreateTable(SMeta *, void *);
+int    metaDropTable(SMeta *, uint64_t tuid_t);
+int    metaAlterTable(SMeta *, void *);
+int    metaCommit(SMeta *);
+
+// Options
+SMetaOptions *metaOptionsCreate();
+void          metaOptionsDestroy(SMetaOptions *);
+void          metaOptionsSetCache(SMetaOptions *, size_t capacity);
+
+// SMetaQueryHandle
+SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *);
+void              metaQueryHandleDestroy(SMetaQueryHandle *);
+
+// SMetaQueryOptions
+SMetaQueryOptions *metaQueryOptionsCreate();
+void               metaQueryOptionsDestroy(SMetaQueryOptions *);
 
 #ifdef __cplusplus
 }

include/server/vnode/tq/tq.h

@@ -25,23 +25,14 @@ extern "C" {
 typedef struct STQ STQ;
 
 STQ* tqInit(void* ref_func(void*), void* unref_func(void*));
-void tqCleanUp(STQ* pTq);
+void tqCleanUp(STQ*);
 
-//create persistent storage for meta info such as consuming offset
-//return value > 0: cgId
-//return value <= 0: error code
-int tqCreateGroup(STQ*);
-//create ring buffer in memory and load consuming offset
-int tqOpenGroup(STQ*, int cgId);
-//destroy ring buffer and persist consuming offset
-int tqCloseGroup(STQ*, int cgId);
-//delete persistent storage for meta info
-int tqDropGroup(STQ*, int cgId);
-
-int tqPushMsg(STQ*, void *, int64_t version);
+//void* will be replace by a msg type
+int tqPushMsg(STQ*, void* msg, int64_t version);
 int tqCommit(STQ*);
 
-int tqHandleMsg(STQ*, void *msg);
+//void* will be replace by a msg type
+int tqHandleMsg(STQ*, void* msg);
 
 #ifdef __cplusplus
 }

include/server/vnode/tsdb/tsdb.h

@@ -23,33 +23,60 @@
 extern "C" {
 #endif
 
-typedef struct STsdb STsdb;
-typedef struct {
-  int32_t id; // TODO: use a global definition
-  int32_t days;
-  int32_t keep;
-  int32_t keep1;
-  int32_t keep2;
-  int32_t minRows;
-  int32_t maxRows;
-  int8_t  precision;
-  int8_t  update;
-} STsdbCfg;
+// Types exported
+typedef struct STsdb             STsdb;
+typedef struct STsdbOptions      STsdbOptions;
+typedef struct STsdbSMAOptions   STsdbSMAOptions;  // SMA stands for Small Materialized Aggregation
+typedef struct STsdbReadOptions  STsdbReadOptions;
+typedef struct STsdbSnapshot     STsdbSnapshot;
+typedef struct STsdbQueryHandle  STsdbQueryHandle;
 
-// Module init and clear
-int tsdbInit();
-int tsdbClear();
+// DB operations
+int    tsdbCreate(const char *path);
+int    tsdbDestroy(const char *path);
+STsdb *tsdbOpen(const STsdbOptions *options);
+void   tsdbClose(STsdb *);
+int    tsdbReset(STsdb *, const STsdbOptions *);
+int    tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *);
+int    tsdbCommit(STsdb *);
+int    tsdbCompact(STsdb *);
 
-// Repository operations
-int    tsdbCreateRepo(int id);
-int    tsdbDropRepo(int id);
-STsdb *tsdbOpenRepo(STsdbCfg *pCfg);
-int    tsdbCloseRepo(STsdb *pTsdb);
-int    tsdbForceCloseRepo(STsdb *pTsdb);
+// Options
+STsdbOptions *tsdbOptionsCreate();
+void          tsdbOptionsDestroy(STsdbOptions *);
+void          tsdbOptionsSetId(STsdbOptions *, int id);
+void          tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours);
+void          tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2);
+void          tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows);
+void          tsdbOptionsSetPrecision(STsdbOptions *, int);
+void          tsdbOptionsSetCache(STsdbOptions *, int);
+typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType;
+void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType);
+void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *);
 
-// Data commit
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg);
-int tsdbCommit(STsdb *pTsdb);
+// STsdbSMAOptions
+STsdbSMAOptions *tsdbSMAOptionsCreate();
+void             tsdbSMAOptionsDestroy(STsdbSMAOptions *);
+// void             tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/);
+// void             tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *);
+// void             tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *);
+
+// STsdbQueryHandle
+STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *);
+void              tsdbQueryHandleDestroy(STsdbQueryHandle *);
+void              tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *);
+bool              tsdbNextDataBlock(STsdbQueryHandle *);
+// void              tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *);
+// void              tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **);
+
+// STsdbReadOptions
+STsdbReadOptions *tsdbReadOptionsCreate();
+void              tsdbReadOptionsDestroy(STsdbReadOptions *);
+void              tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *);
+
+// STsdbSnapshot
+STsdbSnapshot *tsdbSnapshotCreate(STsdb *);
+void           tsdbSnapshotDestroy(STsdbSnapshot *);
 
 #ifdef __cplusplus
 }

include/server/vnode/vnode.h

@@ -49,7 +49,7 @@ typedef struct {
 } SVnodeFp;
 
 typedef struct {
-  SVnodeFp      fp;
+  SVnodeFp fp;
 } SVnodePara;
 
 /**

include/util/compare.h

@@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte
 
 int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
-
-__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr);
 
 int32_t taosArrayCompareString(const void* a, const void* b);
 
@@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
 
 int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
 int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -104,33 +104,6 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
-#define GET_INT8_VAL(x)    (*(int8_t *)(x))
-#define GET_INT16_VAL(x)   (*(int16_t *)(x))
-#define GET_INT32_VAL(x)   (*(int32_t *)(x))
-#define GET_INT64_VAL(x)   (*(int64_t *)(x))
-#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
-#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
-#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
-#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
-
-#ifdef _TD_ARM_32
-  float  taos_align_get_float(const char* pBuf);
-  double taos_align_get_double(const char* pBuf);
-
-  #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
-  #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
-  #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
-  #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
-  #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
-  #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
-#else
-  #define GET_FLOAT_VAL(x)       (*(float *)(x))
-  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
-  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
-  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
-  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
-  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
-#endif
 
 // TODO: check if below is necessary
 #define TSDB_RELATION_INVALID     0

include/util/tstep.h

@@ -20,14 +20,14 @@
 extern "C" {
 #endif
 
-typedef int32_t (*InitFp)(void **obj);
-typedef void (*CleanupFp)(void **obj);
+typedef int32_t (*InitFp)();
+typedef void (*CleanupFp)();
 typedef void (*ReportFp)(char *name, char *desc);
 
 struct SSteps *taosStepInit(int32_t maxsize, ReportFp fp);
 int32_t        taosStepExec(struct SSteps *steps);
 void           taosStepCleanup(struct SSteps *steps);
-int32_t        taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp);
+int32_t        taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp);
 
 #ifdef __cplusplus
 }

include/util/tworker.h

@@ -20,14 +20,16 @@
 extern "C" {
 #endif
 
-typedef int32_t (*ProcessReqFp)(void *ahandle, void *msg);
-typedef void (*SendRspFp)(void *ahandle, void *msg, int32_t qtype, int32_t code);
+typedef int32_t (*ProcessStartFp)(void *ahandle, void *pMsg, int32_t qtype);
+typedef void (*ProcessEndFp)(void *ahandle, void *pMsg, int32_t qtype, int32_t code);
 
-struct SWorkerPool;
+typedef bool (*ProcessWriteStartFp)(void *ahandle, void *pMsg, int32_t qtype);
+typedef void (*ProcessWriteSyncFp)(void *ahandle, int32_t code);
+typedef void (*ProcessWriteEndFp)(void *ahandle, void *pMsg, int32_t qtype);
 
-typedef struct {
-  pthread_t           thread;  // thread
+typedef struct SWorker {
   int32_t             id;      // worker ID
+  pthread_t           thread;  // thread
   struct SWorkerPool *pool;
 } SWorker;
 
@@ -35,18 +37,42 @@ typedef struct SWorkerPool {
   int32_t         max;  // max number of workers
   int32_t         min;  // min number of workers
   int32_t         num;  // current number of workers
-  void *          qset;
+  taos_qset       qset;
   const char *    name;
+  ProcessStartFp  startFp;
+  ProcessEndFp    endFp;
   SWorker *       workers;
-  ProcessReqFp    reqFp;
-  SendRspFp       rspFp;
   pthread_mutex_t mutex;
 } SWorkerPool;
 
-int32_t tWorkerInit(SWorkerPool *pPool);
-void    tWorkerCleanup(SWorkerPool *pPool);
-void *  tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle);
-void    tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue);
+typedef struct SWriteWorker {
+  int32_t                  id;      // worker id
+  pthread_t                thread;  // thread
+  taos_qall                qall;
+  taos_qset                qset;  // queue set
+  struct SWriteWorkerPool *pool;
+} SWriteWorker;
+
+typedef struct SWriteWorkerPool {
+  int32_t             max;     // max number of workers
+  int32_t             nextId;  // from 0 to max-1, cyclic
+  const char *        name;
+  ProcessWriteStartFp startFp;
+  ProcessWriteSyncFp  syncFp;
+  ProcessWriteEndFp   endFp;
+  SWriteWorker *      workers;
+  pthread_mutex_t     mutex;
+} SWriteWorkerPool;
+
+int32_t    tWorkerInit(SWorkerPool *pool);
+void       tWorkerCleanup(SWorkerPool *pool);
+taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle);
+void       tWorkerFreeQueue(SWorkerPool *pool, taos_queue queue);
+
+int32_t    tWriteWorkerInit(SWriteWorkerPool *pool);
+void       tWriteWorkerCleanup(SWriteWorkerPool *pool);
+taos_queue tWriteWorkerAllocQueue(SWriteWorkerPool *pool, void *ahandle);
+void       tWriteWorkerFreeQueue(SWriteWorkerPool *pool, taos_queue queue);
 
 #ifdef __cplusplus
 }

include/util/types.h

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TYPES_H_
+#define _TD_TYPES_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GET_INT8_VAL(x)    (*(int8_t *)(x))
+#define GET_INT16_VAL(x)   (*(int16_t *)(x))
+#define GET_INT32_VAL(x)   (*(int32_t *)(x))
+#define GET_INT64_VAL(x)   (*(int64_t *)(x))
+#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
+#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
+#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
+#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
+
+static FORCE_INLINE float taos_align_get_float(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
+#else
+  assert(sizeof(float) == sizeof(uint32_t));
+#endif
+  float fv = 0;
+  memcpy(&fv, pBuf, sizeof(fv));  // in ARM, return *((const float*)(pBuf)) may cause problem
+  return fv;
+}
+
+static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
+#else
+  assert(sizeof(double) == sizeof(uint64_t));
+#endif
+  double dv = 0;
+  memcpy(&dv, pBuf, sizeof(dv));  // in ARM, return *((const double*)(pBuf)) may cause problem
+  return dv;
+}
+
+// #ifdef _TD_ARM_32
+//   float  taos_align_get_float(const char* pBuf);
+//   double taos_align_get_double(const char* pBuf);
+
+//   #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
+//   #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
+//   #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
+//   #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
+//   #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
+//   #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
+// #else
+  #define GET_FLOAT_VAL(x)       (*(float *)(x))
+  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
+  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
+  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
+  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
+// #endif
+
+typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
+#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
+
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
+
+typedef int32_t VarDataOffsetT;
+typedef int16_t VarDataLenT;  // maxVarDataLen: 32767
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TYPES_H_*/

source/common/src/tcompare.c

@@ -12,334 +12,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _BSD_SOURCE
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _DEFAULT_SOURCE
 
-#include "os.h"
 #include "tcompare.h"
-#include "ulog.h"
-#include "thash.h"
-#include "regex.h"
-#include "ttypes.h"
-
-int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
-}
-
-int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
-}
-
-int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
-}
-
-int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
-}
-
-int32_t compareInt8Val(const void *pLeft, const void *pRight) {
-  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
-  return compareInt8Val(pRight, pLeft);
-}
-
-int32_t compareInt16Val(const void *pLeft, const void *pRight) {
-  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt16Val(pRight, pLeft);
-}
-
-int32_t compareInt32Val(const void *pLeft, const void *pRight) {
-  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt32Val(pRight, pLeft);
-}
-
-int32_t compareInt64Val(const void *pLeft, const void *pRight) {
-  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt64Val(pRight, pLeft);
-}
-
-int32_t compareUint32Val(const void *pLeft, const void *pRight) {
-  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint32Val(pRight, pLeft);
-}
-
-int32_t compareUint64Val(const void *pLeft, const void *pRight) {
-  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint64Val(pRight, pLeft);
-}
-
-int32_t compareUint16Val(const void *pLeft, const void *pRight) {
-  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint16Val(pRight, pLeft);
-}
-
-int32_t compareUint8Val(const void* pLeft, const void* pRight) {
-  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint8Val(pRight, pLeft);
-}
-
-int32_t compareFloatVal(const void *pLeft, const void *pRight) {
-  float p1 = GET_FLOAT_VAL(pLeft);
-  float p2 = GET_FLOAT_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
-  return compareFloatVal(pRight, pLeft);
-}
-
-int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
-  double p1 = GET_DOUBLE_VAL(pLeft);
-  double p2 = GET_DOUBLE_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
-  return compareDoubleVal(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1:-1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedStr(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1 : -1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedWStr(pRight, pLeft);
-}
-
-/*
- * Compare two strings
- *    TSDB_MATCH:            Match
- *    TSDB_NOMATCH:          No match
- *    TSDB_NOWILDCARDMATCH:  No match in spite of having * or % wildcards.
- * Like matching rules:
- *      '%': Matches zero or more characters
- *      '_': Matches one character
- *
- */
-int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
-  char c, c1;
-
-  int32_t i = 0;
-  int32_t j = 0;
-  int32_t o = 0;
-  int32_t m = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == pInfo->matchAll) { /* Match "*" */
-
-      while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
-        if (c == pInfo->matchOne) {
-          if (j > size || str[j++] == 0) {
-            // empty string, return not match
-            return TSDB_PATTERN_NOWILDCARDMATCH;
-          } else {
-            ++o;
-          }
-        }
-      }
-
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
-      }
-
-      char next[3] = {toupper(c), tolower(c), 0};
-      m = o;
-      while (1) {
-        size_t n = strcspn(str + m, next);
-        str += m + n;
-
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-        m = 0;
-      }
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-    ++o; 
-    
-    if (j <= size) {
-      if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
-      if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
-
-int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
-  wchar_t c, c1;
-  wchar_t matchOne = L'_';  // "_"
-  wchar_t matchAll = L'%';  // "%"
-
-  int32_t i = 0;
-  int32_t j = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == matchAll) { /* Match "%" */
-
-      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
-        if (c == matchOne && (j >= size || str[j++] == 0)) {
-          return TSDB_PATTERN_NOWILDCARDMATCH;
-        }
-      }
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH;
-      }
-
-      wchar_t accept[3] = {towupper(c), towlower(c), 0};
-      while (1) {
-        size_t n = wcscspn(str, accept);
-
-        str += n;
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-      }
-
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-
-    if (j <= size) {
-      if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-  
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
 
 int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
@@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
-int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight);
-}
-
-int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
-}
-
-int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
-  size_t sz = varDataLen(pRight);
-  char *pattern = malloc(sz + 1);
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  pattern[sz] = 0;
-
-  sz = varDataLen(pLeft);
-  char *str = malloc(sz + 1);
-  memcpy(str, varDataVal(pLeft), sz);
-  str[sz] = 0;
-
-  int errCode = 0;
-  regex_t regex;
-  char    msgbuf[256] = {0};
-
-  int cflags = REG_EXTENDED;
-  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
-    regfree(&regex);
-    free(str);
-    free(pattern);
-    return 1;
-  }
-
-  errCode = regexec(&regex, str, 0, NULL, 0);
-  if (errCode != 0 && errCode != REG_NOMATCH) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
-  }
-  int32_t result = (errCode == 0) ? 0 : 1;
-  regfree(&regex);
-  free(str);
-  free(pattern);
-  return result;
-}
-
-int32_t taosArrayCompareString(const void* a, const void* b) {
-  const char* x = *(const char**)a;
-  const char* y = *(const char**)b;
-
-  return compareLenPrefixedStr(x, y);
-}
-
-int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
-}
-
 int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
@@ -600,4 +218,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       }
     }
   }
-}
+}

source/common/src/tglobal.c

@@ -60,6 +60,7 @@ float   tsRatioOfQueryCores = 1.0f;
 int8_t  tsDaylight       = 0;
 int8_t  tsEnableCoreFile = 0;
 int32_t tsMaxBinaryDisplayWidth = 30;
+int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,

source/common/src/trow.c

@@ -11,4 +11,79 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
+ */
+
+#include "trow.h"
+
+/* ------------ Structures ---------- */
+struct SRowBatch {
+  int32_t compress : 1;  // if batch row is compressed
+  int32_t nrows : 31;    // number of rows
+  int32_t tlen;          // total length (including `nrows` and `tlen`)
+  char    rows[];
+};
+
+struct SRowBuilder {
+  // TODO
+};
+
+struct SRowBatchIter {
+  int32_t    counter;  // row counter
+  SRowBatch *rb;       // row batch to iter
+  SRow       nrow;     // next row
+};
+
+struct SRowBatchBuilder {
+  // TODO
+};
+
+/* ------------ Methods ---------- */
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate() {
+  SRowBuilder *pRowBuilder = NULL;
+  // TODO
+
+  return pRowBuilder;
+}
+
+void rowBuilderDestroy(SRowBuilder *pRowBuilder) {
+  if (pRowBuilder) {
+    free(pRowBuilder);
+  }
+}
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) {
+  SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter));
+  if (pRowBatchIter == NULL) {
+    return NULL;
+  }
+
+  pRowBatchIter->counter = 0;
+  pRowBatchIter->rb = pRowBatch;
+  pRowBatchIter->nrow = pRowBatch->rows;
+
+  return pRowBatchIter;
+};
+
+void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) {
+  if (pRowBatchIter) {
+    free(pRowBatchIter);
+  }
+}
+
+const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) {
+  SRow r = NULL;
+  if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) {
+    r = pRowBatchIter->nrow;
+    pRowBatchIter->counter += 1;
+    pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r));
+  }
+
+  return r;
+}
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);

source/server/CMakeLists.txt

@@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC})
 target_link_libraries(
     taosd
     PUBLIC dnode
+    PUBLIC util
 )

source/server/dnode/inc/dnodeCfg.h

@@ -21,21 +21,14 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnCfg {
-  int32_t         dnodeId;
-  int32_t         dropped;
-  char            clusterId[TSDB_CLUSTER_ID_LEN];
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnCfg;
 
-int32_t dnodeInitCfg(SDnCfg **cfg);
-void    dnodeCleanupCfg(SDnCfg **cfg);
-void    dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data);
-int32_t dnodeGetDnodeId(SDnCfg *cfg);
-void    dnodeGetClusterId(SDnCfg *cfg, char *clusterId);
-void    dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId);
-void    dnodeSetDropped(SDnCfg *cfg);
+int32_t dnodeInitCfg();
+void    dnodeCleanupCfg();
+void    dnodeUpdateCfg(SDnodeCfg *data);
+int32_t dnodeGetDnodeId();
+void    dnodeGetClusterId(char *clusterId);
+void    dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+void    dnodeSetDropped();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeCheck.h

@@ -21,11 +21,9 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnCheck {
-} SDnCheck;
 
-int32_t dnodeInitCheck(SDnCheck **check);
-void    dnodeCleanupCheck(SDnCheck **check);
+int32_t dnodeInitCheck();
+void    dnodeCleanupCheck();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeEps.h

@@ -19,22 +19,12 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "thash.h"
 #include "dnodeInt.h"
 
-typedef struct SDnEps {
-  int32_t         dnodeId;
-  int32_t         dnodeNum;
-  SDnodeEp *      dnodeList;
-  SHashObj *      dnodeHash;
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnEps;
-
-int32_t dnodeInitEps(SDnEps **eps);
-void    dnodeCleanupEps(SDnEps **eps);
-void    dnodeUpdateEps(SDnEps *eps, SDnodeEps *data);
-bool    dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr);
+int32_t dnodeInitEps();
+void    dnodeCleanupEps();
+void    dnodeUpdateEps(SDnodeEps *data);
+bool    dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr);
 void    dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeInt.h

@@ -19,36 +19,12 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "taoserror.h"
 #include "taosmsg.h"
-#include "tglobal.h"
 #include "tlog.h"
 #include "trpc.h"
-#include "tstep.h"
 #include "dnode.h"
 
-struct SDnCfg;
-struct SDnCheck;
-struct SDnEps;
-struct SDnMnEps;
-struct SDnStatus;
-struct SDnTelem;
-struct SDnTrans;
-struct SDnMain;
-
-typedef struct SDnode {
-  struct SSteps*    steps;
-  struct SDnCfg*    cfg;
-  struct SDnCheck*  check;
-  struct SDnEps*    eps;
-  struct SDnMnEps*  meps;
-  struct SDnStatus* status;
-  struct SDnTelem*  telem;
-  struct SDnTrans*  trans;
-  struct SDnMain*   main;
-} SDnode;
-
-SDnode* dnodeInst();
+extern int32_t dDebugFlag;
 
 #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
 #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}

source/server/dnode/inc/dnodeMain.h

@@ -27,14 +27,8 @@ typedef enum {
   TD_RUN_STAT_STOPPED
 } RunStat;
 
-typedef struct SDnMain {
-  RunStat      runStatus;
-  void *       dnodeTimer;
-  SStartupStep startup;
-} SDnMain;
-
-int32_t dnodeInitMain(SDnMain **main);
-void    dnodeCleanupMain(SDnMain **main);
+int32_t dnodeInitMain();
+void    dnodeCleanupMain();
 int32_t dnodeInitStorage();
 void    dnodeCleanupStorage();
 void    dnodeReportStartup(char *name, char *desc);
@@ -42,6 +36,9 @@ void    dnodeReportStartupFinished(char *name, char *desc);
 void    dnodeProcessStartupReq(SRpcMsg *pMsg);
 void    dnodeProcessCreateMnodeReq(SRpcMsg *pMsg);
 void    dnodeProcessConfigDnodeReq(SRpcMsg *pMsg);
+RunStat dnodeGetRunStat();
+void    dnodeSetRunStat();
+void*   dnodeGetTimer();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeMnodeEps.h

@@ -21,19 +21,12 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnMnEps {
-  SRpcEpSet       mnodeEpSet;
-  SMInfos         mnodeInfos;
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnMnEps;
-
-int32_t dnodeInitMnodeEps(SDnMnEps **meps);
-void    dnodeCleanupMnodeEps(SDnMnEps **meps);
-void    dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *pMinfos);
-void    dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *pEpSet);
-void    dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet);
+int32_t dnodeInitMnodeEps();
+void    dnodeCleanupMnodeEps();
+void    dnodeUpdateMnodeFromStatus(SMInfos *pMinfos);
+void    dnodeUpdateMnodeFromPeer(SRpcEpSet *pEpSet);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
 void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeStatus.h

@@ -21,14 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnStatus {
-  void *   dnodeTimer;
-  void *   statusTimer;
-  uint32_t rebootTime;
-} SDnStatus;
-
-int32_t dnodeInitStatus(SDnStatus **status);
-void    dnodeCleanupStatus(SDnStatus **status);
+int32_t dnodeInitStatus();
+void    dnodeCleanupStatus();
 void    dnodeProcessStatusRsp(SRpcMsg *pMsg);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeTelem.h

@@ -21,21 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-/*
- * sem_timedwait is NOT implemented on MacOSX
- * thus we use pthread_mutex_t/pthread_cond_t to simulate
- */
-typedef struct SDnTelem {
-  bool             enable;
-  pthread_mutex_t  lock;
-  pthread_cond_t   cond;
-  volatile int32_t exit;
-  pthread_t        thread;
-  char             email[TSDB_FQDN_LEN];
-} SDnTelem;
-
-int32_t dnodeInitTelem(SDnTelem **telem);
-void    dnodeCleanupTelem(SDnTelem **telem);
+int32_t dnodeInitTelem();
+void    dnodeCleanupTelem();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeTrans.h

@@ -21,20 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef void (*RpcMsgFp)( SRpcMsg *pMsg);
-
-typedef struct SDnTrans {
-  void *   serverRpc;
-  void *   clientRpc;
-  void *   shellRpc;
-  int32_t  queryReqNum;
-  int32_t  submitReqNum;
-  RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX];
-  RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX];
-} SDnTrans;
-
-int32_t dnodeInitTrans(SDnTrans **rans);
-void    dnodeCleanupTrans(SDnTrans **trans);
+int32_t dnodeInitTrans();
+void    dnodeCleanupTrans();
 void    dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
 void    dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
 void    dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);

source/server/dnode/src/dnodeCfg.c

@@ -16,56 +16,65 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
+#include "tglobal.h"
 #include "dnodeCfg.h"
 
-static int32_t dnodeReadCfg(SDnCfg *cfg) {
+static struct DnCfg {
+  int32_t         dnodeId;
+  int32_t         dropped;
+  char            clusterId[TSDB_CLUSTER_ID_LEN];
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDcfg;
+
+static int32_t dnodeReadCfg() {
   int32_t len = 0;
   int32_t maxLen = 200;
   char *  content = calloc(1, maxLen + 1);
   cJSON * root = NULL;
   FILE *  fp = NULL;
 
-  fp = fopen(cfg->file, "r");
+  fp = fopen(tsDcfg.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", cfg->file);
+    dDebug("file %s not exist", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", cfg->file);
+    dError("failed to read %s since content is null", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", cfg->file);
+    dError("failed to read %s since invalid json format", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
   if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read %s since dnodeId not found", cfg->file);
+    dError("failed to read %s since dnodeId not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  cfg->dnodeId = (int32_t)dnodeId->valueint;
+  tsDcfg.dnodeId = (int32_t)dnodeId->valueint;
 
   cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
   if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", cfg->file);
+    dError("failed to read %s since dropped not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  cfg->dropped = (int32_t)dropped->valueint;
+  tsDcfg.dropped = (int32_t)dropped->valueint;
 
   cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
   if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read %s since clusterId not found", cfg->file);
+    dError("failed to read %s since clusterId not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  tstrncpy(cfg->clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+  tstrncpy(tsDcfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
 
-  dInfo("successed to read %s", cfg->file);
+  dInfo("successed to read %s", tsDcfg.file);
 
 PARSE_CFG_OVER:
   if (content != NULL) free(content);
@@ -76,10 +85,10 @@ PARSE_CFG_OVER:
   return 0;
 }
 
-static int32_t dnodeWriteCfg(SDnCfg *cfg) {
-  FILE *fp = fopen(cfg->file, "w");
+static int32_t dnodeWriteCfg() {
+  FILE *fp = fopen(tsDcfg.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", cfg->file, strerror(errno));
+    dError("failed to write %s since %s", tsDcfg.file, strerror(errno));
     return -1;
   }
 
@@ -88,9 +97,9 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", cfg->dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", cfg->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", cfg->clusterId);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDcfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", tsDcfg.dropped);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsDcfg.clusterId);
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
@@ -99,27 +108,23 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", cfg->file);
+  dInfo("successed to write %s", tsDcfg.file);
   return 0;
 }
 
-int32_t dnodeInitCfg(SDnCfg **out) {
-  SDnCfg* cfg = calloc(1, sizeof(SDnCfg));
-  if (cfg == NULL) return -1;
-
-  cfg->dnodeId = 0;
-  cfg->dropped = 0;
-  cfg->clusterId[0] = 0;
-  snprintf(cfg->file, sizeof(cfg->file), "%s/dnodeCfg.json", tsDnodeDir);
-  pthread_mutex_init(&cfg->mutex, NULL);
-  *out = cfg;
-
-  int32_t ret = dnodeReadCfg(cfg);
+int32_t dnodeInitCfg() {
+  tsDcfg.dnodeId = 0;
+  tsDcfg.dropped = 0;
+  tsDcfg.clusterId[0] = 0;
+  snprintf(tsDcfg.file, sizeof(tsDcfg.file), "%s/dnodeCfg.json", tsDnodeDir);
+  pthread_mutex_init(&tsDcfg.mutex, NULL);
+  
+  int32_t ret = dnodeReadCfg();
   if (ret == 0) {
     dInfo("dnode cfg is initialized");
   }
 
-  if (cfg->dropped) {
+  if (tsDcfg.dropped) {
     dInfo("dnode is dropped and start to exit");
     return -1;
   }
@@ -127,51 +132,47 @@ int32_t dnodeInitCfg(SDnCfg **out) {
   return ret;
 }
 
-void dnodeCleanupCfg(SDnCfg **out) {
-  SDnCfg* cfg = *out;
-  *out = NULL;
-
-  pthread_mutex_destroy(&cfg->mutex);
-  free(cfg);
+void dnodeCleanupCfg() {
+  pthread_mutex_destroy(&tsDcfg.mutex);
 }
 
-void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data) {
-  if (cfg == NULL || cfg->dnodeId == 0) return;
+void dnodeUpdateCfg(SDnodeCfg *data) {
+  if (tsDcfg.dnodeId != 0) return;
 
-  pthread_mutex_lock(&cfg->mutex);
+  pthread_mutex_lock(&tsDcfg.mutex);
 
-  cfg->dnodeId = data->dnodeId;
-  tstrncpy(cfg->clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN);
-  dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+  tsDcfg.dnodeId = data->dnodeId;
+  tstrncpy(tsDcfg.clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN);
+  dInfo("dnodeId is set to %d, clusterId is set to %s", data->dnodeId, data->clusterId);
 
-  dnodeWriteCfg(cfg);
-  pthread_mutex_unlock(&cfg->mutex);
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-void dnodeSetDropped(SDnCfg *cfg) {
-  pthread_mutex_lock(&cfg->mutex);
-  cfg->dropped = 1;
-  dnodeWriteCfg(cfg);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeSetDropped() {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  tsDcfg.dropped = 1;
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-int32_t dnodeGetDnodeId(SDnCfg *cfg) {
+int32_t dnodeGetDnodeId() {
   int32_t dnodeId = 0;
-  pthread_mutex_lock(&cfg->mutex);
-  dnodeId = cfg->dnodeId;
-  pthread_mutex_unlock(&cfg->mutex);
+  pthread_mutex_lock(&tsDcfg.mutex);
+  dnodeId = tsDcfg.dnodeId;
+  pthread_mutex_unlock(&tsDcfg.mutex);
   return dnodeId;
 }
 
-void dnodeGetClusterId(SDnCfg *cfg, char *clusterId) {
-  pthread_mutex_lock(&cfg->mutex);
-  tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeGetClusterId(char *clusterId) {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId) {
-  pthread_mutex_lock(&cfg->mutex);
-  *dnodeId = cfg->dnodeId;
-  tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  *dnodeId = tsDcfg.dnodeId;
+  tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }

source/server/dnode/src/dnodeCheck.c

@@ -118,7 +118,7 @@ static int32_t dnodeCheckMem() {
 }
 
 static int32_t dnodeCheckDisk() {
-#if 0  
+#if 0
   taosGetDisk();
 
   if (tsAvailDataDirGB < tsMinimalDataDirGB) {
@@ -145,12 +145,7 @@ static int32_t dnodeCheckAccess() { return 0; }
 static int32_t dnodeCheckVersion() { return 0; }
 static int32_t dnodeCheckDatafile() { return 0; }
 
-int32_t dnodeInitCheck(SDnCheck **out) {
-  SDnCheck *check = calloc(1, sizeof(SDnCheck));
-  if (check == NULL) return -1;
-
-  *out = check;
-
+int32_t dnodeInitCheck() {
   if (dnodeCheckNetwork() != 0) {
     dError("failed to check network");
     return -1;
@@ -195,9 +190,4 @@ int32_t dnodeInitCheck(SDnCheck **out) {
   return 0;
 }
 
-void dnodeCleanupCheck(SDnCheck **out) {
-  SDnCheck *check = *out;
-  *out = NULL;
-
-  free(check);
-}
+void dnodeCleanupCheck() {}

source/server/dnode/src/dnodeEps.c

@@ -16,86 +16,96 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
+#include "thash.h"
 #include "tglobal.h"
 #include "dnodeEps.h"
 #include "dnodeCfg.h"
 
-static void dnodePrintEps(SDnEps *eps) {
-  dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
-  for (int32_t i = 0; i < eps->dnodeNum; i++) {
-    SDnodeEp *ep = &eps->dnodeList[i];
+static struct {
+  int32_t         dnodeId;
+  int32_t         dnodeNum;
+  SDnodeEp *      dnodeList;
+  SHashObj *      dnodeHash;
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDeps;
+
+static void dnodePrintEps() {
+  dDebug("print dnodeEp, dnodeNum:%d", tsDeps.dnodeNum);
+  for (int32_t i = 0; i < tsDeps.dnodeNum; i++) {
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
     dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
   }
 }
 
-static void dnodeResetEps(SDnEps *eps, SDnodeEps *data) {
+static void dnodeResetEps(SDnodeEps *data) {
   assert(data != NULL);
 
-  if (data->dnodeNum > eps->dnodeNum) {
+  if (data->dnodeNum > tsDeps.dnodeNum) {
     SDnodeEp *tmp = calloc(data->dnodeNum, sizeof(SDnodeEp));
     if (tmp == NULL) return;
 
-    tfree(eps->dnodeList);
-    eps->dnodeList = tmp;
-    eps->dnodeNum = data->dnodeNum;
-    memcpy(eps->dnodeList, data->dnodeEps, eps->dnodeNum * sizeof(SDnodeEp));
-    dnodePrintEps(eps);
+    tfree(tsDeps.dnodeList);
+    tsDeps.dnodeList = tmp;
+    tsDeps.dnodeNum = data->dnodeNum;
+    memcpy(tsDeps.dnodeList, data->dnodeEps, tsDeps.dnodeNum * sizeof(SDnodeEp));
+    dnodePrintEps();
 
-    for (int32_t i = 0; i < eps->dnodeNum; ++i) {
-      SDnodeEp *ep = &eps->dnodeList[i];
-      taosHashPut(eps->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+    for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) {
+      SDnodeEp *ep = &tsDeps.dnodeList[i];
+      taosHashPut(tsDeps.dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
     }
   }
 }
 
-static int32_t dnodeReadEps(SDnEps *eps) {
+static int32_t dnodeReadEps() {
   int32_t len = 0;
   int32_t maxLen = 30000;
   char *  content = calloc(1, maxLen + 1);
   cJSON * root = NULL;
   FILE *  fp = NULL;
 
-  fp = fopen(eps->file, "r");
+  fp = fopen(tsDeps.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", eps->file);
+    dDebug("file %s not exist", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", eps->file);
+    dError("failed to read %s since content is null", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", eps->file);
+    dError("failed to read %s since invalid json format", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
   if (!dnodeNum || dnodeNum->type != cJSON_Number) {
-    dError("failed to read %s since dnodeNum not found", eps->file);
+    dError("failed to read %s since dnodeNum not found", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
   if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
-    dError("failed to read %s since dnodeInfos not found", eps->file);
+    dError("failed to read %s since dnodeInfos not found", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
   if (dnodeInfosSize != dnodeNum->valueint) {
-    dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", eps->file, dnodeInfosSize,
+    dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", tsDeps.file, dnodeInfosSize,
            (int32_t)dnodeNum->valueint);
     goto PRASE_EPS_OVER;
   }
 
-  eps->dnodeNum = dnodeInfosSize;
-  eps->dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp));
-  if (eps->dnodeList == NULL) {
+  tsDeps.dnodeNum = dnodeInfosSize;
+  tsDeps.dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp));
+  if (tsDeps.dnodeList == NULL) {
     dError("failed to calloc dnodeEpList since %s", strerror(errno));
     goto PRASE_EPS_OVER;
   }
@@ -104,40 +114,40 @@ static int32_t dnodeReadEps(SDnEps *eps) {
     cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
     if (dnodeInfo == NULL) break;
 
-    SDnodeEp *ep = &eps->dnodeList[i];
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
 
     cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
     if (!dnodeId || dnodeId->type != cJSON_Number) {
-      dError("failed to read %s, dnodeId not found", eps->file);
+      dError("failed to read %s, dnodeId not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     ep->dnodeId = (int32_t)dnodeId->valueint;
 
     cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
     if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
-      dError("failed to read %s, dnodeFqdn not found", eps->file);
+      dError("failed to read %s, dnodeFqdn not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     tstrncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
 
     cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
     if (!dnodePort || dnodePort->type != cJSON_Number) {
-      dError("failed to read %s, dnodePort not found", eps->file);
+      dError("failed to read %s, dnodePort not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     ep->dnodePort = (uint16_t)dnodePort->valueint;
   }
 
-  dInfo("succcessed to read file %s", eps->file);
-  dnodePrintEps(eps);
+  dInfo("succcessed to read file %s", tsDeps.file);
+  dnodePrintEps();
 
 PRASE_EPS_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
   if (fp != NULL) fclose(fp);
 
-  if (dnodeIsDnodeEpChanged(eps, eps->dnodeId, tsLocalEp)) {
-    dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", eps->dnodeId, tsLocalEp);
+  if (dnodeIsDnodeEpChanged(tsDeps.dnodeId, tsLocalEp)) {
+    dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", tsDeps.dnodeId, tsLocalEp);
     return -1;
   }
 
@@ -145,10 +155,10 @@ PRASE_EPS_OVER:
   return 0;
 }
 
-static int32_t dnodeWriteEps(SDnEps *eps) {
-  FILE *fp = fopen(eps->file, "w");
+static int32_t dnodeWriteEps() {
+  FILE *fp = fopen(tsDeps.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", eps->file, strerror(errno));
+    dError("failed to write %s since %s", tsDeps.file, strerror(errno));
     return -1;
   }
 
@@ -157,14 +167,14 @@ static int32_t dnodeWriteEps(SDnEps *eps) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", eps->dnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", tsDeps.dnodeNum);
   len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
-  for (int32_t i = 0; i < eps->dnodeNum; ++i) {
-    SDnodeEp *ep = &eps->dnodeList[i];
+  for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) {
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
     len += snprintf(content + len, maxLen - len, "    \"dnodeId\": %d,\n", ep->dnodeId);
     len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
     len += snprintf(content + len, maxLen - len, "    \"dnodePort\": %u\n", ep->dnodePort);
-    if (i < eps->dnodeNum - 1) {
+    if (i < tsDeps.dnodeNum - 1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");
@@ -178,24 +188,20 @@ static int32_t dnodeWriteEps(SDnEps *eps) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", eps->file);
+  dInfo("successed to write %s", tsDeps.file);
   return 0;
 }
 
-int32_t dnodeInitEps(SDnEps **out) {
-  SDnEps *eps = calloc(1, sizeof(SDnEps));
-  if (eps == NULL) return -1;
+int32_t dnodeInitEps() {
+  tsDeps.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (tsDeps.dnodeHash == NULL) return -1;
 
-  eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (eps->dnodeHash == NULL) return -1;
+  tsDeps.dnodeId = dnodeGetDnodeId();
+  tsDeps.dnodeNum = 0;
+  snprintf(tsDeps.file, sizeof(tsDeps.file), "%s/dnodeEps.json", tsDnodeDir);
+  pthread_mutex_init(&tsDeps.mutex, NULL);
 
-  eps->dnodeId = dnodeInst()->cfg->dnodeId;
-  eps->dnodeNum = 0;
-  snprintf(eps->file, sizeof(eps->file), "%s/dnodeEps.json", tsDnodeDir);
-  pthread_mutex_init(&eps->mutex, NULL);
-  *out = eps;
-
-  int32_t ret = dnodeReadEps(eps);
+  int32_t ret = dnodeReadEps();
   if (ret == 0) {
     dInfo("dnode eps is initialized");
   }
@@ -203,29 +209,25 @@ int32_t dnodeInitEps(SDnEps **out) {
   return ret;
 }
 
-void dnodeCleanupEps(SDnEps **out) {
-  SDnEps *eps = *out;
-  *out = NULL;
+void dnodeCleanupEps() {
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  pthread_mutex_lock(&eps->mutex);
-
-  if (eps->dnodeList != NULL) {
-    free(eps->dnodeList);
-    eps->dnodeList = NULL;
+  if (tsDeps.dnodeList != NULL) {
+    free(tsDeps.dnodeList);
+    tsDeps.dnodeList = NULL;
   }
 
-  if (eps->dnodeHash) {
-    taosHashCleanup(eps->dnodeHash);
-    eps->dnodeHash = NULL;
+  if (tsDeps.dnodeHash) {
+    taosHashCleanup(tsDeps.dnodeHash);
+    tsDeps.dnodeHash = NULL;
   }
 
-  eps->dnodeNum = 0;
-  pthread_mutex_unlock(&eps->mutex);
-  pthread_mutex_destroy(&eps->mutex);
-  free(eps);
+  tsDeps.dnodeNum = 0;
+  pthread_mutex_unlock(&tsDeps.mutex);
+  pthread_mutex_destroy(&tsDeps.mutex);
 }
 
-void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) {
+void dnodeUpdateEps(SDnodeEps *data) {
   if (data == NULL || data->dnodeNum <= 0) return;
 
   data->dnodeNum = htonl(data->dnodeNum);
@@ -234,28 +236,28 @@ void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) {
     data->dnodeEps[i].dnodePort = htons(data->dnodeEps[i].dnodePort);
   }
 
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  if (data->dnodeNum != eps->dnodeNum) {
-    dnodeResetEps(eps, data);
-    dnodeWriteEps(eps);
+  if (data->dnodeNum != tsDeps.dnodeNum) {
+    dnodeResetEps(data);
+    dnodeWriteEps();
   } else {
     int32_t size = data->dnodeNum * sizeof(SDnodeEp);
-    if (memcmp(eps->dnodeList, data->dnodeEps, size) != 0) {
-      dnodeResetEps(eps, data);
-      dnodeWriteEps(eps);
+    if (memcmp(tsDeps.dnodeList, data->dnodeEps, size) != 0) {
+      dnodeResetEps(data);
+      dnodeWriteEps();
     }
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 }
 
-bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) {
+bool dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr) {
   bool changed = false;
 
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t));
+  SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t));
   if (ep != NULL) {
     char epSaved[TSDB_EP_LEN + 1];
     snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
@@ -263,21 +265,20 @@ bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) {
     tstrncpy(epstr, epSaved, TSDB_EP_LEN);
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 
   return changed;
 }
 
 void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
-  SDnEps *eps = dnodeInst()->eps;
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t));
+  SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t));
   if (ep != NULL) {
     if (port) *port = ep->dnodePort;
     if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
     if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 }

source/server/dnode/src/dnodeInt.c

@@ -33,10 +33,7 @@
 #include "mnode.h"
 #include "vnode.h"
 
-SDnode *dnodeInst() {
-  static SDnode inst = {0};
-  return &inst;
-}
+static struct SSteps *tsSteps;
 
 static int32_t dnodeInitVnodeModule(void **unused) {
   SVnodePara para;
@@ -48,58 +45,50 @@ static int32_t dnodeInitVnodeModule(void **unused) {
 }
 
 static int32_t dnodeInitMnodeModule(void **unused) {
-  SDnode *dnode = dnodeInst();
-
   SMnodePara para;
   para.fp.GetDnodeEp = dnodeGetDnodeEp;
   para.fp.SendMsgToDnode = dnodeSendMsgToDnode;
   para.fp.SendMsgToMnode = dnodeSendMsgToMnode;
   para.fp.SendRedirectMsg = dnodeSendRedirectMsg;
-  para.dnodeId = dnode->cfg->dnodeId;
-  strncpy(para.clusterId, dnode->cfg->clusterId, sizeof(para.clusterId));
+  dnodeGetCfg(&para.dnodeId, para.clusterId);
 
   return mnodeInit(para);
 }
 
 int32_t dnodeInit() {
-  struct SSteps *steps = taosStepInit(24, dnodeReportStartup);
-  if (steps == NULL) return -1;
+  tsSteps = taosStepInit(24, dnodeReportStartup);
+  if (tsSteps == NULL) return -1;
 
-  SDnode *dnode = dnodeInst();
+  taosStepAdd(tsSteps, "dnode-main", dnodeInitMain, dnodeCleanupMain);
+  taosStepAdd(tsSteps, "dnode-storage", dnodeInitStorage, dnodeCleanupStorage);
+  //taosStepAdd(tsSteps, "dnode-tfs", tfInit, tfCleanup);
+  taosStepAdd(tsSteps, "dnode-rpc", rpcInit, rpcCleanup);
+  taosStepAdd(tsSteps, "dnode-check", dnodeInitCheck, dnodeCleanupCheck);
+  taosStepAdd(tsSteps, "dnode-cfg", dnodeInitCfg, dnodeCleanupCfg);
+  taosStepAdd(tsSteps, "dnode-deps", dnodeInitEps, dnodeCleanupEps);
+  taosStepAdd(tsSteps, "dnode-meps", dnodeInitMnodeEps, dnodeCleanupMnodeEps);
+  //taosStepAdd(tsSteps, "dnode-wal", walInit, walCleanUp);
+  //taosStepAdd(tsSteps, "dnode-sync", syncInit, syncCleanUp);
+  taosStepAdd(tsSteps, "dnode-vnode", dnodeInitVnodeModule, vnodeCleanup);
+  taosStepAdd(tsSteps, "dnode-mnode", dnodeInitMnodeModule, mnodeCleanup);
+  taosStepAdd(tsSteps, "dnode-trans", dnodeInitTrans, dnodeCleanupTrans);
+  taosStepAdd(tsSteps, "dnode-status", dnodeInitStatus, dnodeCleanupStatus);
+  taosStepAdd(tsSteps, "dnode-telem", dnodeInitTelem, dnodeCleanupTelem);
+  //taosStepAdd(tsSteps, "dnode-script",scriptEnvPoolInit, scriptEnvPoolCleanup);
 
-  taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain);
-  taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage);
-  //taosStepAdd(steps, "dnode-tfs", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup);
-  taosStepAdd(steps, "dnode-rpc", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup);
-  taosStepAdd(steps, "dnode-check", (void **)&dnode->check, (InitFp)dnodeInitCheck, (CleanupFp)dnodeCleanupCheck);
-  taosStepAdd(steps, "dnode-cfg", (void **)&dnode->cfg, (InitFp)dnodeInitCfg, (CleanupFp)dnodeCleanupCfg);
-  taosStepAdd(steps, "dnode-deps", (void **)&dnode->eps, (InitFp)dnodeInitEps, (CleanupFp)dnodeCleanupEps);
-  taosStepAdd(steps, "dnode-meps", (void **)&dnode->meps, (InitFp)dnodeInitMnodeEps, (CleanupFp)dnodeCleanupMnodeEps);
-  //taosStepAdd(steps, "dnode-wal", NULL, (InitFp)walInit, (CleanupFp)walCleanUp);
-  //taosStepAdd(steps, "dnode-sync", NULL, (InitFp)syncInit, (CleanupFp)syncCleanUp);
-  taosStepAdd(steps, "dnode-vnode", NULL, (InitFp)dnodeInitVnodeModule, (CleanupFp)vnodeCleanup);
-  taosStepAdd(steps, "dnode-mnode", NULL, (InitFp)dnodeInitMnodeModule, (CleanupFp)mnodeCleanup);
-  taosStepAdd(steps, "dnode-trans", (void **)&dnode->trans, (InitFp)dnodeInitTrans, (CleanupFp)dnodeCleanupTrans);
-  taosStepAdd(steps, "dnode-status", (void **)&dnode->status, (InitFp)dnodeInitStatus, (CleanupFp)dnodeCleanupStatus);
-  taosStepAdd(steps, "dnode-telem", (void **)&dnode->telem, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem);
-  //taosStepAdd(steps, "dnode-script", NULL, (InitFp)scriptEnvPoolInit, (CleanupFp)scriptEnvPoolCleanup);
+  taosStepExec(tsSteps);
 
-  dnode->steps = steps;
-  taosStepExec(dnode->steps);
-
-  if (dnode->main) {
-    dnode->main->runStatus = TD_RUN_STAT_RUNNING;
-    dnodeReportStartupFinished("TDengine", "initialized successfully");
-    dInfo("TDengine is initialized successfully");
-  }
+  dnodeSetRunStat(TD_RUN_STAT_RUNNING);
+  dnodeReportStartupFinished("TDengine", "initialized successfully");
+  dInfo("TDengine is initialized successfully");
 
   return 0;
 }
 
 void dnodeCleanup() {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) {
-    dnode->main->runStatus = TD_RUN_STAT_STOPPED;
-    taosStepCleanup(dnode->steps);
+  if (dnodeGetRunStat() != TD_RUN_STAT_STOPPED) {
+    dnodeSetRunStat(TD_RUN_STAT_STOPPED);
+    taosStepCleanup(tsSteps);
+    tsSteps = NULL;
   }
 }

source/server/dnode/src/dnodeMain.c

@@ -17,15 +17,22 @@
 #include "os.h"
 #include "tcache.h"
 #include "tconfig.h"
+#include "tglobal.h"
 #if 0
 #include "tfs.h"
 #endif
+#include "tnote.h"
+#include "tcompression.h"
+#include "ttimer.h"
 #include "dnodeCfg.h"
 #include "dnodeMain.h"
 #include "mnode.h"
-#include "tcompression.h"
-#include "tnote.h"
-#include "ttimer.h"
+
+static struct {
+  RunStat      runStatus;
+  void *       dnodeTimer;
+  SStartupStep startup;
+} tsDmain;
 
 static void dnodeCheckDataDirOpenned(char *dir) {
 #if 0
@@ -47,27 +54,14 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 #endif
 }
 
-void dnodePrintDiskInfo() {
-  dInfo("==================================");
-  dInfo(" os totalDisk:           %f(GB)", tsTotalDataDirGB);
-  dInfo(" os usedDisk:            %f(GB)", tsUsedDataDirGB);
-  dInfo(" os availDisk:           %f(GB)", tsAvailDataDirGB);
-  dInfo("==================================");
-}
-
-int32_t dnodeInitMain(SDnMain **out) {
-  SDnMain* main = calloc(1, sizeof(SDnMain));
-  if (main == NULL) return -1;
-
-  main->runStatus = TD_RUN_STAT_STOPPED;
-  main->dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR");
-  if (main->dnodeTimer == NULL) {
+int32_t dnodeInitMain() {
+  tsDmain.runStatus = TD_RUN_STAT_STOPPED;
+  tsDmain.dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR");
+  if (tsDmain.dnodeTimer == NULL) {
     dError("failed to init dnode timer");
     return -1;
   }
 
-  *out = main;
-
   tscEmbedded = 1;
   taosIgnSIGPIPE();
   taosBlockSIGPIPE();
@@ -76,7 +70,6 @@ int32_t dnodeInitMain(SDnMain **out) {
   taosReadGlobalLogCfg();
   taosSetCoreDump(tsEnableCoreFile);
 
-
   if (!taosMkDir(tsLogDir)) {
    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
    return -1;
@@ -101,13 +94,10 @@ int32_t dnodeInitMain(SDnMain **out) {
   return taosCheckGlobalCfg();
 }
 
-void dnodeCleanupMain(SDnMain **out) {
-  SDnMain *main = *out;
-  *out = NULL;
-
-  if (main->dnodeTimer != NULL) {
-    taosTmrCleanUp(main->dnodeTimer);
-    main->dnodeTimer = NULL;
+void dnodeCleanupMain() {
+  if (tsDmain.dnodeTimer != NULL) {
+    taosTmrCleanUp(tsDmain.dnodeTimer);
+    tsDmain.dnodeTimer = NULL;
   }
 
 #if 0
@@ -115,8 +105,6 @@ void dnodeCleanupMain(SDnMain **out) {
 #endif
   taosCloseLog();
   taosStopCacheRefreshWorker();
-
-  free(main);
 }
 
 int32_t dnodeInitStorage() {
@@ -138,7 +126,7 @@ int32_t dnodeInitStorage() {
   }
 
   strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
-#endif    
+#endif
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
@@ -164,7 +152,6 @@ int32_t dnodeInitStorage() {
     return -1;
   }
 
-
   TDIR *tdir = tfsOpendir("vnode_bak/.staging");
   bool  stagingNotEmpty = tfsReaddir(tdir) != NULL;
   tfsClosedir(tdir);
@@ -190,7 +177,7 @@ int32_t dnodeInitStorage() {
 }
 
 void dnodeCleanupStorage() {
-#if 0  
+#if 0
   // storage destroy
   tfsDestroy();
 
@@ -202,18 +189,14 @@ void dnodeCleanupStorage() {
 }
 
 void dnodeReportStartup(char *name, char *desc) {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main != NULL) {
-    SStartupStep *startup = &dnode->main->startup;
-    tstrncpy(startup->name, name, strlen(startup->name));
-    tstrncpy(startup->desc, desc, strlen(startup->desc));
-    startup->finished = 0;
-  }
+  SStartupStep *startup = &tsDmain.startup;
+  tstrncpy(startup->name, name, strlen(startup->name));
+  tstrncpy(startup->desc, desc, strlen(startup->desc));
+  startup->finished = 0;
 }
 
 void dnodeReportStartupFinished(char *name, char *desc) {
-  SDnode *dnode = dnodeInst();
-  SStartupStep *startup = &dnode->main->startup;
+  SStartupStep *startup = &tsDmain.startup;
   tstrncpy(startup->name, name, strlen(startup->name));
   tstrncpy(startup->desc, desc, strlen(startup->desc));
   startup->finished = 1;
@@ -222,9 +205,8 @@ void dnodeReportStartupFinished(char *name, char *desc) {
 void dnodeProcessStartupReq(SRpcMsg *pMsg) {
   dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont);
 
-  SDnode *dnode = dnodeInst();
   SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep));
-  memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep));
+  memcpy(pStep, &tsDmain.startup, sizeof(SStartupStep));
 
   dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished);
 
@@ -234,12 +216,11 @@ void dnodeProcessStartupReq(SRpcMsg *pMsg) {
 }
 
 static int32_t dnodeStartMnode(SRpcMsg *pMsg) {
-  SDnode *dnode = dnodeInst();
   SCreateMnodeMsg *pCfg = pMsg->pCont;
   pCfg->dnodeId = htonl(pCfg->dnodeId);
-  if (pCfg->dnodeId != dnode->cfg->dnodeId) {
+  if (pCfg->dnodeId != dnodeGetDnodeId()) {
     dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId,
-           dnodeGetDnodeId(dnode->cfg));
+           dnodeGetDnodeId());
     return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
   }
 
@@ -277,4 +258,10 @@ void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) {
 
   rpcSendResponse(&rspMsg);
   rpcFreeCont(pMsg->pCont);
-}
+}
+
+RunStat dnodeGetRunStat() { return tsDmain.runStatus; }
+
+void dnodeSetRunStat(RunStat stat) { tsDmain.runStatus = stat; }
+
+void* dnodeGetTimer() { return tsDmain.dnodeTimer; }

source/server/dnode/src/dnodeMnodeEps.c

@@ -22,43 +22,51 @@
 #include "dnodeMnodeEps.h"
 #include "mnode.h"
 
-static void dnodePrintMnodeEps(SDnMnEps *meps) {
-  SRpcEpSet *epset = &meps->mnodeEpSet;
+static struct {
+  SRpcEpSet       mnodeEpSet;
+  SMInfos         mnodeInfos;
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDmeps;
+
+
+static void dnodePrintMnodeEps() {
+  SRpcEpSet *epset = &tsDmeps.mnodeEpSet;
   dInfo("print mnode eps, num:%d inuse:%d", epset->numOfEps, epset->inUse);
   for (int32_t i = 0; i < epset->numOfEps; i++) {
     dInfo("ep index:%d, %s:%u", i, epset->fqdn[i], epset->port[i]);
   }
 }
 
-static void dnodeResetMnodeEps(SDnMnEps *meps, SMInfos *mInfos) {
+static void dnodeResetMnodeEps(SMInfos *mInfos) {
   if (mInfos == NULL || mInfos->mnodeNum == 0) {
-    meps->mnodeEpSet.numOfEps = 1;
-    taosGetFqdnPortFromEp(tsFirst, meps->mnodeEpSet.fqdn[0], &meps->mnodeEpSet.port[0]);
+    tsDmeps.mnodeEpSet.numOfEps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsDmeps.mnodeEpSet.fqdn[0], &tsDmeps.mnodeEpSet.port[0]);
 
     if (strcmp(tsSecond, tsFirst) != 0) {
-      meps->mnodeEpSet.numOfEps = 2;
-      taosGetFqdnPortFromEp(tsSecond, meps->mnodeEpSet.fqdn[1], &meps->mnodeEpSet.port[1]);
+      tsDmeps.mnodeEpSet.numOfEps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsDmeps.mnodeEpSet.fqdn[1], &tsDmeps.mnodeEpSet.port[1]);
     }
-    dnodePrintMnodeEps(meps);
+    dnodePrintMnodeEps();
     return;
   }
 
-    int32_t size = sizeof(SMInfos);
-  memcpy(&meps->mnodeInfos, mInfos, size);
+  int32_t size = sizeof(SMInfos);
+  memcpy(&tsDmeps.mnodeInfos, mInfos, size);
 
-  meps->mnodeEpSet.inUse = meps->mnodeInfos.inUse;
-  meps->mnodeEpSet.numOfEps = meps->mnodeInfos.mnodeNum;
-  for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) {
-    taosGetFqdnPortFromEp(meps->mnodeInfos.mnodeInfos[i].mnodeEp, meps->mnodeEpSet.fqdn[i], &meps->mnodeEpSet.port[i]);
+  tsDmeps.mnodeEpSet.inUse = tsDmeps.mnodeInfos.inUse;
+  tsDmeps.mnodeEpSet.numOfEps = tsDmeps.mnodeInfos.mnodeNum;
+  for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) {
+    taosGetFqdnPortFromEp(tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp, tsDmeps.mnodeEpSet.fqdn[i], &tsDmeps.mnodeEpSet.port[i]);
   }
 
-  dnodePrintMnodeEps(meps);
+  dnodePrintMnodeEps();
 }
 
-static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
-  FILE *fp = fopen(meps->file, "w");
+static int32_t dnodeWriteMnodeEps() {
+  FILE *fp = fopen(tsDmeps.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", meps->file, strerror(errno));
+    dError("failed to write %s since %s", tsDmeps.file, strerror(errno));
     return -1;
   }
 
@@ -67,13 +75,13 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", meps->mnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", meps->mnodeInfos.mnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsDmeps.mnodeInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsDmeps.mnodeInfos.mnodeNum);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", meps->mnodeInfos.mnodeInfos[i].mnodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", meps->mnodeInfos.mnodeInfos[i].mnodeEp);
-    if (i < meps->mnodeInfos.mnodeNum - 1) {
+  for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp);
+    if (i < tsDmeps.mnodeInfos.mnodeNum - 1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");
@@ -87,11 +95,11 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", meps->file);
+  dInfo("successed to write %s", tsDmeps.file);
   return 0;
 }
 
-static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
+static int32_t dnodeReadMnodeEps() {
   int32_t len = 0;
   int32_t maxLen = 2000;
   char *  content = calloc(1, maxLen + 1);
@@ -100,22 +108,22 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
   SMInfos mInfos = {0};
   bool    nodeChanged = false;
 
-  fp = fopen(meps->file, "r");
+  fp = fopen(tsDmeps.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", meps->file);
+    dDebug("file %s not exist", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", meps->file);
+    dError("failed to read %s since content is null", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", meps->file);
+    dError("failed to read %s since invalid json format", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
@@ -124,7 +132,7 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
     dError("failed to read mnodeEpSet.json since inUse not found");
     goto PARSE_MINFOS_OVER;
   }
-  meps->mnodeInfos.inUse = (int8_t)inUse->valueint;
+  tsDmeps.mnodeInfos.inUse = (int8_t)inUse->valueint;
 
   cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
   if (!nodeNum || nodeNum->type != cJSON_Number) {
@@ -165,11 +173,11 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
     mInfo->mnodeId = (int32_t)nodeId->valueint;
     tstrncpy(mInfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
 
-    bool changed = dnodeIsDnodeEpChanged(deps, mInfo->mnodeId, mInfo->mnodeEp);
+    bool changed = dnodeIsDnodeEpChanged(mInfo->mnodeId, mInfo->mnodeEp);
     if (changed) nodeChanged = changed;
   }
 
-  dInfo("successed to read file %s", meps->file);
+  dInfo("successed to read file %s", tsDmeps.file);
 
 PARSE_MINFOS_OVER:
   if (content != NULL) free(content);
@@ -182,25 +190,24 @@ PARSE_MINFOS_OVER:
     dnodeGetDnodeEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
   }
 
-  dnodeResetMnodeEps(meps, &mInfos);
+  dnodeResetMnodeEps(&mInfos);
 
   if (nodeChanged) {
-    dnodeWriteMnodeEps(meps);
+    dnodeWriteMnodeEps();
   }
 
   return 0;
 }
 
 void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
-  SDnMnEps *meps = dnodeInst()->meps;
   SRpcConnInfo connInfo = {0};
   rpcGetConnInfo(rpcMsg->handle, &connInfo);
 
   SRpcEpSet epSet = {0};
   if (forShell) {
-    dnodeGetEpSetForShell(meps, &epSet);
+    dnodeGetEpSetForShell(&epSet);
   } else {
-    dnodeGetEpSetForPeer(meps, &epSet);
+    dnodeGetEpSetForPeer(&epSet);
   }
 
   dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
@@ -222,16 +229,12 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
   rpcSendRedirectRsp(rpcMsg->handle, &epSet);
 }
 
-int32_t dnodeInitMnodeEps(SDnMnEps **out) {
-  SDnMnEps *meps = calloc(1, sizeof(SDnMnEps));
-  if (meps == NULL) return -1;
+int32_t dnodeInitMnodeEps() {
+  snprintf(tsDmeps.file, sizeof(tsDmeps.file), "%s/mnodeEpSet.json", tsDnodeDir);
+  pthread_mutex_init(&tsDmeps.mutex, NULL);
 
-  snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir);
-  pthread_mutex_init(&meps->mutex, NULL);
-  *out = meps;
-
-  dnodeResetMnodeEps(meps, NULL);
-  int32_t ret = dnodeReadMnodeEps(meps, dnodeInst()->eps);
+  dnodeResetMnodeEps(NULL);
+  int32_t ret = dnodeReadMnodeEps();
   if (ret == 0) {
     dInfo("dnode mInfos is initialized");
   }
@@ -239,17 +242,11 @@ int32_t dnodeInitMnodeEps(SDnMnEps **out) {
   return ret;
 }
 
-void dnodeCleanupMnodeEps(SDnMnEps **out) {
-  SDnMnEps *meps = *out;
-  *out = NULL;
-
-  if (meps != NULL) {
-    pthread_mutex_destroy(&meps->mutex);
-    free(meps);
-  }
+void dnodeCleanupMnodeEps() {
+  pthread_mutex_destroy(&tsDmeps.mutex);
 }
 
-void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) {
+void dnodeUpdateMnodeFromStatus(SMInfos *mInfos) {
   if (mInfos->mnodeNum <= 0 || mInfos->mnodeNum > TSDB_MAX_REPLICA) {
     dError("invalid mInfos since num:%d invalid", mInfos->mnodeNum);
     return;
@@ -264,53 +261,51 @@ void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) {
     }
   }
 
-  pthread_mutex_lock(&meps->mutex);
-  if (mInfos->mnodeNum != meps->mnodeInfos.mnodeNum) {
-    dnodeResetMnodeEps(meps, mInfos);
-    dnodeWriteMnodeEps(meps);
+  pthread_mutex_lock(&tsDmeps.mutex);
+  if (mInfos->mnodeNum != tsDmeps.mnodeInfos.mnodeNum) {
+    dnodeResetMnodeEps(mInfos);
+    dnodeWriteMnodeEps();
   } else {
     int32_t size = sizeof(SMInfos);
-    if (memcmp(mInfos, &meps->mnodeInfos, size) != 0) {
-      dnodeResetMnodeEps(meps, mInfos);
-      dnodeWriteMnodeEps(meps);
+    if (memcmp(mInfos, &tsDmeps.mnodeInfos, size) != 0) {
+      dnodeResetMnodeEps(mInfos);
+      dnodeWriteMnodeEps();
     }
   }
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *ep) {
+void dnodeUpdateMnodeFromPeer(SRpcEpSet *ep) {
   if (ep->numOfEps <= 0) {
     dError("mInfos is changed, but content is invalid, discard it");
     return;
   }
 
-  pthread_mutex_lock(&meps->mutex);
+  pthread_mutex_lock(&tsDmeps.mutex);
 
   dInfo("mInfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
   for (int32_t i = 0; i < ep->numOfEps; ++i) {
     ep->port[i] -= TSDB_PORT_DNODEDNODE;
     dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
   }
-  meps->mnodeEpSet = *ep;
+  tsDmeps.mnodeEpSet = *ep;
 
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet) {
-  pthread_mutex_lock(&meps->mutex);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsDmeps.mutex);
 
-  *epSet = meps->mnodeEpSet;
+  *epSet = tsDmeps.mnodeEpSet;
   for (int32_t i = 0; i < epSet->numOfEps; ++i) {
     epSet->port[i] += TSDB_PORT_DNODEDNODE;
   }
 
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet) {
-  pthread_mutex_lock(&meps->mutex);
-
-  *epSet = meps->mnodeEpSet;
-
-  pthread_mutex_unlock(&meps->mutex);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsDmeps.mutex);
+  *epSet = tsDmeps.mnodeEpSet;
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }

source/server/dnode/src/dnodeStatus.c

@@ -25,15 +25,15 @@
 #include "dnodeMain.h"
 #include "vnode.h"
 
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  SDnStatus *status = handle;
-  if (status->dnodeTimer == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
+static struct {
+  void *   dnodeTimer;
+  void *   statusTimer;
+  uint32_t rebootTime;
+} tsStatus;
 
-  if (status->statusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+static void dnodeSendStatusMsg(void *handle, void *tmrId) {
+  if (tsStatus.statusTimer == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     dError("failed to start status timer");
     return;
   }
@@ -41,16 +41,15 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   int32_t     contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
   SStatusMsg *pStatus = rpcMallocCont(contLen);
   if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     dError("failed to malloc status message");
     return;
   }
 
-  SDnode *dnode = dnodeInst();
-  dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId);
-  pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg));
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId = htonl(dnodeGetDnodeId());
   pStatus->version = htonl(tsVersion);
-  pStatus->lastReboot = htonl(status->rebootTime);
+  pStatus->lastReboot = htonl(tsStatus.rebootTime);
   pStatus->numOfCores = htons((uint16_t)tsNumOfCores);
   pStatus->diskAvailable = tsAvailDataDirGB;
   pStatus->alternativeRole = tsAlternativeRole;
@@ -80,69 +79,58 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
   pStatus->openVnodes = htons(pStatus->openVnodes);
 
-  SRpcMsg rpcMsg = {.ahandle = status, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS};
+  SRpcMsg rpcMsg = {.ahandle = NULL, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS};
 
   dnodeSendMsgToMnode(&rpcMsg);
 }
 
 void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  SDnode *dnode = dnodeInst();
-  SDnStatus *status = pMsg->ahandle;
-
   if (pMsg->code != TSDB_CODE_SUCCESS) {
     dError("status rsp is received, error:%s", tstrerror(pMsg->code));
     if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) {
       char clusterId[TSDB_CLUSTER_ID_LEN];
-      dnodeGetClusterId(dnode->cfg, clusterId);
+      dnodeGetClusterId(clusterId);
       if (clusterId[0] != '\0') {
-        dnodeSetDropped(dnode->cfg);
+        dnodeSetDropped();
         dError("exit zombie dropped dnode");
         exit(EXIT_FAILURE);
       }
     }
 
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     return;
   }
 
   SStatusRsp *pStatusRsp = pMsg->pCont;
   SMInfos *   minfos = &pStatusRsp->mnodes;
-  dnodeUpdateMnodeFromStatus(dnode->meps, minfos);
+  dnodeUpdateMnodeFromStatus(minfos);
 
   SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
   pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
   pCfg->moduleStatus = htonl(pCfg->moduleStatus);
   pCfg->dnodeId = htonl(pCfg->dnodeId);
-  dnodeUpdateCfg(dnode->cfg, pCfg);
+  dnodeUpdateCfg(pCfg);
 
   vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
 
   SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
-  dnodeUpdateEps(dnode->eps, pEps);
+  dnodeUpdateEps(pEps);
 
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
 }
 
-int32_t dnodeInitStatus(SDnStatus **out) {
-  SDnStatus *status = calloc(1, sizeof(SDnStatus));
-  if (status == NULL) return -1;
-  status->statusTimer = NULL;
-  status->dnodeTimer = dnodeInst()->main->dnodeTimer;
-  status->rebootTime = taosGetTimestampSec();
-  taosTmrReset(dnodeSendStatusMsg, 500, status, status->dnodeTimer, &status->statusTimer);
-  *out = status;
+int32_t dnodeInitStatus() {
+  tsStatus.statusTimer = NULL;
+  tsStatus.dnodeTimer = dnodeGetTimer();
+  tsStatus.rebootTime = taosGetTimestampSec();
+  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
   dInfo("dnode status timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeCleanupStatus(SDnStatus **out) {
-  SDnStatus *status = *out;
-  *out = NULL;
-
-  if (status->statusTimer != NULL) {
-    taosTmrStopA(&status->statusTimer);
-    status->statusTimer = NULL;
+void dnodeCleanupStatus() {
+  if (tsStatus.statusTimer != NULL) {
+    taosTmrStopA(&tsStatus.statusTimer);
+    tsStatus.statusTimer = NULL;
   }
-
-  free(status);
 }

source/server/dnode/src/dnodeTelem.c

@@ -25,6 +25,19 @@
 #define TELEMETRY_PORT 80
 #define REPORT_INTERVAL 86400
 
+/*
+ * sem_timedwait is NOT implemented on MacOSX
+ * thus we use pthread_mutex_t/pthread_cond_t to simulate
+ */
+static struct {
+  bool             enable;
+  pthread_mutex_t  lock;
+  pthread_cond_t   cond;
+  volatile int32_t exit;
+  pthread_t        thread;
+  char             email[TSDB_FQDN_LEN];
+} tsTelem;
+
 static void dnodeBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
 
 static void dnodeCloseObject(SBufferWriter* bw) {
@@ -154,14 +167,14 @@ static void dnodeAddMemoryInfo(SBufferWriter* bw) {
   fclose(fp);
 }
 
-static void dnodeAddVersionInfo(SDnTelem* telem, SBufferWriter* bw) {
+static void dnodeAddVersionInfo(SBufferWriter* bw) {
   dnodeAddStringField(bw, "version", version);
   dnodeAddStringField(bw, "buildInfo", buildinfo);
   dnodeAddStringField(bw, "gitInfo", gitinfo);
-  dnodeAddStringField(bw, "email", telem->email);
+  dnodeAddStringField(bw, "email", tsTelem.email);
 }
 
-static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) {
+static void dnodeAddRuntimeInfo(SBufferWriter* bw) {
   SMnodeStat stat = {0};
   if (mnodeGetStatistics(&stat) != 0) {
     return;
@@ -179,7 +192,7 @@ static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) {
   dnodeAddIntField(bw, "compStorage", stat.compStorage);
 }
 
-static void dnodeSendTelemetryReport(SDnTelem* telem) {
+static void dnodeSendTelemetryReport() {
   char     buf[128] = {0};
   uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
   if (ip == 0xffffffff) {
@@ -192,16 +205,18 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) {
     return;
   }
 
-  SDnode *dnode = dnodeInst();
+  char clusterId[TSDB_CLUSTER_ID_LEN] = {0};
+  dnodeGetClusterId(clusterId);
+
   SBufferWriter bw = tbufInitWriter(NULL, false);
   dnodeBeginObject(&bw);
-  dnodeAddStringField(&bw, "instanceId", dnode->cfg->clusterId);
+  dnodeAddStringField(&bw, "instanceId", clusterId);
   dnodeAddIntField(&bw, "reportVersion", 1);
   dnodeAddOsInfo(&bw);
   dnodeAddCpuInfo(&bw);
   dnodeAddMemoryInfo(&bw);
-  dnodeAddVersionInfo(telem, &bw);
-  dnodeAddRuntimeInfo(telem, &bw);
+  dnodeAddVersionInfo(&bw);
+  dnodeAddRuntimeInfo(&bw);
   dnodeCloseObject(&bw);
 
   const char* header =
@@ -227,25 +242,23 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) {
 }
 
 static void* dnodeTelemThreadFp(void* param) {
-  SDnTelem* telem = param;
-
   struct timespec end = {0};
   clock_gettime(CLOCK_REALTIME, &end);
   end.tv_sec += 300;  // wait 5 minutes before send first report
 
   setThreadName("dnode-telem");
 
-  while (!telem->exit) {
+  while (!tsTelem.exit) {
     int32_t         r = 0;
     struct timespec ts = end;
-    pthread_mutex_lock(&telem->lock);
-    r = pthread_cond_timedwait(&telem->cond, &telem->lock, &ts);
-    pthread_mutex_unlock(&telem->lock);
+    pthread_mutex_lock(&tsTelem.lock);
+    r = pthread_cond_timedwait(&tsTelem.cond, &tsTelem.lock, &ts);
+    pthread_mutex_unlock(&tsTelem.lock);
     if (r == 0) break;
     if (r != ETIMEDOUT) continue;
 
     if (mnodeIsServing()) {
-      dnodeSendTelemetryReport(telem);
+      dnodeSendTelemetryReport();
     }
     end.tv_sec += REPORT_INTERVAL;
   }
@@ -253,40 +266,35 @@ static void* dnodeTelemThreadFp(void* param) {
   return NULL;
 }
 
-static void dnodeGetEmail(SDnTelem* telem, char* filepath) {
+static void dnodeGetEmail(char* filepath) {
   int32_t fd = taosOpenFileRead(filepath);
   if (fd < 0) {
     return;
   }
 
-  if (taosReadFile(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) {
+  if (taosReadFile(fd, (void*)tsTelem.email, TSDB_FQDN_LEN) < 0) {
     dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
   }
 
   taosCloseFile(fd);
 }
 
-int32_t dnodeInitTelem(SDnTelem** out) {
-  SDnTelem* telem = calloc(1, sizeof(SDnTelem));
-  if (telem == NULL) return -1;
+int32_t dnodeInitTelem() {
+  tsTelem.enable = tsEnableTelemetryReporting;
+  if (!tsTelem.enable) return 0;
 
-  telem->enable = tsEnableTelemetryReporting;
-  *out = telem;
+  tsTelem.exit = 0;
+  pthread_mutex_init(&tsTelem.lock, NULL);
+  pthread_cond_init(&tsTelem.cond, NULL);
+  tsTelem.email[0] = 0;
 
-  if (!telem->enable) return 0;
-
-  telem->exit = 0;
-  pthread_mutex_init(&telem->lock, NULL);
-  pthread_cond_init(&telem->cond, NULL);
-  telem->email[0] = 0;
-
-  dnodeGetEmail(telem, "/usr/local/taos/email");
+  dnodeGetEmail("/usr/local/taos/email");
 
   pthread_attr_t attr;
   pthread_attr_init(&attr);
   pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
 
-  int32_t code = pthread_create(&telem->thread, &attr, dnodeTelemThreadFp, telem);
+  int32_t code = pthread_create(&tsTelem.thread, &attr, dnodeTelemThreadFp, NULL);
   pthread_attr_destroy(&attr);
   if (code != 0) {
     dTrace("failed to create telemetry thread since :%s", strerror(code));
@@ -296,26 +304,18 @@ int32_t dnodeInitTelem(SDnTelem** out) {
   return 0;
 }
 
-void dnodeCleanupTelem(SDnTelem** out) {
-  SDnTelem* telem = *out;
-  *out = NULL;
+void dnodeCleanupTelem() {
+  if (!tsTelem.enable) return;
 
-  if (!telem->enable) {
-    free(telem);
-    return;
+  if (taosCheckPthreadValid(tsTelem.thread)) {
+    pthread_mutex_lock(&tsTelem.lock);
+    tsTelem.exit = 1;
+    pthread_cond_signal(&tsTelem.cond);
+    pthread_mutex_unlock(&tsTelem.lock);
+
+    pthread_join(tsTelem.thread, NULL);
   }
 
-  if (taosCheckPthreadValid(telem->thread)) {
-    pthread_mutex_lock(&telem->lock);
-    telem->exit = 1;
-    pthread_cond_signal(&telem->cond);
-    pthread_mutex_unlock(&telem->lock);
-
-    pthread_join(telem->thread, NULL);
-  }
-
-  pthread_mutex_destroy(&telem->lock);
-  pthread_cond_destroy(&telem->cond);
-
-  free(telem);
+  pthread_mutex_destroy(&tsTelem.lock);
+  pthread_cond_destroy(&tsTelem.cond);
 }

source/server/dnode/src/dnodeTrans.c

@@ -14,7 +14,7 @@
  */
 
 /* this file is mainly responsible for the communication between DNODEs. Each 
- * dnode works as both server and client. SDnode may send status, grant, config
+ * dnode works as both server and client. Dnode may send status, grant, config
  * messages to mnode, mnode may send create/alter/drop table/vnode messages 
  * to dnode. All theses messages are handled from here
  */
@@ -29,8 +29,19 @@
 #include "vnode.h"
 #include "mnode.h"
 
+typedef void (*RpcMsgFp)( SRpcMsg *pMsg);
+
+static struct {
+  void *   serverRpc;
+  void *   clientRpc;
+  void *   shellRpc;
+  int32_t  queryReqNum;
+  int32_t  submitReqNum;
+  RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX];
+  RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX];
+} tsTrans;
+
 static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode * dnode = dnodeInst();
   SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0};
 
   if (pMsg->pCont == NULL) return;
@@ -39,7 +50,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) {
+  if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);
@@ -53,7 +64,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -64,27 +75,27 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 }
 
-int32_t dnodeInitServer(SDnTrans *trans) {
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE]  = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]   = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]   = vnodeProcessMsg;
+int32_t dnodeInitServer() {
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]   = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]   = vnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE]  = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]   = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]   = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM]  = vnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE]  = dnodeProcessConfigDnodeReq;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE]  = dnodeProcessCreateMnodeReq;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE]  = dnodeProcessConfigDnodeReq;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE]  = dnodeProcessCreateMnodeReq;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH]          = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT]         = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS]        = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH]          = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT]         = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS]        = mnodeProcessMsg;
 
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
@@ -96,8 +107,8 @@ int32_t dnodeInitServer(SDnTrans *trans) {
   rpcInit.connType = TAOS_CONN_SERVER;
   rpcInit.idleTime = tsShellActivityTimer * 1000;
 
-  trans->serverRpc = rpcOpen(&rpcInit);
-  if (trans->serverRpc == NULL) {
+  tsTrans.serverRpc = rpcOpen(&rpcInit);
+  if (tsTrans.serverRpc == NULL) {
     dError("failed to init peer rpc server");
     return -1;
   }
@@ -106,17 +117,16 @@ int32_t dnodeInitServer(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupServer(SDnTrans *trans) {
-  if (trans->serverRpc) {
-    rpcClose(trans->serverRpc);
-    trans->serverRpc = NULL;
+void dnodeCleanupServer() {
+  if (tsTrans.serverRpc) {
+    rpcClose(tsTrans.serverRpc);
+    tsTrans.serverRpc = NULL;
     dInfo("dnode peer server is closed");
   }
 }
 
 static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) {
+  if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
     dTrace("msg:%p is ignored since dnode is stopping", pMsg);
     rpcFreeCont(pMsg->pCont);
@@ -124,10 +134,10 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 
   if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
-    dnodeUpdateMnodeFromPeer(dnode->meps, pEpSet);
+    dnodeUpdateMnodeFromPeer(pEpSet);
   }
 
-  RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -141,27 +151,27 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   rpcFreeCont(pMsg->pCont);
 }
 
-int32_t dnodeInitClient(SDnTrans *trans) {
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP]   = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP]   = mnodeProcessMsg;
+int32_t dnodeInitClient() {
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP]   = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP]   = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP]   = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP]   = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP]  = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP]  = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP]          = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP]         = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP]        = dnodeProcessStatusRsp;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP]          = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP]         = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP]        = dnodeProcessStatusRsp;
 
   char secret[TSDB_KEY_LEN] = "secret";
   SRpcInit rpcInit;
@@ -176,8 +186,8 @@ int32_t dnodeInitClient(SDnTrans *trans) {
   rpcInit.ckey         = "key";
   rpcInit.secret       = secret;
 
-  trans->clientRpc = rpcOpen(&rpcInit);
-  if (trans->clientRpc == NULL) {
+  tsTrans.clientRpc = rpcOpen(&rpcInit);
+  if (tsTrans.clientRpc == NULL) {
     dError("failed to init peer rpc client");
     return -1;
   }
@@ -186,26 +196,25 @@ int32_t dnodeInitClient(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupClient(SDnTrans *trans) {
-  if (trans->clientRpc) {
-    rpcClose(trans->clientRpc);
-    trans->clientRpc = NULL;
+void dnodeCleanupClient() {
+  if (tsTrans.clientRpc) {
+    rpcClose(tsTrans.clientRpc);
+    tsTrans.clientRpc = NULL;
     dInfo("dnode peer rpc client is closed");
   }
 }
 
 static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode * dnode = dnodeInst();
   SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0};
 
   if (pMsg->pCont == NULL) return;
-  if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) {
+  if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) {
     dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_DND_EXITING;
     rpcSendResponse(&rpcMsg);
     rpcFreeCont(pMsg->pCont);
     return;
-  } else if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) {
+  } else if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) {
     dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rpcMsg);
@@ -213,14 +222,13 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  SDnTrans *trans = dnode->trans;
   if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
-    atomic_fetch_add_32(&trans->queryReqNum, 1);
+    atomic_fetch_add_32(&tsTrans.queryReqNum, 1);
   } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
-    atomic_fetch_add_32(&trans->submitReqNum, 1);
+    atomic_fetch_add_32(&tsTrans.submitReqNum, 1);
   } else {}
 
-  RpcMsgFp fp = trans->shellMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.shellMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -247,27 +255,23 @@ static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secr
 }
 
 void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
-  SDnode *dnode = dnodeInst();
-  rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL);
+  rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL);
 }
 
 void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
-  SDnode *   dnode = dnodeInst();
   SRpcEpSet epSet = {0};
-  dnodeGetEpSetForPeer(dnode->meps, &epSet);
+  dnodeGetEpSetForPeer(&epSet);
   dnodeSendMsgToDnode(&epSet, rpcMsg);
 }
 
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
-  SDnode *   dnode = dnodeInst();
   SRpcEpSet epSet = {0};
-  dnodeGetEpSetForPeer(dnode->meps, &epSet);
-  rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp);
+  dnodeGetEpSetForPeer(&epSet);
+  rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp);
 }
 
 void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
-  SDnode *dnode = dnodeInst();
-  rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp);
+  rpcSendRecv(tsTrans.clientRpc, epSet, rpcMsg, rpcRsp);
 }
 
 static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
@@ -303,52 +307,52 @@ static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, c
   return rpcRsp.code;
 }
 
-int32_t dnodeInitShell(SDnTrans *trans) {
-  trans->shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_QUERY]  = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_FETCH]  = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg;
+int32_t dnodeInitShell() {
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_QUERY]  = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_FETCH]  = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg;
 
   // the following message shall be treated as mnode write
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION]   = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE]   = mnodeProcessMsg;
 
   // the following message shall be treated as mnode query
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META]   = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_SHOW]          = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SHOW]          = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg;
 
-  trans->shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]     = dnodeProcessStartupReq;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]     = dnodeProcessStartupReq;
 
   int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
   if (numOfThreads < 1) {
@@ -366,8 +370,8 @@ int32_t dnodeInitShell(SDnTrans *trans) {
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.afp          = dnodeRetrieveUserAuthInfo;
 
-  trans->shellRpc = rpcOpen(&rpcInit);
-  if (trans->shellRpc == NULL) {
+  tsTrans.shellRpc = rpcOpen(&rpcInit);
+  if (tsTrans.shellRpc == NULL) {
     dError("failed to init shell rpc server");
     return -1;
   }
@@ -376,41 +380,31 @@ int32_t dnodeInitShell(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupShell(SDnTrans *trans) {
-  if (trans->shellRpc) {
-    rpcClose(trans->shellRpc);
-    trans->shellRpc = NULL;
+void dnodeCleanupShell() {
+  if (tsTrans.shellRpc) {
+    rpcClose(tsTrans.shellRpc);
+    tsTrans.shellRpc = NULL;
   }
 }
 
-int32_t dnodeInitTrans(SDnTrans **out) {
-  SDnTrans *trans = calloc(1, sizeof(SDnTrans));
-  if (trans == NULL) return -1;
-
-  *out = trans;
-
-  if (dnodeInitClient(trans) != 0) {
+int32_t dnodeInitTrans() {
+  if (dnodeInitClient() != 0) {
     return -1;
   }
 
-  if (dnodeInitServer(trans) != 0) {
+  if (dnodeInitServer() != 0) {
     return -1;
   }
 
-  if (dnodeInitShell(trans) != 0) {
+  if (dnodeInitShell() != 0) {
     return -1;
   }
 
   return 0;
 }
 
-void dnodeCleanupTrans(SDnTrans **out) {
-  SDnTrans* trans = *out;
-  *out = NULL;
-
-  dnodeCleanupShell(trans);
-  dnodeCleanupServer(trans);
-  dnodeCleanupClient(trans);
-
-  free(trans);
+void dnodeCleanupTrans() {
+  dnodeCleanupShell();
+  dnodeCleanupServer();
+  dnodeCleanupClient();
 }

source/server/vnode/CMakeLists.txt

@@ -3,7 +3,7 @@ add_subdirectory(tq)
 add_subdirectory(tsdb)
 
 aux_source_directory(src VNODE_SRC)
-add_library(vnode ${VNODE_SRC})
+add_library(vnode STATIC ${VNODE_SRC})
 target_include_directories(
     vnode
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode"
@@ -15,4 +15,11 @@ target_link_libraries(
     PUBLIC meta
     PUBLIC tq
     PUBLIC tsdb
-)
+    PUBLIC wal
+    PUBLIC cjson
+)
+
+# test
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/server/vnode/inc/vnodeCfg.h

@@ -13,19 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_CFG_H
-#define TDENGINE_VNODE_CFG_H
+#ifndef _TD_VNODE_CFG_H_
+#define _TD_VNODE_CFG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeReadCfg(SVnodeObj *pVnode);
+int32_t vnodeReadCfg(SVnode *pVnode);
 int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_CFG_H_*/

source/server/vnode/inc/vnodeInt.h

@@ -16,7 +16,6 @@
 #ifndef _TD_VNODE_INT_H_
 #define _TD_VNODE_INT_H_
 
-
 #include "os.h"
 #include "amalloc.h"
 #include "meta.h"
@@ -25,20 +24,83 @@
 #include "trpc.h"
 #include "tsdb.h"
 #include "vnode.h"
+#include "tlog.h"
+#include "tqueue.h"
+#include "wal.h"
+#include "tworker.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SVnode {
+extern int32_t vDebugFlag;
+
+#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }}
+#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }}
+#define vWarn(...)  { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }}
+#define vInfo(...)  { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("VND ", 255, __VA_ARGS__); }}
+#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
+#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
+
+typedef struct {
   SMeta *        pMeta;
   STsdb *        pTsdb;
   STQ *          pTQ;
   SMemAllocator *allocator;
+
+  int32_t  vgId;      // global vnode group ID
+  int32_t  refCount;  // reference count
+  int64_t  queuedWMsgSize;
+  int32_t  queuedWMsg;
+  int32_t  queuedRMsg;
+  int32_t  numOfExistQHandle;  // current initialized and existed query handle in current dnode
+  int32_t  flowctrlLevel;
+  int8_t   preClose;  // drop and close switch
+  int8_t   reserved[3];
+  int64_t  sequence;  // for topic
+  int8_t   status;
+  int8_t   role;
+  int8_t   accessState;
+  int8_t   isFull;
+  int8_t   isCommiting;
+  int8_t   dbReplica;
+  int8_t   dropped;
+  int8_t   dbType;
+  uint64_t version;   // current version
+  uint64_t cversion;  // version while commit start
+  uint64_t fversion;  // version on saved data file
+  void *   wqueue;    // write queue
+  void *   qqueue;    // read query queue
+  void *   fqueue;    // read fetch/cancel queue
+  void *   wal;
+  void *   tsdb;
+  int64_t  sync;
+  void *   events;
+  void *   cq;  // continuous query
+  int32_t  dbCfgVersion;
+  int32_t  vgCfgVersion;
+  // STsdbCfg tsdbCfg;
+#if 0  
+  SSyncCfg syncCfg;
+#endif  
+  SWalCfg  walCfg;
+  void *   qMgmt;
+  char *   rootDir;
+  tsem_t   sem;
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
+  pthread_mutex_t statusMutex;
 } SVnode;
 
+typedef struct {
+  int32_t len;
+  void *  rsp;
+  void *  qhandle;  // used by query and retrieve msg
+} SVnRsp;
+
+void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_VNODE_INT_H_*/
+#endif /*_TD_VNODE_INT_H_*/

source/server/vnode/inc/vnodeMain.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_MAIN_H_
+#define _TD_VNODE_MAIN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitMain();
+void    vnodeCleanupMain();
+
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeDrop(int32_t vgId);
+int32_t vnodeOpen(int32_t vgId);
+int32_t vnodeAlter(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeSync(int32_t vgId);
+int32_t vnodeClose(int32_t vgId);
+void    vnodeCleanUp(SVnode *pVnode);
+void    vnodeDestroy(SVnode *pVnode);
+int32_t vnodeCompact(int32_t vgId);
+void    vnodeBackup(int32_t vgId);
+void    vnodeGetStatus(struct SStatusMsg *status);
+
+SVnode *vnodeAcquire(int32_t vgId);
+SVnode *vnodeAcquireNotClose(int32_t vgId);
+void    vnodeRelease(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_MAIN_H_*/

source/server/vnode/inc/vnodeMemAllocator.h

@@ -16,12 +16,38 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "amalloc.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SVnodeMemAllocator SVnodeMemAllocator;
+
+SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
+                              size_t threshold /* threshold size when full*/);
+void                VMADestroy(SVnodeMemAllocator *pvma);
+void                VMAReset(SVnodeMemAllocator *pvma);
+void *              VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
+void                VMAFree(SVnodeMemAllocator *pvma, void *ptr);
+bool                VMAIsFull(SVnodeMemAllocator *pvma);
+
+// ------------------ FOR TEST ONLY ------------------
+typedef struct SVMANode {
+  struct SVMANode *prev;
+  size_t           tsize;
+  size_t           used;
+  char             data[];
+} SVMANode;
+
+struct SVnodeMemAllocator {
+  bool      full;       // if allocator is full
+  size_t    threshold;  // threshold;
+  size_t    ssize;      // step size to allocate
+  SVMANode *inuse;      // inuse node to allocate
+  SVMANode  node;       // basic node to use
+};
+
 #ifdef __cplusplus
 }
 #endif

source/server/vnode/inc/vnodeMgmt.h

@@ -13,20 +13,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_BACKUP_H
-#define TDENGINE_VNODE_BACKUP_H
+#ifndef _TD_VNODE_MGMT_H_
+#define _TD_VNODE_MGMT_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitBackup();
-void    vnodeCleanupBackup();
-int32_t vnodeBackup(int32_t vgId);
+int32_t vnodeInitMgmt();
+void    vnodeCleanupMgmt();
+void    vnodeProcessMgmtMsg(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_MGMT_H_*/

source/server/vnode/inc/vnodeMgmtMsg.h

@@ -13,25 +13,23 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_MAIN_H
-#define TDENGINE_VNODE_MAIN_H
+#ifndef _TD_VNODE_MGMT_MSG_H_
+#define _TD_VNODE_MGMT_MSG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
-int32_t vnodeDrop(int32_t vgId);
-int32_t vnodeOpen(int32_t vgId);
-int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
-int32_t vnodeSync(int32_t vgId);
-int32_t vnodeClose(int32_t vgId);
-void    vnodeCleanUp(SVnodeObj *pVnode);
-void    vnodeDestroy(SVnodeObj *pVnode);
+int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_MGMT_H_*/

source/server/vnode/inc/vnodeRead.h

@@ -13,30 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_MGMT_H
-#define TDENGINE_VNODE_MGMT_H
+#ifndef _TD_VNODE_READ_H_
+#define _TD_VNODE_READ_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitMgmt();
-void    vnodeCleanupMgmt();
+int32_t    vnodeInitRead();
+void       vnodeCleanupRead();
+taos_queue vnodeAllocQueryQueue(SVnode *pVnode);
+taos_queue vnodeAllocFetchQueue(SVnode *pVnode);
+void       vnodeFreeQueryQueue(taos_queue pQueue);
+void       vnodeFreeFetchQueue(taos_queue pQueue);
 
-void*   vnodeAcquire(int32_t vgId);
-void    vnodeRelease(void *pVnode);
-void*   vnodeGetWal(void *pVnode);
+void vnodeProcessReadMsg(SRpcMsg *pRpcMsg);
+int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle);
 
-int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
-void    vnodeBuildStatusMsg(void *pStatus);
-void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
-
-void    vnodeAddIntoHash(SVnodeObj* pVnode);
-void    vnodeRemoveFromHash(SVnodeObj * pVnode);
+void vnodeStartRead(SVnode *pVnode);
+void vnodeStopRead(SVnode *pVnode);
+void vnodeWaitReadCompleted(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_READ_H_*/

source/server/vnode/inc/vnodeReadMsg.h

@@ -13,24 +13,32 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_WRITE_H
-#define TDENGINE_VNODE_WRITE_H
+#ifndef _TD_VNODE_READ_MSG_H_
+#define _TD_VNODE_READ_MSG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitWrite(void);
-void    vnodeCleanupWrite(void);
+typedef struct SReadMsg {
+  int32_t code;
+  int32_t contLen;
+  int8_t  qtype;
+  int8_t  msgType;
+  SVnode *pVnode;
+  SVnRsp rspRet;
+  void *  rpcHandle;
+  void *  rpcAhandle;
+  void *  qhandle;
+  char    pCont[];
+} SReadMsg;
 
-int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
-void    vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
-int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
-void    vnodeWaitWriteCompleted(SVnodeObj *pVnode);
+int32_t vnodeProcessQueryMsg(SVnode *pVnode, SReadMsg *pRead);
+int32_t vnodeProcessFetchMsg(SVnode *pVnode, SReadMsg *pRead);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_READ_MSG_H_*/

source/server/vnode/inc/vnodeStatus.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_STATUS_H
-#define TDENGINE_VNODE_STATUS_H
+#ifndef _TD_VNODE_STATUS_H_
+#define _TD_VNODE_STATUS_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -25,24 +25,23 @@ typedef enum _VN_STATUS {
   TAOS_VN_STATUS_INIT = 0,
   TAOS_VN_STATUS_READY = 1,
   TAOS_VN_STATUS_CLOSING = 2,
-  TAOS_VN_STATUS_UPDATING = 3,
-  TAOS_VN_STATUS_RESET = 4,
+  TAOS_VN_STATUS_UPDATING = 3
 } EVnodeStatus;
 
-bool vnodeSetInitStatus(SVnodeObj* pVnode);
-bool vnodeSetReadyStatus(SVnodeObj* pVnode);
-bool vnodeSetClosingStatus(SVnodeObj* pVnode);
-bool vnodeSetUpdatingStatus(SVnodeObj* pVnode);
-bool vnodeSetResetStatus(SVnodeObj* pVnode);
+// vnodeStatus
+extern char* vnodeStatus[];
 
-bool vnodeInInitStatus(SVnodeObj* pVnode);
-bool vnodeInReadyStatus(SVnodeObj* pVnode);
-bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode);
-bool vnodeInClosingStatus(SVnodeObj* pVnode);
-bool vnodeInResetStatus(SVnodeObj* pVnode);
+bool vnodeSetInitStatus(SVnode* pVnode);
+bool vnodeSetReadyStatus(SVnode* pVnode);
+bool vnodeSetClosingStatus(SVnode* pVnode);
+bool vnodeSetUpdatingStatus(SVnode* pVnode);
+
+bool vnodeInInitStatus(SVnode* pVnode);
+bool vnodeInReadyStatus(SVnode* pVnode);
+bool vnodeInClosingStatus(SVnode* pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_STATUS_H_*/

source/server/vnode/inc/vnodeVersion.h

@@ -13,19 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_VERSION_H
-#define TDENGINE_VNODE_VERSION_H
+#ifndef _TD_VNODE_VERSION_H_
+#define _TD_VNODE_VERSION_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeReadVersion(SVnodeObj *pVnode);
-int32_t vnodeSaveVersion(SVnodeObj *pVnode);
+int32_t vnodeReadVersion(SVnode *pVnode);
+int32_t vnodeSaveVersion(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_VERSION_H_*/

source/server/vnode/inc/vnodeWorker.h

@@ -13,21 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_WORKER_H
-#define TDENGINE_VNODE_WORKER_H
+#ifndef _TD_VNODE_WORKER_H_
+#define _TD_VNODE_WORKER_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitMWorker();
-void    vnodeCleanupMWorker();
-int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode);
-int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode);
+int32_t vnodeInitWorker();
+void    vnodeCleanupWorker();
+void    vnodeProcessCleanupTask(SVnode *pVnode);
+void    vnodeProcessDestroyTask(SVnode *pVnode);
+void    vnodeProcessBackupTask(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_WORKER_H_*/

source/server/vnode/inc/vnodeWrite.h

@@ -16,6 +16,25 @@
 #ifndef _TD_VNODE_WRITE_H_
 #define _TD_VNODE_WRITE_H_
 
-int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp);
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t    vnodeInitWrite();
+void       vnodeCleanupWrite();
+taos_queue vnodeAllocWriteQueue(SVnode *pVnode);
+void       vnodeFreeWriteQueue(taos_queue pQueue);
+
+void    vnodeProcessWriteMsg(SRpcMsg *pRpcMsg);
+int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead);
+
+void vnodeStartWrite(SVnode *pVnode);
+void vnodeStopWrite(SVnode *pVnode);
+void vnodeWaitWriteCompleted(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /*_TD_VNODE_WRITE_H_*/

source/server/vnode/inc/vnodeWriteMsg.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_WRITE_MSG_H_
+#define _TD_VNODE_WRITE_MSG_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+ 
+int32_t vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp);
+int32_t vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp);
+int32_t vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp);
+int32_t vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp);
+int32_t vnodeProcessDropStableReq(SVnode *pVnode, SDropStableReq *pReq, SDropStableRsp *pRsp);
+int32_t vnodeProcessUpdateTagValReq(SVnode *pVnode, SUpdateTagValReq *pReq, SUpdateTagValRsp *pRsp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_WRITE_MSG_H_*/

source/server/vnode/meta/CMakeLists.txt

@@ -1,5 +1,5 @@
 aux_source_directory(src META_SRC)
-add_library(meta ${META_SRC})
+add_library(meta STATIC ${META_SRC})
 target_include_directories(
     meta
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
@@ -8,4 +8,9 @@ target_include_directories(
 target_link_libraries(
     meta
     PUBLIC common
-)
+)
+target_link_libraries(meta PUBLIC rocksdb)
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/server/vnode/meta/src/meta.c

@@ -13,24 +13,72 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <rocksdb/c.h>
+
+#include "thash.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "ttypes.h"
+
 #include "meta.h"
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) {
-  // TODO
-  return 0;
+typedef struct STable {
+  uint64_t uid;
+  tstr *   name;
+  uint64_t suid;
+  SArray * schema;
+} STable;
+
+typedef struct STableObj {
+  bool     pin;
+  uint64_t ref;
+  SRWLatch latch;
+  uint64_t offset;
+  SList *  ctbList;  //  child table list
+  STable * pTable;
+} STableObj;
+
+struct SMeta {
+  pthread_rwlock_t rwLock;
+
+  SHashObj * pTableObjHash;  // uid --> STableObj
+  SList *    stbList;        // super table list
+  rocksdb_t *tbnameDb;       // tbname --> uid
+  rocksdb_t *tagDb;          // uid --> tag
+  rocksdb_t *schemaDb;
+  size_t     totalUsed;
+};
+
+SMeta *metaOpen(SMetaOptions *options) {
+  SMeta *pMeta = NULL;
+  char * err = NULL;
+
+  pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
+  if (pMeta == NULL) {
+    return NULL;
+  }
+
+  pthread_rwlock_init(&(pMeta->rwLock), NULL);
+
+  pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+
+  pMeta->stbList = tdListNew(sizeof(STableObj *));
+
+  // Open tbname DB
+  rocksdb_options_t *tbnameDbOptions = rocksdb_options_create();
+  pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err);
+
+  // Open tag DB
+  pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err);
+
+  // Open schema DB
+  pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err);
+
+  return pMeta;
 }
 
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) {
+void metaClose(SMeta *pMeta) {
   // TODO
-  return 0;
 }
 
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) {
-  // TODO
-  return 0;
-}
-
-int metaCommit(SMeta *pMeta) {
-  // TODO
-  return 0;
-}
+int metaCommit(SMeta *meta) { return 0; }

source/server/vnode/meta/test/CMakeLists.txt

@@ -0,0 +1,18 @@
+add_executable(metaTest "")
+target_sources(metaTest
+    PRIVATE
+    "../src/meta.c"
+    "metaTests.cpp"
+)
+target_include_directories(metaTest 
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_link_libraries(metaTest
+    os
+    util
+    common
+    rocksdb
+    gtest_main
+)

source/server/vnode/meta/test/metaTests.cpp

@@ -0,0 +1,9 @@
+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "meta.h"
+
+TEST(MetaTest, meta_open_test) {
+    metaOpen(NULL);
+    std::cout << "Hello META!" << std::endl;
+}

source/server/vnode/src/vnodeCfg.c

@@ -17,10 +17,10 @@
 #include "os.h"
 #include "cJSON.h"
 #include "tglobal.h"
-#include "dnode.h"
 #include "vnodeCfg.h"
 
-static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
+static void vnodeLoadCfg(SVnode *pVnode, SCreateVnodeMsg *vnodeMsg) {
+#if 0
   tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
   pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion;
   pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion;
@@ -56,9 +56,11 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
     SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i];
     vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort);
   }
+#endif  
 }
 
-int32_t vnodeReadCfg(SVnodeObj *pVnode) {
+int32_t vnodeReadCfg(SVnode *pVnode) {
+#if 0
   int32_t ret = TSDB_CODE_VND_APP_ERROR;
   int32_t len = 0;
   int     maxLen = 1000;
@@ -66,6 +68,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
   cJSON * root = NULL;
   FILE *  fp = NULL;
   bool    nodeChanged = false;
+
   SCreateVnodeMsg vnodeMsg;
 
   char file[TSDB_FILENAME_LEN + 30] = {0};
@@ -286,8 +289,13 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
     }
     tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
 
-    bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
-    if (changed) nodeChanged = changed;
+    char nodeEpStr[TSDB_EP_LEN];
+    vnodeGetDnodeEp(node->nodeId, nodeEpStr, NULL, NULL);
+    bool changed = (strcmp(node->nodeEp, nodeEpStr) != 0);
+    if (changed) {
+      tstrncpy(node->nodeEp, nodeEpStr, TSDB_EP_LEN);
+      nodeChanged = changed;
+    }
   }
 
   ret = TSDB_CODE_SUCCESS;
@@ -350,7 +358,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
   for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {
     SVnodeDesc *node = &pMsg->nodes[i];
-    dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL);
+    vnodeGetDnodeEp(node->nodeId, node->nodeEp, NULL, NULL);
     len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", node->nodeId);
     len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", node->nodeEp);
     if (i < pMsg->cfg.vgReplica - 1) {
@@ -368,5 +376,6 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   terrno = 0;
 
   vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file);
+#endif  
   return TSDB_CODE_SUCCESS;
 }

source/server/vnode/src/vnodeInt.c

@@ -13,16 +13,39 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tstep.h"
+#include "vnodeMain.h"
+#include "vnodeMgmt.h"
+#include "vnodeRead.h"
+#include "vnodeWorker.h"
+#include "vnodeWrite.h"
 
-int32_t vnodeInit(SVnodePara para) { return 0; }
+static struct {
+  struct SSteps *steps;
+  SVnodeFp       fp;
+} tsVint;
 
-void vnodeCleanup() {}
+int32_t vnodeInit(SVnodePara para) {
+  tsVint.fp = para.fp;
 
-int32_t vnodeGetStatistics(SVnodeStat *stat) { return 0; }
+  struct SSteps *steps = taosStepInit(8, NULL);
+  if (steps == NULL) return -1;
 
-void vnodeGetStatus(struct SStatusMsg *status) {}
+  taosStepAdd(steps, "vnode-main", vnodeInitMain, vnodeCleanupMain);
+  taosStepAdd(steps, "vnode-worker",vnodeInitWorker, vnodeCleanupWorker);
+  taosStepAdd(steps, "vnode-read", vnodeInitRead, vnodeCleanupRead);
+  taosStepAdd(steps, "vnode-mgmt", vnodeInitMgmt, vnodeCleanupMgmt);
+  taosStepAdd(steps, "vnode-write", vnodeInitWrite, vnodeCleanupWrite);
+  // taosStepAdd(steps, "vnode-queue", tsdbInitCommitQueue, tsdbDestroyCommitQueue);
 
-void vnodeSetAccess(struct SVgroupAccess *access, int32_t numOfVnodes) {}
+  tsVint.steps = steps;
+  return taosStepExec(tsVint.steps);
+}
 
-void vnodeProcessMsg(SRpcMsg *msg) {}
+void vnodeCleanup() { taosStepCleanup(tsVint.steps); }
+
+void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) {
+  return (*tsVint.fp.GetDnodeEp)(dnodeId, ep, fqdn, port);
+}

source/server/vnode/src/vnodeMain.c

@@ -18,30 +18,104 @@
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tglobal.h"
-#include "tfs.h"
-#include "query.h"
-#include "dnode.h"
+#include "ttimer.h"
+#include "thash.h"
+// #include "query.h"
 #include "vnodeCfg.h"
-#include "vnodeStatus.h"
-#include "vnodeSync.h"
-#include "vnodeVersion.h"
-#include "vnodeMgmt.h"
-#include "vnodeWorker.h"
-#include "vnodeBackup.h"
 #include "vnodeMain.h"
+#include "vnodeMgmt.h"
+#include "vnodeRead.h"
+#include "vnodeStatus.h"
+#include "vnodeVersion.h"
+#include "vnodeWorker.h"
+#include "vnodeWrite.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   threadIndex;
+  int32_t   failed;
+  int32_t   opened;
+  int32_t   vnodeNum;
+  int32_t * vnodeList;
+} SOpenVnodeThread;
+
+static struct {
+  void *    timer;
+  SHashObj *hash;
+  int32_t   openVnodes;
+  int32_t   totalVnodes;
+  void (*msgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
+} tsVmain;
+
+static void vnodeIncRef(void *ptNode) {
+  assert(ptNode != NULL);
+
+  SVnode **ppVnode = (SVnode **)ptNode;
+  assert(ppVnode);
+  assert(*ppVnode);
+
+  SVnode *pVnode = *ppVnode;
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
+}
+
+SVnode *vnodeAcquire(int32_t vgId) {
+  SVnode *pVnode = NULL;
+
+#if 0
+  taosHashGetClone(tsVmain.hash, &vgId, sizeof(int32_t), vnodeIncRef, &pVnode);
+#endif 
+  if (pVnode == NULL) {
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+    vDebug("vgId:%d, not exist", vgId);
+    return NULL;
+  }
+
+  return pVnode;
+}
+
+SVnode *vnodeAcquireNotClose(int32_t vgId) {
+  SVnode *pVnode = vnodeAcquire(vgId);
+  if (pVnode != NULL && pVnode->preClose == 1) {
+    vnodeRelease(pVnode);
+    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+    vDebug("vgId:%d, not exist, pre closing", vgId);
+    return NULL;
+  }
+
+  return pVnode;
+}
+
+void vnodeRelease(SVnode *pVnode) {
+  if (pVnode == NULL) return;
+
+  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+  int32_t vgId = pVnode->vgId;
+
+  vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode);
+  assert(refCount >= 0);
+
+  if (refCount <= 0) {
+    vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode);
+    vnodeProcessDestroyTask(pVnode);
+    int32_t count = taosHashGetSize(tsVmain.hash);
+    vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
+  }
+}
 
 static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
 
 int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
   int32_t code;
 
-  SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId);
+  SVnode *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId);
   if (pVnode != NULL) {
     vDebug("vgId:%d, vnode already exist, refCount:%d pVnode:%p", pVnodeCfg->cfg.vgId, pVnode->refCount, pVnode);
     vnodeRelease(pVnode);
     return TSDB_CODE_SUCCESS;
   }
 
+#if 0
   if (tfsMkdir("vnode") < 0) {
     vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
     return terrno;
@@ -64,7 +138,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
     vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
     return TSDB_CODE_VND_INIT_FAILED;
   }
-
+#endif
   vInfo("vgId:%d, vnode dir is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel,
         pVnodeCfg->cfg.fsyncPeriod);
   code = vnodeOpen(pVnodeCfg->cfg.vgId);
@@ -73,7 +147,8 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
 }
 
 int32_t vnodeSync(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquireNotClose(vgId);
+#if 0  
+  SVnode *pVnode = vnodeAcquireNotClose(vgId);
   if (pVnode == NULL) {
     vDebug("vgId:%d, failed to sync, vnode not find", vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
@@ -90,13 +165,12 @@ int32_t vnodeSync(int32_t vgId) {
   }
 
   vnodeRelease(pVnode);
-
+#endif
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t vnodeDrop(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquireNotClose(vgId);
+  SVnode *pVnode = vnodeAcquireNotClose(vgId);
   if (pVnode == NULL) {
     vDebug("vgId:%d, failed to drop, vnode not find", vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
@@ -110,25 +184,29 @@ int32_t vnodeDrop(int32_t vgId) {
   pVnode->dropped = 1;
 
   vnodeRelease(pVnode);
-  vnodeCleanupInMWorker(pVnode);
+  vnodeProcessCleanupTask(pVnode);
 
   return TSDB_CODE_SUCCESS;
 }
+
 int32_t vnodeCompact(int32_t vgId) {
-  void *pVnode = vnodeAcquire(vgId);
+#if 0  
+  SVnode *pVnode = vnodeAcquire(vgId);
   if (pVnode != NULL) {
     vDebug("vgId:%d, compact vnode msg is received", vgId);
-    //not care success or not
-    tsdbCompact(((SVnodeObj*)pVnode)->tsdb);  
+    // not care success or not
+    tsdbCompact(((SVnode *)pVnode)->tsdb);
     vnodeRelease(pVnode);
   } else {
     vInfo("vgId:%d, vnode not exist, can't compact it", vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
   }
-  return TSDB_CODE_SUCCESS;  
+#endif   
+  return TSDB_CODE_SUCCESS;
 }
 
-static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) {
+static int32_t vnodeAlterImp(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg) {
+#if 0  
   STsdbCfg tsdbCfg = pVnode->tsdbCfg;
   SSyncCfg syncCfg = pVnode->syncCfg;
   int32_t  dbCfgVersion = pVnode->dbCfgVersion;
@@ -204,13 +282,11 @@ static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) {
 
     vnodeSetReadyStatus(pVnode);
   }
-
+#endif
   return 0;
 }
 
-int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
-  SVnodeObj *pVnode = vparam;
-
+int32_t vnodeAlter(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg) {
   vDebug("vgId:%d, current dbCfgVersion:%d vgCfgVersion:%d, input dbCfgVersion:%d vgCfgVersion:%d", pVnode->vgId,
          pVnode->dbCfgVersion, pVnode->vgCfgVersion, pVnodeCfg->cfg.dbCfgVersion, pVnodeCfg->cfg.vgCfgVersion);
 
@@ -231,6 +307,7 @@ int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
 }
 
 static void vnodeFindWalRootDir(int32_t vgId, char *walRootDir) {
+#if 0  
   char vnodeDir[TSDB_FILENAME_LEN] = "\0";
   snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d/wal", vgId);
 
@@ -246,15 +323,17 @@ static void vnodeFindWalRootDir(int32_t vgId, char *walRootDir) {
   sprintf(walRootDir, "%s/vnode/vnode%d", TFS_DISK_PATH(tfile->level, tfile->id), vgId);
 
   tfsClosedir(tdir);
+#endif  
 }
 
 int32_t vnodeOpen(int32_t vgId) {
+#if 0  
   char temp[TSDB_FILENAME_LEN * 3];
   char rootDir[TSDB_FILENAME_LEN * 2];
   char walRootDir[TSDB_FILENAME_LEN * 2] = {0};
   snprintf(rootDir, TSDB_FILENAME_LEN * 2, "%s/vnode%d", tsVnodeDir, vgId);
 
-  SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1);
+  SVnode *pVnode = calloc(sizeof(SVnode), 1);
   if (pVnode == NULL) {
     vError("vgId:%d, failed to open vnode since no enough memory", vgId);
     return TAOS_SYSTEM_ERROR(errno);
@@ -262,9 +341,9 @@ int32_t vnodeOpen(int32_t vgId) {
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
 
-  pVnode->vgId     = vgId;
+  pVnode->vgId = vgId;
   pVnode->fversion = 0;
-  pVnode->version  = 0;  
+  pVnode->version = 0;
   pVnode->tsdbCfg.tsdbId = pVnode->vgId;
   pVnode->rootDir = strdup(rootDir);
   pVnode->accessState = TSDB_VN_ALL_ACCCESS;
@@ -279,7 +358,7 @@ int32_t vnodeOpen(int32_t vgId) {
     vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId);
     vnodeCleanUp(pVnode);
     return 0;
-  } 
+  }
 
   code = vnodeReadVersion(pVnode);
   if (code != TSDB_CODE_SUCCESS) {
@@ -291,29 +370,15 @@ int32_t vnodeOpen(int32_t vgId) {
   }
 
   pVnode->fversion = pVnode->version;
-  
-  pVnode->wqueue = dnodeAllocVWriteQueue(pVnode);
-  pVnode->qqueue = dnodeAllocVQueryQueue(pVnode);
-  pVnode->fqueue = dnodeAllocVFetchQueue(pVnode);
+
+  pVnode->wqueue = vnodeAllocWriteQueue(pVnode);
+  pVnode->qqueue = vnodeAllocQueryQueue(pVnode);
+  pVnode->fqueue = vnodeAllocFetchQueue(pVnode);
   if (pVnode->wqueue == NULL || pVnode->qqueue == NULL || pVnode->fqueue == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
   }
 
-  if (tsEnableStream) {
-    SCqCfg cqCfg = {0};
-    sprintf(cqCfg.user, "_root");
-    strcpy(cqCfg.pass, tsInternalPass);
-    strcpy(cqCfg.db, pVnode->db);
-    cqCfg.vgId = vgId;
-    cqCfg.cqWrite = vnodeWriteToCache;
-    pVnode->cq = cqOpen(pVnode, &cqCfg);
-    if (pVnode->cq == NULL) {
-      vnodeCleanUp(pVnode);
-      return terrno;
-    }
-  }
-
   STsdbAppH appH = {0};
   appH.appH = (void *)pVnode;
   appH.notifyStatus = vnodeProcessTsdbStatus;
@@ -327,8 +392,8 @@ int32_t vnodeOpen(int32_t vgId) {
     vnodeCleanUp(pVnode);
     return terrno;
   } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) {
-    vError("vgId:%d, failed to open tsdb(state: %d), replica:%d reason:%s", pVnode->vgId,
-           tsdbGetState(pVnode->tsdb), pVnode->syncCfg.replica, tstrerror(terrno));
+    vError("vgId:%d, failed to open tsdb(state: %d), replica:%d reason:%s", pVnode->vgId, tsdbGetState(pVnode->tsdb),
+           pVnode->syncCfg.replica, tstrerror(terrno));
     if (pVnode->syncCfg.replica <= 1) {
       vnodeCleanUp(pVnode);
       return TSDB_CODE_VND_INVALID_TSDB_STATE;
@@ -355,12 +420,12 @@ int32_t vnodeOpen(int32_t vgId) {
   sprintf(temp, "%s/wal", walRootDir);
   pVnode->walCfg.vgId = pVnode->vgId;
   pVnode->wal = walOpen(temp, &pVnode->walCfg);
-  if (pVnode->wal == NULL) { 
+  if (pVnode->wal == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
   }
 
-  walRestore(pVnode->wal, pVnode, vnodeProcessWrite);
+  walRestore(pVnode->wal, pVnode, (FWalWrite)vnodeProcessWalMsg);
   if (pVnode->version == 0) {
     pVnode->fversion = 0;
     pVnode->version = walGetVersion(pVnode->wal);
@@ -386,40 +451,16 @@ int32_t vnodeOpen(int32_t vgId) {
 
   vDebug("vgId:%d, vnode is opened in %s - %s, pVnode:%p", pVnode->vgId, rootDir, walRootDir, pVnode);
 
-  vnodeAddIntoHash(pVnode);
-  
-  SSyncInfo syncInfo;
-  syncInfo.vgId = pVnode->vgId;
-  syncInfo.version = pVnode->version;
-  syncInfo.syncCfg = pVnode->syncCfg;
-  tstrncpy(syncInfo.path, walRootDir, TSDB_FILENAME_LEN);
-  syncInfo.getWalInfoFp = vnodeGetWalInfo;
-  syncInfo.writeToCacheFp = vnodeWriteToCache;
-  syncInfo.confirmForward = vnodeConfirmForard; 
-  syncInfo.notifyRoleFp = vnodeNotifyRole;
-  syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow;
-  syncInfo.startSyncFileFp = vnodeStartSyncFile;
-  syncInfo.stopSyncFileFp = vnodeStopSyncFile;
-  syncInfo.getVersionFp = vnodeGetVersion;
-  syncInfo.sendFileFp = tsdbSyncSend;
-  syncInfo.recvFileFp = tsdbSyncRecv;
-  syncInfo.pTsdb = pVnode->tsdb;
-  pVnode->sync = syncStart(&syncInfo);
-
-  if (pVnode->sync <= 0) {
-    vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
-           tstrerror(terrno));
-    vnodeRemoveFromHash(pVnode);
-    vnodeCleanUp(pVnode);
-    return terrno;
-  }
+  taosHashPut(tsVmain.hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnode *));
 
   vnodeSetReadyStatus(pVnode);
+  pVnode->role = TAOS_SYNC_ROLE_MASTER;
+#endif  
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t vnodeClose(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquireNotClose(vgId);
+  SVnode *pVnode = vnodeAcquireNotClose(vgId);
   if (pVnode == NULL) return 0;
   if (pVnode->dropped) {
     vnodeRelease(pVnode);
@@ -435,10 +476,11 @@ int32_t vnodeClose(int32_t vgId) {
   return 0;
 }
 
-void vnodeDestroy(SVnodeObj *pVnode) {
+void vnodeDestroy(SVnode *pVnode) {
+#if 0  
   int32_t code = 0;
   int32_t vgId = pVnode->vgId;
-  
+
   if (pVnode->qMgmt) {
     qCleanupQueryMgmt(pVnode->qMgmt);
     pVnode->qMgmt = NULL;
@@ -475,47 +517,49 @@ void vnodeDestroy(SVnodeObj *pVnode) {
   }
 
   if (pVnode->wqueue) {
-    dnodeFreeVWriteQueue(pVnode->wqueue);
+    vnodeFreeWriteQueue(pVnode->wqueue);
     pVnode->wqueue = NULL;
   }
 
   if (pVnode->qqueue) {
-    dnodeFreeVQueryQueue(pVnode->qqueue);
+    vnodeFreeQueryQueue(pVnode->qqueue);
     pVnode->qqueue = NULL;
   }
 
   if (pVnode->fqueue) {
-    dnodeFreeVFetchQueue(pVnode->fqueue);
+    vnodeFreeFetchQueue(pVnode->fqueue);
     pVnode->fqueue = NULL;
   }
 
   tfree(pVnode->rootDir);
 
   if (pVnode->dropped) {
-    char rootDir[TSDB_FILENAME_LEN] = {0};    
+    char rootDir[TSDB_FILENAME_LEN] = {0};
     char stagingDir[TSDB_FILENAME_LEN] = {0};
     sprintf(rootDir, "%s/vnode%d", "vnode", vgId);
     sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
 
     tfsRename(rootDir, stagingDir);
 
-    vnodeBackup(vgId);
+    vnodeProcessBackupTask(pVnode);
 
-    dnodeSendStatusMsgToMnode();
+    // dnodeSendStatusMsgToMnode();
   }
 
   tsem_destroy(&pVnode->sem);
   pthread_mutex_destroy(&pVnode->statusMutex);
   free(pVnode);
   tsdbDecCommitRef(vgId);
+#endif  
 }
 
-void vnodeCleanUp(SVnodeObj *pVnode) {
+void vnodeCleanUp(SVnode *pVnode) {
+#if 0  
   vDebug("vgId:%d, vnode will cleanup, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
 
   vnodeSetClosingStatus(pVnode);
 
-  vnodeRemoveFromHash(pVnode);
+  taosHashRemove(tsVmain.hash, &pVnode->vgId, sizeof(int32_t));
 
   // stop replication module
   if (pVnode->sync > 0) {
@@ -526,10 +570,12 @@ void vnodeCleanUp(SVnodeObj *pVnode) {
 
   vDebug("vgId:%d, vnode is cleaned, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
   vnodeRelease(pVnode);
+#endif  
 }
 
+#if 0
 static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
-  SVnodeObj *pVnode = arg;
+  SVnode *pVnode = arg;
 
   if (eno != TSDB_CODE_SUCCESS) {
     vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno),
@@ -561,9 +607,301 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
   }
 
   // timer thread callback
-  if(status == TSDB_STATUS_COMMIT_NOBLOCK) {
+  if (status == TSDB_STATUS_COMMIT_NOBLOCK) {
     qSolveCommitNoBlock(pVnode->tsdb, pVnode->qMgmt);
   }
 
   return 0;
 }
+#endif
+
+static void *vnodeOpenVnode(void *param) {
+  SOpenVnodeThread *pThread = param;
+
+  vDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+  setThreadName("vnodeOpenVnode");
+
+  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+    int32_t vgId = pThread->vnodeList[v];
+
+    char stepDesc[TSDB_STEP_DESC_LEN] = {0};
+    snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId,
+             tsVmain.openVnodes, tsVmain.totalVnodes);
+    // (*vnodeInst()->fp.ReportStartup)("open-vnodes", stepDesc);
+
+    if (vnodeOpen(vgId) < 0) {
+      vError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
+      pThread->failed++;
+    } else {
+      vDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex);
+      pThread->opened++;
+    }
+
+    atomic_add_fetch_32(&tsVmain.openVnodes, 1);
+  }
+
+  vDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+         pThread->failed);
+  return NULL;
+}
+
+static int32_t vnodeGetVnodeListFromDisk(int32_t vnodeList[], int32_t *numOfVnodes) {
+#if 0
+  DIR *dir = opendir(tsVnodeDir);
+  if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS;
+
+  *numOfVnodes = 0;
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+    if (de->d_type & DT_DIR) {
+      if (strncmp("vnode", de->d_name, 5) != 0) continue;
+      int32_t vnode = atoi(de->d_name + 5);
+      if (vnode == 0) continue;
+
+      (*numOfVnodes)++;
+
+      if (*numOfVnodes >= TSDB_MAX_VNODES) {
+        vError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
+        closedir(dir);
+        return TSDB_CODE_DND_TOO_MANY_VNODES;
+      } else {
+        vnodeList[*numOfVnodes - 1] = vnode;
+      }
+    }
+  }
+  closedir(dir);
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t vnodeOpenVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
+  int32_t numOfVnodes = 0;
+  int32_t status = vnodeGetVnodeListFromDisk(vnodeList, &numOfVnodes);
+
+  if (status != TSDB_CODE_SUCCESS) {
+    vInfo("failed to get vnode list from disk since code:%d", status);
+    return status;
+  }
+
+  tsVmain.totalVnodes = numOfVnodes;
+
+  int32_t threadNum = tsNumOfCores;
+  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+
+  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
+  for (int32_t t = 0; t < threadNum; ++t) {
+    threads[t].threadIndex = t;
+    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
+  }
+
+  for (int32_t v = 0; v < numOfVnodes; ++v) {
+    int32_t           t = v % threadNum;
+    SOpenVnodeThread *pThread = &threads[t];
+    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
+  }
+
+  vInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum == 0) continue;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+    if (pthread_create(&pThread->thread, &thAttr, vnodeOpenVnode, pThread) != 0) {
+      vError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+  }
+
+  int32_t openVnodes = 0;
+  int32_t failedVnodes = 0;
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
+      pthread_join(pThread->thread, NULL);
+    }
+    openVnodes += pThread->opened;
+    failedVnodes += pThread->failed;
+    free(pThread->vnodeList);
+  }
+
+  free(threads);
+  vInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes);
+
+  if (failedVnodes != 0) {
+    vError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes);
+    return -1;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
+  void *pIter = taosHashIterate(tsVmain.hash, NULL);
+  while (pIter) {
+    SVnode **pVnode = pIter;
+    if (*pVnode) {
+      (*numOfVnodes)++;
+      if (*numOfVnodes >= TSDB_MAX_VNODES) {
+        vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES);
+        continue;
+      } else {
+        vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
+      }
+    }
+
+    pIter = taosHashIterate(tsVmain.hash, pIter);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void vnodeCleanupVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
+  int32_t numOfVnodes = 0;
+
+  int32_t code = vnodeGetVnodeList(vnodeList, &numOfVnodes);
+
+  if (code != TSDB_CODE_SUCCESS) {
+    vInfo("failed to get dnode list since code %d", code);
+    return;
+  }
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    vnodeClose(vnodeList[i]);
+  }
+
+  vInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static void vnodeInitMsgFp() {
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessMgmtMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_TABLE] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_MD_DROP_STABLE] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessWriteMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessReadMsg;
+  tsVmain.msgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessReadMsg;
+}
+
+void vnodeProcessMsg(SRpcMsg *pMsg) {
+  if (tsVmain.msgFp[pMsg->msgType]) {
+    (*tsVmain.msgFp[pMsg->msgType])(pMsg);
+  } else {
+    assert(0);
+  }
+}
+
+int32_t vnodeInitMain() {
+  vnodeInitMsgFp();
+
+  tsVmain.timer = taosTmrInit(100, 200, 60000, "VND-TIMER");
+  if (tsVmain.timer == NULL) {
+    vError("failed to init vnode timer");
+    return -1;
+  }
+
+  tsVmain.hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (tsVmain.hash == NULL) {
+    taosTmrCleanUp(tsVmain.timer);
+    vError("failed to init vnode mgmt");
+    return -1;
+  }
+
+  vInfo("vnode main is initialized");
+  return vnodeOpenVnodes();
+}
+
+void vnodeCleanupMain() {
+  taosTmrCleanUp(tsVmain.timer);
+  tsVmain.timer = NULL;
+
+  vnodeCleanupVnodes();
+
+  taosHashCleanup(tsVmain.hash);
+  tsVmain.hash = NULL;
+}
+
+static void vnodeBuildVloadMsg(SVnode *pVnode, SStatusMsg *pStatus) {
+#if 0  
+  int64_t totalStorage = 0;
+  int64_t compStorage = 0;
+  int64_t pointsWritten = 0;
+
+  if (vnodeInClosingStatus(pVnode)) return;
+  if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
+
+  if (pVnode->tsdb) {
+    tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
+  }
+
+  SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
+  pLoad->vgId = htonl(pVnode->vgId);
+  pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion);
+  pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion);
+  pLoad->totalStorage = htobe64(totalStorage);
+  pLoad->compStorage = htobe64(compStorage);
+  pLoad->pointsWritten = htobe64(pointsWritten);
+  pLoad->vnodeVersion = htobe64(pVnode->version);
+  pLoad->status = pVnode->status;
+  pLoad->role = pVnode->role;
+  pLoad->replica = pVnode->syncCfg.replica;
+  pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0;
+#endif  
+}
+
+void vnodeGetStatus(struct SStatusMsg *pStatus) {
+  void *pIter = taosHashIterate(tsVmain.hash, NULL);
+  while (pIter) {
+    SVnode **pVnode = pIter;
+    if (*pVnode) {
+      vnodeBuildVloadMsg(*pVnode, pStatus);
+    }
+    pIter = taosHashIterate(tsVmain.hash, pIter);
+  }
+}
+
+void vnodeSetAccess(struct SVgroupAccess *pAccess, int32_t numOfVnodes) {
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    pAccess[i].vgId = htonl(pAccess[i].vgId);
+    SVnode *pVnode = vnodeAcquireNotClose(pAccess[i].vgId);
+    if (pVnode != NULL) {
+      pVnode->accessState = pAccess[i].accessState;
+      if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) {
+        vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState);
+      }
+      vnodeRelease(pVnode);
+    }
+  }
+}
+
+void vnodeBackup(int32_t vgId) {
+  char newDir[TSDB_FILENAME_LEN] = {0};
+  char stagingDir[TSDB_FILENAME_LEN] = {0};
+
+  sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
+  sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
+
+#if 0
+  if (tsEnableVnodeBak) {
+    tfsRmdir(newDir);
+    tfsRename(stagingDir, newDir);
+  } else {
+    vInfo("vgId:%d, vnode backup not enabled", vgId);
+
+    tfsRmdir(stagingDir);
+  }
+#endif  
+}

source/server/vnode/src/vnodeMemAllocator.c

@@ -11,4 +11,114 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
+ */
+
+#include "vnodeMemAllocator.h"
+
+#define VMA_IS_FULL(pvma) \
+  (((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
+
+static SVMANode *VMANodeNew(size_t size);
+static void      VMANodeFree(SVMANode *node);
+
+SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
+  SVnodeMemAllocator *pvma = NULL;
+
+  if (size < threshold) {
+    return NULL;
+  }
+
+  pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
+  if (pvma) {
+    pvma->full = false;
+    pvma->threshold = threshold;
+    pvma->ssize = ssize;
+    pvma->inuse = &(pvma->node);
+
+    pvma->inuse->prev = NULL;
+    pvma->inuse->tsize = size;
+    pvma->inuse->used = 0;
+  }
+
+  return pvma;
+}
+
+void VMADestroy(SVnodeMemAllocator *pvma) {
+  if (pvma) {
+    VMAReset(pvma);
+    free(pvma);
+  }
+}
+
+void VMAReset(SVnodeMemAllocator *pvma) {
+  while (pvma->inuse != &(pvma->node)) {
+    SVMANode *node = pvma->inuse;
+    pvma->inuse = node->prev;
+    VMANodeFree(node);
+  }
+
+  pvma->inuse->used = 0;
+  pvma->full = false;
+}
+
+void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
+  void * ptr = NULL;
+  size_t tsize = size + sizeof(size_t);
+
+  if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
+    SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
+    if (pNode == NULL) {
+      return NULL;
+    }
+
+    pNode->prev = pvma->inuse;
+    pvma->inuse = pNode;
+  }
+
+  ptr = pvma->inuse->data + pvma->inuse->used;
+  pvma->inuse->used += tsize;
+  *(size_t *)ptr = size;
+  ptr = POINTER_SHIFT(ptr, sizeof(size_t));
+
+  pvma->full = VMA_IS_FULL(pvma);
+
+  return ptr;
+}
+
+void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
+  if (ptr) {
+    size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
+    if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
+      pvma->inuse->used -= (size + sizeof(size_t));
+
+      if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
+        SVMANode *node = pvma->inuse;
+        pvma->inuse = node->prev;
+        VMANodeFree(node);
+      }
+
+      pvma->full = VMA_IS_FULL(pvma);
+    }
+  }
+}
+
+bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
+
+static SVMANode *VMANodeNew(size_t size) {
+  SVMANode *node = NULL;
+
+  node = (SVMANode *)malloc(sizeof(*node) + size);
+  if (node) {
+    node->prev = NULL;
+    node->tsize = size;
+    node->used = 0;
+  }
+
+  return node;
+}
+
+static void VMANodeFree(SVMANode *node) {
+  if (node) {
+    free(node);
+  }
+}

source/server/vnode/src/vnodeMgmt.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeMain.h"
+#include "vnodeMgmt.h"
+#include "vnodeMgmtMsg.h"
+
+typedef struct {
+  SRpcMsg rpcMsg;
+  char    pCont[];
+} SVnMgmtMsg;
+
+static struct {
+  SWorkerPool pool;
+  taos_queue  pQueue;
+  int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
+} tsVmgmt = {0};
+
+static int32_t vnodeProcessMgmtStart(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype) {
+  SRpcMsg *pMsg = &pMgmt->rpcMsg;
+  int32_t  msgType = pMsg->msgType;
+
+  if (tsVmgmt.msgFp[msgType]) {
+    vTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[msgType]);
+    return (*tsVmgmt.msgFp[msgType])(pMsg);
+  } else {
+    vError("msg:%p, ahandle:%p type:%s not processed since no handle", pMgmt, pMsg->ahandle, taosMsg[msgType]);
+    return TSDB_CODE_DND_MSG_NOT_PROCESSED;
+  }
+}
+
+static void vnodeSendMgmtEnd(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype, int32_t code) {
+  SRpcMsg *pMsg = &pMgmt->rpcMsg;
+  SRpcMsg  rsp = {0};
+
+  rsp.code = code;
+  vTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
+  if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) {
+    rsp.handle = pMsg->handle;
+    rsp.pCont = NULL;
+    rpcSendResponse(&rsp);
+  }
+
+  taosFreeQitem(pMsg);
+}
+
+static void vnodeInitMgmtReqFp() {
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessCreateVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = vnodeProcessAlterVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]   = vnodeProcessSyncVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE]= vnodeProcessCompactVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = vnodeProcessDropVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessAlterStreamReq;
+}
+
+static int32_t vnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t     size = sizeof(SVnMgmtMsg) + pMsg->contLen;
+  SVnMgmtMsg *pMgmt = taosAllocateQitem(size);
+  if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
+
+  pMgmt->rpcMsg = *pMsg;
+  pMgmt->rpcMsg.pCont = pMgmt->pCont;
+  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+  taosWriteQitem(tsVmgmt.pQueue, TAOS_QTYPE_RPC, pMgmt);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeProcessMgmtMsg(SRpcMsg *pMsg) {
+  int32_t code = vnodeWriteToMgmtQueue(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+int32_t vnodeInitMgmt() {
+  vnodeInitMgmtReqFp();
+
+  SWorkerPool *pPool = &tsVmgmt.pool;
+  pPool->name = "vmgmt";
+  pPool->startFp = (ProcessStartFp)vnodeProcessMgmtStart;
+  pPool->endFp = (ProcessEndFp)vnodeSendMgmtEnd;
+  pPool->min = 1;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  tsVmgmt.pQueue = tWorkerAllocQueue(pPool, NULL);
+
+  vInfo("vmgmt is initialized, max worker %d", pPool->max);
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeCleanupMgmt() {
+  tWorkerFreeQueue(&tsVmgmt.pool, tsVmgmt.pQueue);
+  tWorkerCleanup(&tsVmgmt.pool);
+  tsVmgmt.pQueue = NULL;
+  vInfo("vmgmt is closed");
+}

source/server/vnode/src/vnodeMgmtMsg.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeMain.h"
+#include "vnodeMgmtMsg.h"
+
+static SCreateVnodeMsg* vnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
+  pCreate->cfg.dbCfgVersion        = htonl(pCreate->cfg.dbCfgVersion);
+  pCreate->cfg.vgCfgVersion        = htonl(pCreate->cfg.vgCfgVersion);
+  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
+  pCreate->cfg.cacheBlockSize      = htonl(pCreate->cfg.cacheBlockSize);
+  pCreate->cfg.totalBlocks         = htonl(pCreate->cfg.totalBlocks);
+  pCreate->cfg.daysPerFile         = htonl(pCreate->cfg.daysPerFile);
+  pCreate->cfg.daysToKeep1         = htonl(pCreate->cfg.daysToKeep1);
+  pCreate->cfg.daysToKeep2         = htonl(pCreate->cfg.daysToKeep2);
+  pCreate->cfg.daysToKeep          = htonl(pCreate->cfg.daysToKeep);
+  pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
+  pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
+  pCreate->cfg.fsyncPeriod         = htonl(pCreate->cfg.fsyncPeriod);
+  pCreate->cfg.commitTime          = htonl(pCreate->cfg.commitTime);
+
+  for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) {
+    pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
+  }
+
+  return pCreate;
+}
+
+int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = vnodeParseVnodeMsg(rpcMsg);
+  SVnode *pVnode = vnodeAcquire(pCreate->cfg.vgId);
+  if (pVnode != NULL) {
+    vDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
+    vnodeRelease(pVnode);
+    return TSDB_CODE_SUCCESS;
+  } else {
+    vDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
+    return vnodeCreate(pCreate);
+  }
+}
+
+int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
+  SAlterVnodeMsg *pAlter = vnodeParseVnodeMsg(rpcMsg);
+
+  void *pVnode = vnodeAcquireNotClose(pAlter->cfg.vgId);
+  if (pVnode != NULL) {
+    vDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
+    int32_t code = vnodeAlter(pVnode, pAlter);
+    vnodeRelease(pVnode);
+    return code;
+  } else {
+    vInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    return TSDB_CODE_VND_INVALID_VGROUP_ID;
+  }
+}
+
+int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) {
+  SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont;
+  pSyncVnode->vgId = htonl(pSyncVnode->vgId);
+
+  return vnodeSync(pSyncVnode->vgId);
+}
+
+int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg) {
+  SCompactVnodeMsg *pCompactVnode = rpcMsg->pCont;
+  pCompactVnode->vgId = htonl(pCompactVnode->vgId);
+  return vnodeCompact(pCompactVnode->vgId);
+}
+
+int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
+  pDrop->vgId = htonl(pDrop->vgId);
+
+  return vnodeDrop(pDrop->vgId);
+}
+
+int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg) { return 0; }

source/server/vnode/src/vnodeRead.c

@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "tglobal.h"
+// #include "query.h"
+#include "vnodeMain.h"
+#include "vnodeRead.h"
+#include "vnodeReadMsg.h"
+#include "vnodeStatus.h"
+
+static struct {
+  SWorkerPool query;
+  SWorkerPool fetch;
+  int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SVnode *, struct SReadMsg *);
+} tsVread = {0};
+
+void vnodeStartRead(SVnode *pVnode) {}
+void vnodeStopRead(SVnode *pVnode) {}
+
+void vnodeWaitReadCompleted(SVnode *pVnode) {
+  while (pVnode->queuedRMsg > 0) {
+    vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg);
+    taosMsleep(10);
+  }
+}
+
+static int32_t vnodeWriteToRQueue(SVnode *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) {
+  if (pVnode->dropped) {
+    return TSDB_CODE_APP_NOT_READY;
+  }
+
+#if 0
+  if (!((pVnode->role == TAOS_SYNC_ROLE_MASTER) || (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE))) {
+    return TSDB_CODE_APP_NOT_READY;
+  }
+#endif  
+
+  if (!vnodeInReadyStatus(pVnode)) {
+    vDebug("vgId:%d, failed to write into vread queue, vnode status is %s", pVnode->vgId, vnodeStatus[pVnode->status]);
+    return TSDB_CODE_APP_NOT_READY;
+  }
+
+  int32_t   size = sizeof(SReadMsg) + contLen;
+  SReadMsg *pRead = taosAllocateQitem(size);
+  if (pRead == NULL) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  if (pRpcMsg != NULL) {
+    pRead->rpcHandle = pRpcMsg->handle;
+    pRead->rpcAhandle = pRpcMsg->ahandle;
+    pRead->msgType = pRpcMsg->msgType;
+    pRead->code = pRpcMsg->code;
+  }
+
+  if (contLen != 0) {
+    pRead->contLen = contLen;
+    memcpy(pRead->pCont, pCont, contLen);
+  } else {
+    pRead->qhandle = pCont;
+  }
+
+  pRead->qtype = qtype;
+
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  atomic_add_fetch_32(&pVnode->queuedRMsg, 1);
+
+  if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) {
+    return taosWriteQitem(pVnode->fqueue, qtype, pRead);
+  } else {
+    return taosWriteQitem(pVnode->qqueue, qtype, pRead);
+  }
+}
+
+static void vnodeFreeFromRQueue(SVnode *pVnode, SReadMsg *pRead) {
+  atomic_sub_fetch_32(&pVnode->queuedRMsg, 1);
+
+  taosFreeQitem(pRead);
+  vnodeRelease(pVnode);
+}
+
+int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle) {
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+  rpcMsg.ahandle = ahandle;
+
+  int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
+  if (code == TSDB_CODE_SUCCESS) {
+    vTrace("QInfo:%p add to vread queue for exec query", *qhandle);
+  }
+
+  return code;
+}
+
+void vnodeProcessReadMsg(SRpcMsg *pMsg) {
+  int32_t queuedMsgNum = 0;
+  int32_t leftLen = pMsg->contLen;
+  int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID;
+  char *  pCont = pMsg->pCont;
+
+  while (leftLen > 0) {
+    SMsgHead *pHead = (SMsgHead *)pCont;
+    pHead->vgId = htonl(pHead->vgId);
+    pHead->contLen = htonl(pHead->contLen);
+
+    assert(pHead->contLen > 0);
+    SVnode *pVnode = vnodeAcquireNotClose(pHead->vgId);
+    if (pVnode != NULL) {
+      code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+      if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
+      vnodeRelease(pVnode);
+    }
+
+    leftLen -= pHead->contLen;
+    pCont -= pHead->contLen;
+  }
+
+  if (queuedMsgNum == 0) {
+    SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rpcRsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+static void vnodeInitReadMsgFp() {
+  tsVread.msgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg;
+  tsVread.msgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg;
+}
+
+static int32_t vnodeProcessReadStart(SVnode *pVnode, SReadMsg *pRead, int32_t qtype) {
+  int32_t msgType = pRead->msgType;
+  if (tsVread.msgFp[msgType] == NULL) {
+    vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
+    return TSDB_CODE_VND_MSG_NOT_PROCESSED;
+  } else {
+    vTrace("msg:%p, app:%p type:%s will be processed", pRead, pRead->rpcAhandle, taosMsg[msgType]);
+  }
+
+  return (*tsVread.msgFp[msgType])(pVnode, pRead);
+}
+
+static void vnodeSendReadRsp(SReadMsg *pRead, int32_t code) {
+  SRpcMsg rpcRsp = {
+      .handle = pRead->rpcHandle,
+      .pCont = pRead->rspRet.rsp,
+      .contLen = pRead->rspRet.len,
+      .code = code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+}
+
+static void vnodeProcessReadEnd(SVnode *pVnode, SReadMsg *pRead, int32_t qtype, int32_t code) {
+  if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
+    vnodeSendReadRsp(pRead, code);
+  } else {
+    if (code == TSDB_CODE_QRY_HAS_RSP) {
+      vnodeSendReadRsp(pRead, pRead->code);
+    } else {  // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
+      assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
+    }
+  }
+
+  vnodeFreeFromRQueue(pVnode, pRead);
+}
+
+int32_t vnodeInitRead() {
+  vnodeInitReadMsgFp();
+
+  int32_t maxFetchThreads = 4;
+  float   threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
+
+  SWorkerPool *pPool = &tsVread.query;
+  pPool->name = "vquery";
+  pPool->startFp = (ProcessStartFp)vnodeProcessReadStart;
+  pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd;
+  pPool->min = (int32_t)threadsForQuery;
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) return -1;
+
+  pPool = &tsVread.fetch;
+  pPool->name = "vfetch";
+  pPool->startFp = (ProcessStartFp)vnodeProcessReadStart;
+  pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd;
+  pPool->min = MIN(maxFetchThreads, tsNumOfCores);
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) return -1;
+
+  vInfo("vread is initialized, max worker %d", pPool->max);
+  return 0;
+}
+
+void vnodeCleanupRead() {
+  tWorkerCleanup(&tsVread.fetch);
+  tWorkerCleanup(&tsVread.query);
+  vInfo("vread is closed");
+}
+
+taos_queue vnodeAllocQueryQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.query, pVnode); }
+
+taos_queue vnodeAllocFetchQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.fetch, pVnode); }
+
+void vnodeFreeQueryQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.query, pQueue); }
+
+void vnodeFreeFetchQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.fetch, pQueue); }

source/server/vnode/src/vnodeReadMsg.c

@@ -16,155 +16,26 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
-#include "tqueue.h"
 #include "tglobal.h"
-#include "query.h"
+// #include "query.h"
 #include "vnodeStatus.h"
+#include "vnodeRead.h"
+#include "vnodeReadMsg.h"
 
-int32_t vNumOfExistedQHandle;   // current initialized and existed query handle in current dnode
+#if 0
+// notify connection(handle) that current qhandle is created, if current connection from
+// client is broken, the query needs to be killed immediately.
+static int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) {
+  SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
+  pMsg->qId = htobe64(qId);
+  pMsg->header.vgId = htonl(vgId);
+  pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
-static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
-static int32_t  vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
-static int32_t  vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
-
-static int32_t  vnodeNotifyCurrentQhandle(void* handle, uint64_t qId, void* qhandle, int32_t vgId);
-
-int32_t vnodeInitRead(void) {
-  vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg;
-  vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg;
-  return 0;
-}
-
-void vnodeCleanupRead() {}
-
-//
-// After the fetch request enters the vnode queue, if the vnode cannot provide services, the process function are
-// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
-// request enters the queue
-//
-int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) {
-  SVnodeObj *pVnode = vparam;
-  int32_t    msgType = pRead->msgType;
-
-  if (vnodeProcessReadMsgFp[msgType] == NULL) {
-    vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
-    return TSDB_CODE_VND_MSG_NOT_PROCESSED;
-  }
-
-  return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
-}
-
-static int32_t vnodeCheckRead(SVnodeObj *pVnode) {
-  if (!vnodeInReadyStatus(pVnode)) {
-    vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
-           pVnode->refCount, pVnode);
-    return TSDB_CODE_APP_NOT_READY;
-  }
-
-  // tsdb may be in reset state
-  if (pVnode->tsdb == NULL) {
-    vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-    return TSDB_CODE_APP_NOT_READY;
-  }
-
-  if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica,
-         syncRole[pVnode->role], pVnode->refCount, pVnode);
-  return TSDB_CODE_APP_NOT_READY;
-}
-
-void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) {
-  SVnodeObj *pVnode = vparam;
-
-  atomic_sub_fetch_32(&pVnode->queuedRMsg, 1);
-  vTrace("vgId:%d, free from vrqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedRMsg);
-
-  taosFreeQitem(pRead);
-  vnodeRelease(pVnode);
-}
-
-static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) {
-  int32_t size = sizeof(SVReadMsg) + contLen;
-  SVReadMsg *pRead = taosAllocateQitem(size);
-  if (pRead == NULL) {
-    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  if (pRpcMsg != NULL) {
-    pRead->rpcHandle = pRpcMsg->handle;
-    pRead->rpcAhandle = pRpcMsg->ahandle;
-    pRead->msgType = pRpcMsg->msgType;
-    pRead->code = pRpcMsg->code;
-  }
-
-  if (contLen != 0) {
-    pRead->contLen = contLen;
-    memcpy(pRead->pCont, pCont, contLen);
-  } else {
-    pRead->qhandle = pCont;
-  }
-
-  pRead->qtype = qtype;
-  atomic_add_fetch_32(&pVnode->refCount, 1);
-
-  return pRead;
-}
-
-int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) {
-  SVnodeObj *pVnode = vparam;
-  if (pVnode->dropped) {
-    return TSDB_CODE_APP_NOT_READY;
-  }
-
-  SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam);
-  if (pRead == NULL) {
-    assert(terrno != 0);
-    return terrno;
-  }
-
-  int32_t code = vnodeCheckRead(pVnode);
-  if (code != TSDB_CODE_SUCCESS) {
-    taosFreeQitem(pRead);
-    vnodeRelease(pVnode);
-    return code;
-  }
-
-  atomic_add_fetch_32(&pVnode->queuedRMsg, 1);
-
-  if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) {
-    vTrace("vgId:%d, write into vfetch queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount,
-           pVnode->queuedRMsg);
-    return taosWriteQitem(pVnode->fqueue, qtype, pRead);
-  } else {
-    vTrace("vgId:%d, write into vquery queue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount,
-           pVnode->queuedRMsg);
-    return taosWriteQitem(pVnode->qqueue, qtype, pRead);
-  }
-}
-
-static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) {
-  SRpcMsg rpcMsg = {0};
-  rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
-  rpcMsg.ahandle = ahandle;
-
-  int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
-  if (code == TSDB_CODE_SUCCESS) {
-    vTrace("QInfo:%p add to vread queue for exec query", *qhandle);
-  }
-
-  return code;
+  vTrace("QInfo:0x%" PRIx64 "-%p register qhandle to connect:%p", qId, qhandle, handle);
+  return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg));
 }
 
 /**
- *
  * @param pRet         response message object
  * @param pVnode       the vnode object
  * @param handle       qhandle for executing query
@@ -172,14 +43,16 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void
  * @param ahandle      sqlObj address at client side
  * @return
  */
-static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, void **handle, bool *freeHandle, void *ahandle) {
+static int32_t vnodeDumpQueryResult(SVnRsp *pRet, void *pVnode, uint64_t qId, void **handle, bool *freeHandle,
+                                    void *ahandle) {
   bool continueExec = false;
 
   int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) {
+  if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) ==
+      TSDB_CODE_SUCCESS) {
     if (continueExec) {
       *freeHandle = false;
-      code = vnodePutItemIntoReadQueue(pVnode, handle, ahandle);
+      code = vnodeReputPutToRQueue(pVnode, handle, ahandle);
       if (code != TSDB_CODE_SUCCESS) {
         *freeHandle = true;
         return code;
@@ -188,7 +61,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, v
       }
     } else {
       *freeHandle = true;
-      vTrace("QInfo:0x%"PRIx64"-%p exec completed, free handle:%d", qId, *handle, *freeHandle);
+      vTrace("QInfo:0x%" PRIx64 "-%p exec completed, free handle:%d", qId, *handle, *freeHandle);
     }
   } else {
     SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@@ -203,7 +76,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, v
   return code;
 }
 
-static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
+static void vnodeBuildNoResultQueryRsp(SVnRsp *pRet) {
   pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
   pRet->len = sizeof(SRetrieveTableRsp);
 
@@ -212,15 +85,16 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
 
   pRsp->completed = true;
 }
+#endif
 
-
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+int32_t vnodeProcessQueryMsg(SVnode *pVnode, SReadMsg *pRead) {
+#if 0  
   void *   pCont = pRead->pCont;
   int32_t  contLen = pRead->contLen;
-  SRspRet *pRet = &pRead->rspRet;
+  SVnRsp *pRet = &pRead->rspRet;
 
   SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
-  memset(pRet, 0, sizeof(SRspRet));
+  memset(pRet, 0, sizeof(SVnRsp));
 
   // qHandle needs to be freed correctly
   if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
@@ -231,13 +105,13 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
   void ** handle = NULL;
 
   if (contLen != 0) {
-    qinfo_t pQInfo = NULL;
+    qinfo_t  pQInfo = NULL;
     uint64_t qId = genQueryId();
     code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId);
 
     SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code = code;
-    pRsp->qId  = 0;
+    pRsp->qId = 0;
 
     pRet->len = sizeof(SQueryTableRsp);
     pRet->rsp = pRsp;
@@ -250,8 +124,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
         pRsp->code = terrno;
         terrno = 0;
 
-        vError("vgId:%d, QInfo:0x%"PRIx64 "-%p register qhandle failed, return to app, code:%s,", pVnode->vgId, qId, (void *)pQInfo,
-               tstrerror(pRsp->code));
+        vError("vgId:%d, QInfo:0x%" PRIx64 "-%p register qhandle failed, return to app, code:%s,", pVnode->vgId, qId,
+               (void *)pQInfo, tstrerror(pRsp->code));
         qDestroyQueryInfo(pQInfo);  // destroy it directly
         return pRsp->code;
       } else {
@@ -261,7 +135,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
 
       if (handle != NULL &&
           vnodeNotifyCurrentQhandle(pRead->rpcHandle, qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-        vError("vgId:%d, QInfo:0x%"PRIx64 "-%p, query discarded since link is broken, %p", pVnode->vgId, qId, *handle,
+        vError("vgId:%d, QInfo:0x%" PRIx64 "-%p, query discarded since link is broken, %p", pVnode->vgId, qId, *handle,
                pRead->rpcHandle);
 
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
@@ -274,8 +148,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
     }
 
     if (handle != NULL) {
-      vTrace("vgId:%d, QInfo:0x%"PRIx64 "-%p, dnode query msg disposed, create qhandle and returns to app", vgId, qId, *handle);
-      code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
+      vTrace("vgId:%d, QInfo:0x%" PRIx64 "-%p, query msg disposed, create qhandle and returns to app", vgId, qId,
+             *handle);
+      code = vnodeReputPutToRQueue(pVnode, handle, pRead->rpcHandle);
       if (code != TSDB_CODE_SUCCESS) {
         pRsp->code = code;
         qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -283,14 +158,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
       }
     }
 
-    int32_t remain = atomic_add_fetch_32(&vNumOfExistedQHandle, 1);
+    int32_t remain = atomic_add_fetch_32(&pVnode->numOfExistQHandle, 1);
     vTrace("vgId:%d, new qhandle created, total qhandle:%d", pVnode->vgId, remain);
   } else {
     assert(pCont != NULL);
-    void **qhandle = (void **)pRead->qhandle;
+    void **  qhandle = (void **)pRead->qhandle;
     uint64_t qId = 0;
 
-    vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
+    vTrace("vgId:%d, QInfo:%p, continues to exec query", pVnode->vgId, *qhandle);
 
     // In the retrieve blocking model, only 50% CPU will be used in query processing
     if (tsRetrieveBlockingModel) {
@@ -315,10 +190,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
         // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
         code = TSDB_CODE_QRY_HAS_RSP;
       } else {
-        //void *h1 = qGetResultRetrieveMsg(*qhandle);
+        // void *h1 = qGetResultRetrieveMsg(*qhandle);
 
-        /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */
-        //assert(h1 == NULL);
+        /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then
+         * FETCH thread execute twice before query thread reach here */
+        // assert(h1 == NULL);
 
         freehandle = qQueryCompleted(*qhandle);
       }
@@ -327,22 +203,24 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
       // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle
       if (freehandle || (!buildRes)) {
         if (freehandle) {
-          int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1);
+          int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1);
           vTrace("vgId:%d, QInfo:%p, start to free qhandle, remain qhandle:%d", pVnode->vgId, *qhandle, remain);
         }
 
         qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle);
       }
-
     }
   }
 
   return code;
+#endif
+  return 0;  
 }
 
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
-  void    *pCont = pRead->pCont;
-  SRspRet *pRet  = &pRead->rspRet;
+int32_t vnodeProcessFetchMsg(SVnode *pVnode, SReadMsg *pRead) {
+#if 0  
+  void *   pCont = pRead->pCont;
+  SVnRsp *pRet = &pRead->rspRet;
 
   SRetrieveTableMsg *pRetrieve = pCont;
   pRetrieve->free = htons(pRetrieve->free);
@@ -351,7 +229,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
   vTrace("vgId:%d, qId:0x%" PRIx64 ", retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, pRetrieve->qId,
          pRetrieve->free, pRead->rpcHandle);
 
-  memset(pRet, 0, sizeof(SRspRet));
+  memset(pRet, 0, sizeof(SVnRsp));
 
   terrno = TSDB_CODE_SUCCESS;
   int32_t code = TSDB_CODE_SUCCESS;
@@ -364,16 +242,17 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
   }
 
   if (code != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, invalid qId in retrieving result, code:%s, QInfo:%" PRIu64, pVnode->vgId, tstrerror(code), pRetrieve->qId);
+    vError("vgId:%d, invalid qId in retrieving result, code:%s, QInfo:%" PRIu64, pVnode->vgId, tstrerror(code),
+           pRetrieve->qId);
     vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
   // kill current query and free corresponding resources.
   if (pRetrieve->free == 1) {
-    int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1);
-    vWarn("vgId:%d, QInfo:%"PRIx64 "-%p, retrieve msg received to kill query and free qhandle, remain qhandle:%d", pVnode->vgId, pRetrieve->qId,
-        *handle, remain);
+    int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1);
+    vWarn("vgId:%d, QInfo:%" PRIx64 "-%p, retrieve msg received to kill query and free qhandle, remain qhandle:%d",
+          pVnode->vgId, pRetrieve->qId, *handle, remain);
 
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -385,9 +264,9 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
 
   // register the qhandle to connect to quit query immediate if connection is broken
   if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, pRetrieve->qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1);
-    vError("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve discarded since link is broken, conn:%p, remain qhandle:%d", pVnode->vgId, pRetrieve->qhandle,
-        *handle, pRead->rpcHandle, remain);
+    int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1);
+    vError("vgId:%d, QInfo:%" PRIu64 "-%p, retrieve discarded since link is broken, conn:%p, remain qhandle:%d",
+           pVnode->vgId, pRetrieve->qhandle, *handle, pRead->rpcHandle, remain);
 
     code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     qKillQuery(*handle);
@@ -422,29 +301,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
   // If qhandle is not added into vread queue, the query should be completed already or paused with error.
   // Here free qhandle immediately
   if (freeHandle) {
-    int32_t remain = atomic_sub_fetch_32(&vNumOfExistedQHandle, 1);
+    int32_t remain = atomic_sub_fetch_32(&pVnode->numOfExistQHandle, 1);
     vTrace("vgId:%d, QInfo:%p, start to free qhandle, remain qhandle:%d", pVnode->vgId, *handle, remain);
     qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
   }
 
   return code;
+#endif 
+  return 0;
 }
 
-// notify connection(handle) that current qhandle is created, if current connection from
-// client is broken, the query needs to be killed immediately.
-int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) {
-  SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
-  pMsg->qId = htobe64(qId);
-  pMsg->header.vgId = htonl(vgId);
-  pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
-
-  vTrace("QInfo:0x%"PRIx64"-%p register qhandle to connect:%p", qId, qhandle, handle);
-  return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg));
-}
-
-void vnodeWaitReadCompleted(SVnodeObj *pVnode) {
-  while (pVnode->queuedRMsg > 0) {
-    vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg);
-    taosMsleep(10);
-  }
-}

source/server/vnode/src/vnodeStatus.c

@@ -16,9 +16,9 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
-#include "query.h"
-#include "vnodeStatus.h"
+// #include "query.h"
 #include "vnodeRead.h"
+#include "vnodeStatus.h"
 #include "vnodeWrite.h"
 
 char* vnodeStatus[] = {
@@ -29,30 +29,32 @@ char* vnodeStatus[] = {
   "reset"
 };
 
-bool vnodeSetInitStatus(SVnodeObj* pVnode) {
+bool vnodeSetInitStatus(SVnode* pVnode) {
   pthread_mutex_lock(&pVnode->statusMutex);
   pVnode->status = TAOS_VN_STATUS_INIT;
   pthread_mutex_unlock(&pVnode->statusMutex);
   return true;
 }
 
-bool vnodeSetReadyStatus(SVnodeObj* pVnode) {
+bool vnodeSetReadyStatus(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
   if (pVnode->status == TAOS_VN_STATUS_INIT || pVnode->status == TAOS_VN_STATUS_READY ||
-      pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) {
+      pVnode->status == TAOS_VN_STATUS_UPDATING) {
     pVnode->status = TAOS_VN_STATUS_READY;
     set = true;
   }
 
+#if 0
   qQueryMgmtReOpen(pVnode->qMgmt);
+#endif   
 
   pthread_mutex_unlock(&pVnode->statusMutex);
   return set;
 }
 
-static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
+static bool vnodeSetClosingStatusImp(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -65,7 +67,7 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
   return set;
 }
 
-bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
+bool vnodeSetClosingStatus(SVnode* pVnode) {
   if (pVnode->status == TAOS_VN_STATUS_CLOSING)
     return true;
 
@@ -73,15 +75,17 @@ bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
     taosMsleep(1);
   }
 
+#if 0
   // release local resources only after cutting off outside connections
   qQueryMgmtNotifyClosed(pVnode->qMgmt);
+#endif   
   vnodeWaitReadCompleted(pVnode);
   vnodeWaitWriteCompleted(pVnode);
 
   return true;
 }
 
-bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
+bool vnodeSetUpdatingStatus(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -94,35 +98,7 @@ bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
   return set;
 }
 
-static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) {
-  bool set = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) {
-    pVnode->status = TAOS_VN_STATUS_RESET;
-    set = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return set;
-}
-
-bool vnodeSetResetStatus(SVnodeObj* pVnode) {
-  while (!vnodeSetResetStatusImp(pVnode)) {
-    taosMsleep(1);
-  }
-
-  vInfo("vgId:%d, set to reset status", pVnode->vgId);
-
-  // release local resources only after cutting off outside connections
-  qQueryMgmtNotifyClosed(pVnode->qMgmt);
-  vnodeWaitReadCompleted(pVnode);
-  vnodeWaitWriteCompleted(pVnode);
-
-  return true;
-}
-
-bool vnodeInInitStatus(SVnodeObj* pVnode) {
+bool vnodeInInitStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -134,7 +110,7 @@ bool vnodeInInitStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInReadyStatus(SVnodeObj* pVnode) {
+bool vnodeInReadyStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -146,19 +122,7 @@ bool vnodeInReadyStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) {
-  bool in = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_UPDATING) {
-    in = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return in;
-}
-
-bool vnodeInClosingStatus(SVnodeObj* pVnode) {
+bool vnodeInClosingStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -170,14 +134,3 @@ bool vnodeInClosingStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInResetStatus(SVnodeObj* pVnode) {
-  bool in = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_RESET) {
-    in = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return in;
-}

source/server/vnode/src/vnodeVersion.c

@@ -19,7 +19,7 @@
 #include "tglobal.h"
 #include "vnodeVersion.h"
 
-int32_t vnodeReadVersion(SVnodeObj *pVnode) {
+int32_t vnodeReadVersion(SVnode *pVnode) {
   int32_t len = 0;
   int32_t maxLen = 100;
   char *  content = calloc(1, maxLen + 1);
@@ -71,7 +71,7 @@ PARSE_VER_ERROR:
   return terrno;
 }
 
-int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
+int32_t vnodeSaveVersion(SVnode *pVnode) {
   char file[TSDB_FILENAME_LEN + 30] = {0};
   sprintf(file, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId);
 
@@ -90,7 +90,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
-  taosFsync(fileno(fp));
+  taosFsyncFile(fileno(fp));
   fclose(fp);
   free(content);
   terrno = 0;

source/server/vnode/src/vnodeWorker.c

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeMain.h"
+#include "vnodeWorker.h"
+
+enum { CLEANUP_TASK = 0, DESTROY_TASK = 1, BACKUP_TASK = 2 };
+
+typedef struct {
+  int32_t vgId;
+  int32_t code;
+  int32_t type;
+  void *  rpcHandle;
+  SVnode *pVnode;
+} SVnTask;
+
+static struct {
+  SWorkerPool pool;
+  taos_queue  pQueue;
+} tsVworker = {0};
+
+static void vnodeProcessTaskStart(void *unused, SVnTask *pTask, int32_t qtype) {
+  pTask->code = 0;
+
+  switch (pTask->type) {
+    case CLEANUP_TASK:
+      vnodeCleanUp(pTask->pVnode);
+      break;
+    case DESTROY_TASK:
+      vnodeDestroy(pTask->pVnode);
+      break;
+    case BACKUP_TASK:
+      vnodeBackup(pTask->vgId);
+      break;
+    default:
+      break;
+  }
+}
+
+static void vnodeProcessTaskEnd(void *unused, SVnTask *pTask, int32_t qtype, int32_t code) {
+  if (pTask->rpcHandle != NULL) {
+    SRpcMsg rpcRsp = {.handle = pTask->rpcHandle, .code = pTask->code};
+    rpcSendResponse(&rpcRsp);
+  }
+
+  taosFreeQitem(pTask);
+}
+
+static int32_t vnodeWriteIntoTaskQueue(SVnode *pVnode, int32_t type, void *rpcHandle) {
+  SVnTask *pTask = taosAllocateQitem(sizeof(SVnTask));
+  if (pTask == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
+
+  pTask->vgId = pVnode->vgId;
+  pTask->pVnode = pVnode;
+  pTask->rpcHandle = rpcHandle;
+  pTask->type = type;
+
+
+  return taosWriteQitem(tsVworker.pQueue, TAOS_QTYPE_RPC, pTask);
+}
+
+void vnodeProcessCleanupTask(SVnode *pVnode) {
+  vnodeWriteIntoTaskQueue(pVnode, CLEANUP_TASK, NULL);
+}
+
+void vnodeProcessDestroyTask(SVnode *pVnode) {
+  vnodeWriteIntoTaskQueue(pVnode, DESTROY_TASK, NULL);
+}
+
+void vnodeProcessBackupTask(SVnode *pVnode) {
+  vnodeWriteIntoTaskQueue(pVnode, BACKUP_TASK, NULL);
+}
+
+int32_t vnodeInitWorker() {
+  SWorkerPool *pPool = &tsVworker.pool;
+  pPool->name = "vworker";
+  pPool->startFp = (ProcessStartFp)vnodeProcessTaskStart;
+  pPool->endFp = (ProcessEndFp)vnodeProcessTaskEnd;
+  pPool->min = 0;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  tsVworker.pQueue = tWorkerAllocQueue(pPool, NULL);
+
+  vInfo("vworker is initialized, max worker %d", pPool->max);
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeCleanupWorker() {
+  tWorkerFreeQueue(&tsVworker.pool, tsVworker.pQueue);
+  tWorkerCleanup(&tsVworker.pool);
+  tsVworker.pQueue = NULL;
+  vInfo("vworker is closed");
+}

source/server/vnode/src/vnodeWrite.c

@@ -13,55 +13,224 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tqueue.h"
+#include "tworker.h"
+#include "taosmsg.h"
+#include "vnodeMain.h"
+#include "vnodeStatus.h"
+#include "vnodeWrite.h"
+#include "vnodeWriteMsg.h"
 
-int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) {
-  // TODO: Check inputs
+typedef int32_t (*WriteMsgFp)(SVnode *, void *pCont, SVnRsp *);
 
+typedef struct {
+  int32_t  code;
+  int8_t   qtype;
+  SVnode * pVnode;
+  SRpcMsg  rpcMsg;
+  SVnRsp  rspRet;
+  char     reserveForSync[24];
+  SWalHead walHead;
+} SVnWriteMsg;
+
+static struct {
+  SWriteWorkerPool pool;
+  int64_t          queuedBytes;
+  int32_t          queuedMsgs;
+} tsVwrite = {0};
+
+void vnodeStartWrite(SVnode *pVnode) {}
+void vnodeStoprite(SVnode *pVnode) {}
+
+void vnodeWaitWriteCompleted(SVnode *pVnode) {
+  while (pVnode->queuedWMsg > 0) {
+    vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg);
+    taosMsleep(10);
+  }
+}
+
+static int32_t vnodeWriteToWQueue(SVnode *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) {
+  if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
+    vWarn("vgId:%d, no write auth", pVnode->vgId);
+    return TSDB_CODE_VND_NO_WRITE_AUTH;
+  }
+
+  if (tsAvailDataDirGB <= tsMinimalDataDirGB) {
+    vWarn("vgId:%d, failed to write into vwqueue since no diskspace, avail:%fGB", pVnode->vgId, tsAvailDataDirGB);
+    return TSDB_CODE_VND_NO_DISKSPACE;
+  }
+
+  if (pHead->len > TSDB_MAX_WAL_SIZE) {
+    vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version);
+    return TSDB_CODE_WAL_SIZE_LIMIT;
+  }
+
+  if (!vnodeInReadyStatus(pVnode)) {
+    vError("vgId:%d, failed to write into vwqueue, vstatus is %s", pVnode->vgId, vnodeStatus[pVnode->status]);
+    return TSDB_CODE_APP_NOT_READY;
+  }
+
+  if (tsVwrite.queuedBytes > tsMaxVnodeQueuedBytes) {
+    vDebug("vgId:%d, too many bytes:%" PRId64 " in vwqueue, flow control", pVnode->vgId, tsVwrite.queuedBytes);
+    return TSDB_CODE_VND_IS_FLOWCTRL;
+  }
+
+  int32_t      size = sizeof(SVnWriteMsg) + pHead->len;
+  SVnWriteMsg *pWrite = taosAllocateQitem(size);
+  if (pWrite == NULL) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  if (pRpcMsg != NULL) {
+    pWrite->rpcMsg = *pRpcMsg;
+  }
+
+  memcpy(&pWrite->walHead, pHead, sizeof(SWalHead) + pHead->len);
+  pWrite->pVnode = pVnode;
+  pWrite->qtype = qtype;
+
+  atomic_add_fetch_64(&tsVwrite.queuedBytes, size);
+  atomic_add_fetch_32(&tsVwrite.queuedMsgs, 1);
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  atomic_add_fetch_32(&pVnode->queuedWMsg, 1);
+  taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void vnodeFreeFromWQueue(SVnode *pVnode, SVnWriteMsg *pWrite) {
+  int64_t size = sizeof(SVnWriteMsg) + pWrite->walHead.len;
+  atomic_sub_fetch_64(&tsVwrite.queuedBytes, size);
+  atomic_sub_fetch_32(&tsVwrite.queuedMsgs, 1);
+  atomic_sub_fetch_32(&pVnode->queuedWMsg, 1);
+
+  taosFreeQitem(pWrite);
+  vnodeRelease(pVnode);
+}
+
+int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead) {
+  return vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_WAL, NULL);
+}
+
+void vnodeProcessWriteMsg(SRpcMsg *pRpcMsg) {
+  int32_t code;
+
+  SMsgHead *pMsg = pRpcMsg->pCont;
+  pMsg->vgId = htonl(pMsg->vgId);
+  pMsg->contLen = htonl(pMsg->contLen);
+
+  SVnode *pVnode = vnodeAcquireNotClose(pMsg->vgId);
+  if (pVnode == NULL) {
+    code = TSDB_CODE_VND_INVALID_VGROUP_ID;
+  } else {
+    SWalHead *pHead = (SWalHead *)((char *)pRpcMsg->pCont - sizeof(SWalHead));
+    pHead->msgType = pRpcMsg->msgType;
+    pHead->version = 0;
+    pHead->len = pMsg->contLen;
+    code = vnodeWriteToWQueue(pVnode, pHead, TAOS_QTYPE_RPC, pRpcMsg);
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rpcRsp = {.handle = pRpcMsg->handle, .code = code};
+    rpcSendResponse(&rpcRsp);
+  }
+
+  vnodeRelease(pVnode);
+  rpcFreeCont(pRpcMsg->pCont);
+}
+
+static bool vnodeProcessWriteStart(SVnode *pVnode, SVnWriteMsg *pWrite, int32_t qtype) {
+  SWalHead *pHead = &pWrite->walHead;
+  SVnRsp * pRet = &pWrite->rspRet;
+  int32_t   msgType = pHead->msgType;
+
+  vTrace("vgId:%d, msg:%s will be processed, hver:%" PRIu64, pVnode->vgId, taosMsg[pHead->msgType], pHead->version);
+
+  // write into WAL
+#if 0  
+  pWrite->code = walWrite(pVnode->wal, pHead);
+  if (pWrite->code < 0) return false;
+#endif  
+
+  pVnode->version = pHead->version;
+
+  // write data locally
+  switch (msgType) {
+    case TSDB_MSG_TYPE_SUBMIT:
+      pRet->len = sizeof(SSubmitRsp);
+      pRet->rsp = rpcMallocCont(pRet->len);
+      pWrite->code = vnodeProcessSubmitReq(pVnode, (void*)pHead->cont, pRet->rsp);
+      break;
+    case TSDB_MSG_TYPE_MD_CREATE_TABLE:
+      pWrite->code = vnodeProcessCreateTableReq(pVnode, (void*)pHead->cont, NULL);
+      break;
+    case TSDB_MSG_TYPE_MD_DROP_TABLE:
+      pWrite->code = vnodeProcessDropTableReq(pVnode, (void*)pHead->cont, NULL);
+      break;
+    case TSDB_MSG_TYPE_MD_ALTER_TABLE:
+      pWrite->code = vnodeProcessAlterTableReq(pVnode, (void*)pHead->cont, NULL);
+      break;
+    case TSDB_MSG_TYPE_MD_DROP_STABLE:
+      pWrite->code = vnodeProcessDropStableReq(pVnode, (void*)pHead->cont, NULL);
+      break;
+    case TSDB_MSG_TYPE_UPDATE_TAG_VAL:
+      pWrite->code = vnodeProcessUpdateTagValReq(pVnode, (void*)pHead->cont, NULL);
+      break;
+    default:
+      pWrite->code = TSDB_CODE_VND_MSG_NOT_PROCESSED;
+      break;
+  }
+
+  if (pWrite->code < 0) return false;
+
+  // update fync
+  return (pWrite->code == 0 && msgType != TSDB_MSG_TYPE_SUBMIT);
+}
+
+static void vnodeFsync(SVnode *pVnode, bool fsync) { 
 #if 0
-  void *pMem = NULL;
-  if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) {
-    // No more memory to allocate, schedule an async commit
-    // and continue
-    vnodeAsyncCommit(pVnode);
+  walFsync(pVnode->wal, fsync); 
+#endif
+}
 
-    // Reset allocator and allocat more
-    vnodeResetAllocator(pVnode);
-    pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq));
-    if (pMem == NULL) {
-      // TODO: handle the error
+static void vnodeProcessWriteEnd(SVnode *pVnode, SVnWriteMsg *pWrite, int32_t qtype, int32_t code) {
+  if (qtype == TAOS_QTYPE_RPC) {
+    SRpcMsg rpcRsp = {
+        .handle = pWrite->rpcMsg.handle,
+        .pCont = pWrite->rspRet.rsp,
+        .contLen = pWrite->rspRet.len,
+        .code = pWrite->code,
+    };
+    rpcSendResponse(&rpcRsp);
+  } else {
+    if (pWrite->rspRet.rsp) {
+      rpcFreeCont(pWrite->rspRet.rsp);
     }
   }
-
-  // TODO: if SSubmitReq is compressed or encoded, we need to decode the request
-  memcpy(pMem, pReq, REQ_SIZE(pReq));
-
-  if (tqPushMsg((SSubmitReq *)pReq) < 0) {
-    // TODO: handle error
-  }
-
-  SSubmitReqReader reader;
-  taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem);
-
-  if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) {
-    // TODO: handler error
-  }
-#endif
-
-  return 0;
+  vnodeFreeFromWQueue(pVnode, pWrite);
 }
 
-int vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) {
-  // TODO
-  return 0;
+int32_t vnodeInitWrite() {
+  SWriteWorkerPool *pPool = &tsVwrite.pool;
+  pPool->name = "vwrite";
+  pPool->max = tsNumOfCores;
+  pPool->startFp = (ProcessWriteStartFp)vnodeProcessWriteStart;
+  pPool->syncFp = (ProcessWriteSyncFp)vnodeFsync;
+  pPool->endFp = (ProcessWriteEndFp)vnodeProcessWriteEnd;
+  if (tWriteWorkerInit(pPool) != 0) return -1;
+
+  vInfo("vwrite is initialized, max worker %d", pPool->max);
+  return TSDB_CODE_SUCCESS;
 }
 
-int vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) {
-  // TODO
-  return 0;
+void vnodeCleanupWrite() {
+  tWriteWorkerCleanup(&tsVwrite.pool);
+  vInfo("vwrite is closed");
 }
 
-int vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) {
-  // TODO
-  return 0;
-}
+taos_queue vnodeAllocWriteQueue(SVnode *pVnode) { return tWriteWorkerAllocQueue(&tsVwrite.pool, pVnode); }
+
+void vnodeFreeWriteQueue(taos_queue pQueue) { tWriteWorkerFreeQueue(&tsVwrite.pool, pQueue); }

source/server/vnode/src/vnodeWriteMsg.c

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeWriteMsg.h"
+
+int32_t vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp) {
+  // TODO: Check inputs
+
+#if 0
+  void *pMem = NULL;
+  if ((pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq))) == NULL) {
+    // No more memory to allocate, schedule an async commit
+    // and continue
+    vnodeAsyncCommit(pVnode);
+
+    // Reset allocator and allocat more
+    vnodeResetAllocator(pVnode);
+    pMem = amalloc(pVnode->allocator, REQ_SIZE(pReq));
+    if (pMem == NULL) {
+      // TODO: handle the error
+    }
+  }
+
+  // TODO: if SSubmitReq is compressed or encoded, we need to decode the request
+  memcpy(pMem, pReq, REQ_SIZE(pReq));
+
+  if (tqPushMsg((SSubmitReq *)pReq) < 0) {
+    // TODO: handle error
+  }
+
+  SSubmitReqReader reader;
+  taosInitSubmitReqReader(&reader, (SSubmitReq *)pMem);
+
+  if (tsdbInsert(pVnode->pTsdb, (SSubmitReq *)pMem) < 0) {
+    // TODO: handler error
+  }
+#endif
+
+  return 0;
+}
+
+int32_t vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+ int32_t vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+int32_t vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+int32_t vnodeProcessDropStableReq(SVnode *pVnode, SDropStableReq *pReq, SDropStableRsp *pRsp) {
+  // TODO
+  return 0;
+}
+
+int32_t vnodeProcessUpdateTagValReq(SVnode *pVnode, SUpdateTagValReq *pReq, SUpdateTagValRsp *pRsp) {
+  // TODO
+  return 0;
+}

source/server/vnode/test/CMakeLists.txt

@@ -0,0 +1,9 @@
+# vnodeMemAllocatorTest
+add_executable(VMATest "")
+target_sources(VMATest
+    PRIVATE
+    "../src/vnodeMemAllocator.c"
+    "vnodeMemAllocatorTest.cpp"
+)
+target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
+target_link_libraries(VMATest os gtest_main)

source/server/vnode/test/vnodeMemAllocatorTest.cpp

@@ -0,0 +1,22 @@
+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "vnodeMemAllocator.h"
+
+TEST(VMATest, basic_create_and_destroy_test) {
+  SVnodeMemAllocator *vma = VMACreate(1024, 512, 64);
+  EXPECT_TRUE(vma != nullptr);
+  EXPECT_EQ(vma->full, false);
+  EXPECT_EQ(vma->ssize, 512);
+  EXPECT_EQ(vma->threshold, 64);
+  EXPECT_EQ(vma->inuse->tsize, 1024);
+  VMADestroy(vma);
+
+  vma = VMACreate(1024, 512, 1024);
+  EXPECT_TRUE(vma != nullptr);
+  VMADestroy(vma);
+
+  vma = VMACreate(1024, 512, 1025);
+  EXPECT_TRUE(vma == nullptr);
+  VMADestroy(vma);
+}

source/server/vnode/tq/inc/tqInt.h

@@ -25,6 +25,17 @@ extern "C" {
 //implement the array index
 //implement the ring buffer
 
+//create persistent storage for meta info such as consuming offset
+//return value > 0: cgId
+//return value <= 0: error code
+int tqCreateGroup(STQ*);
+//create ring buffer in memory and load consuming offset
+int tqOpenGroup(STQ*, int cgId);
+//destroy ring buffer and persist consuming offset
+int tqCloseGroup(STQ*, int cgId);
+//delete persistent storage for meta info
+int tqDropGroup(STQ*, int cgId);
+
 #ifdef __cplusplus
 }
 #endif

source/server/vnode/tsdb/src/tsdb.c

@@ -15,5 +15,5 @@
 
 #include "tsdb.h"
 
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; }
+int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; }
 int tsdbCommit(STsdb *pTsdb) { return 0; }

source/util/CMakeLists.txt

@@ -1,5 +1,5 @@
 aux_source_directory(src UTIL_SRC)
-add_library(util ${UTIL_SRC})
+add_library(util STATIC ${UTIL_SRC})
 target_include_directories(
     util
     PUBLIC "${CMAKE_SOURCE_DIR}/include/util"

source/util/src/compare.c

@@ -0,0 +1,398 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#define _BSD_SOURCE
+#define _GNU_SOURCE
+#define _XOPEN_SOURCE
+#define _DEFAULT_SOURCE
+
+#include "os.h"
+#include "types.h"
+#include "compare.h"
+#include "ulog.h"
+#include "thash.h"
+#include "regex.h"
+
+int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
+}
+
+int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
+}
+
+int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
+}
+
+int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
+}
+
+int32_t compareInt8Val(const void *pLeft, const void *pRight) {
+  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
+  return compareInt8Val(pRight, pLeft);
+}
+
+int32_t compareInt16Val(const void *pLeft, const void *pRight) {
+  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt16Val(pRight, pLeft);
+}
+
+int32_t compareInt32Val(const void *pLeft, const void *pRight) {
+  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt32Val(pRight, pLeft);
+}
+
+int32_t compareInt64Val(const void *pLeft, const void *pRight) {
+  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt64Val(pRight, pLeft);
+}
+
+int32_t compareUint32Val(const void *pLeft, const void *pRight) {
+  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint32Val(pRight, pLeft);
+}
+
+int32_t compareUint64Val(const void *pLeft, const void *pRight) {
+  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint64Val(pRight, pLeft);
+}
+
+int32_t compareUint16Val(const void *pLeft, const void *pRight) {
+  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint16Val(pRight, pLeft);
+}
+
+int32_t compareUint8Val(const void* pLeft, const void* pRight) {
+  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint8Val(pRight, pLeft);
+}
+
+int32_t compareFloatVal(const void *pLeft, const void *pRight) {
+  float p1 = GET_FLOAT_VAL(pLeft);
+  float p2 = GET_FLOAT_VAL(pRight);
+
+  if (isnan(p1) && isnan(p2)) {
+    return 0;
+  }
+
+  if (isnan(p1)) {
+    return -1;
+  }
+
+  if (isnan(p2)) {
+    return 1;
+  }
+  if (FLT_EQUAL(p1, p2)) {
+    return 0;
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
+  return compareFloatVal(pRight, pLeft);
+}
+
+int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
+  double p1 = GET_DOUBLE_VAL(pLeft);
+  double p2 = GET_DOUBLE_VAL(pRight);
+
+  if (isnan(p1) && isnan(p2)) {
+    return 0;
+  }
+
+  if (isnan(p1)) {
+    return -1;
+  }
+
+  if (isnan(p2)) {
+    return 1;
+  }
+  if (FLT_EQUAL(p1, p2)) {
+    return 0;
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
+  return compareDoubleVal(pRight, pLeft);
+}
+
+int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1:-1;
+    }
+  }
+}
+
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedStr(pRight, pLeft);
+}
+
+int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1 : -1;
+    }
+  }
+}
+
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedWStr(pRight, pLeft);
+}
+
+/*
+ * Compare two strings
+ *    TSDB_MATCH:            Match
+ *    TSDB_NOMATCH:          No match
+ *    TSDB_NOWILDCARDMATCH:  No match in spite of having * or % wildcards.
+ * Like matching rules:
+ *      '%': Matches zero or more characters
+ *      '_': Matches one character
+ *
+ */
+int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
+  char c, c1;
+
+  int32_t i = 0;
+  int32_t j = 0;
+  int32_t o = 0;
+  int32_t m = 0;
+
+  while ((c = patterStr[i++]) != 0) {
+    if (c == pInfo->matchAll) { /* Match "*" */
+
+      while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
+        if (c == pInfo->matchOne) {
+          if (j > size || str[j++] == 0) {
+            // empty string, return not match
+            return TSDB_PATTERN_NOWILDCARDMATCH;
+          } else {
+            ++o;
+          }
+        }
+      }
+
+      if (c == 0) {
+        return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
+      }
+
+      char next[3] = {toupper(c), tolower(c), 0};
+      m = o;
+      while (1) {
+        size_t n = strcspn(str + m, next);
+        str += m + n;
+
+        if (str[0] == 0 || (n >= size)) {
+          break;
+        }
+
+        int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
+        if (ret != TSDB_PATTERN_NOMATCH) {
+          return ret;
+        }
+        m = 0;
+      }
+      return TSDB_PATTERN_NOWILDCARDMATCH;
+    }
+
+    c1 = str[j++];
+    ++o; 
+    
+    if (j <= size) {
+      if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
+      if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
+        continue;
+      }
+    }
+
+    return TSDB_PATTERN_NOMATCH;
+  }
+
+  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
+}
+
+int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
+  wchar_t c, c1;
+  wchar_t matchOne = L'_';  // "_"
+  wchar_t matchAll = L'%';  // "%"
+
+  int32_t i = 0;
+  int32_t j = 0;
+
+  while ((c = patterStr[i++]) != 0) {
+    if (c == matchAll) { /* Match "%" */
+
+      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
+        if (c == matchOne && (j >= size || str[j++] == 0)) {
+          return TSDB_PATTERN_NOWILDCARDMATCH;
+        }
+      }
+      if (c == 0) {
+        return TSDB_PATTERN_MATCH;
+      }
+
+      wchar_t accept[3] = {towupper(c), towlower(c), 0};
+      while (1) {
+        size_t n = wcscspn(str, accept);
+
+        str += n;
+        if (str[0] == 0 || (n >= size)) {
+          break;
+        }
+
+        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
+        if (ret != TSDB_PATTERN_NOMATCH) {
+          return ret;
+        }
+      }
+
+      return TSDB_PATTERN_NOWILDCARDMATCH;
+    }
+
+    c1 = str[j++];
+
+    if (j <= size) {
+      if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
+        continue;
+      }
+    }
+
+    return TSDB_PATTERN_NOMATCH;
+  }
+  
+  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
+}
+
+int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
+  return compareStrRegexComp(pLeft, pRight);
+}
+
+int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
+  return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
+}
+
+int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
+  size_t sz = varDataLen(pRight);
+  char *pattern = malloc(sz + 1);
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+  pattern[sz] = 0;
+
+  sz = varDataLen(pLeft);
+  char *str = malloc(sz + 1);
+  memcpy(str, varDataVal(pLeft), sz);
+  str[sz] = 0;
+
+  int errCode = 0;
+  regex_t regex;
+  char    msgbuf[256] = {0};
+
+  int cflags = REG_EXTENDED;
+  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
+    regfree(&regex);
+    free(str);
+    free(pattern);
+    return 1;
+  }
+
+  errCode = regexec(&regex, str, 0, NULL, 0);
+  if (errCode != 0 && errCode != REG_NOMATCH) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
+  }
+  int32_t result = (errCode == 0) ? 0 : 1;
+  regfree(&regex);
+  free(str);
+  free(pattern);
+  return result;
+}
+
+int32_t taosArrayCompareString(const void* a, const void* b) {
+  const char* x = *(const char**)a;
+  const char* y = *(const char**)b;
+
+  return compareLenPrefixedStr(x, y);
+}
+
+int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
+  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+}

source/util/src/thashutil.c

@@ -15,8 +15,9 @@
 
 #include "os.h"
 #include "thash.h"
-#include "tcompare.h"
+#include "compare.h"
 #include "tdef.h"
+#include "types.h"
 
 #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
 

source/util/src/tskiplist.c

@@ -15,7 +15,7 @@
  */
 #include "tskiplist.h"
 #include "os.h"
-#include "tcompare.h"
+#include "compare.h"
 #include "ulog.h"
 #include "tutil.h"
 

source/util/src/tstep.c

@@ -20,15 +20,14 @@
 
 typedef struct SStepObj {
   char *    name;
-  void **   self;
   InitFp    initFp;
   CleanupFp cleanupFp;
 } SStep;
 
 typedef struct SSteps {
-  int32_t cursize;
-  int32_t maxsize;
-  SStep * steps;
+  int32_t  cursize;
+  int32_t  maxsize;
+  SStep *  steps;
   ReportFp reportFp;
 } SSteps;
 
@@ -44,14 +43,14 @@ SSteps *taosStepInit(int32_t maxsize, ReportFp fp) {
   return steps;
 }
 
-int32_t taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp) {
+int32_t taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp) {
   if (steps == NULL) return -1;
   if (steps->cursize >= steps->maxsize) {
     uError("failed to add step since up to the maxsize");
     return -1;
   }
 
-  SStep step = {.name = name, .self = obj, .initFp = initFp, .cleanupFp = cleanupFp};
+  SStep step = {.name = name, .initFp = initFp, .cleanupFp = cleanupFp};
   steps->steps[steps->cursize++] = step;
   return 0;
 }
@@ -61,7 +60,7 @@ static void taosStepCleanupImp(SSteps *steps, int32_t pos) {
     SStep *step = steps->steps + s;
     uDebug("step:%s will cleanup", step->name);
     if (step->cleanupFp != NULL) {
-      (*step->cleanupFp)(step->self);
+      (*step->cleanupFp)();
     }
   }
 }
@@ -77,7 +76,7 @@ int32_t taosStepExec(SSteps *steps) {
       (*steps->reportFp)(step->name, "start initialize");
     }
 
-    int32_t code = (*step->initFp)(step->self);
+    int32_t code = (*step->initFp)();
     if (code != 0) {
       uDebug("step:%s will cleanup", step->name);
       taosStepCleanupImp(steps, s);

source/util/src/tutil.c

@@ -382,24 +382,3 @@ char *taosIpStr(uint32_t ipInt) {
   return ipStr;
 }
 
-FORCE_INLINE float taos_align_get_float(const char* pBuf) {
-#if __STDC_VERSION__ >= 201112L
-  static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
-#else
-  assert(sizeof(float) == sizeof(uint32_t));
-#endif
-  float fv = 0;
-  memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
-  return fv; 
-}
-
-FORCE_INLINE double taos_align_get_double(const char* pBuf) {
-#if __STDC_VERSION__ >= 201112L
-  static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
-#else
-  assert(sizeof(double) == sizeof(uint64_t));
-#endif
-  double dv = 0;
-  memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
-  return dv; 
-}

source/util/src/tworker.c

@@ -19,37 +19,16 @@
 #include "tqueue.h"
 #include "tworker.h"
 
-static void *taosWorkerThreadFp(void *wparam) {
-  SWorker *    worker = wparam;
-  SWorkerPool *pool = worker->pool;
-  void *       msg = NULL;
-  int32_t      qtype = 0;
-  void *       ahandle = NULL;
-  int32_t      code = 0;
-
-  setThreadName(pool->name);
-
-  while (1) {
-    if (taosReadQitemFromQset(pool->qset, &qtype, (void **)&msg, &ahandle) == 0) {
-      uDebug("pool:%s, worker:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset);
-      break;
-    }
-
-    code = (*pool->reqFp)(ahandle, msg);
-    (*pool->rspFp)(ahandle, msg, qtype, code);
-  }
-
-  return NULL;
-}
+typedef void* (*ThreadFp)(void *param);
 
 int32_t tWorkerInit(SWorkerPool *pool) {
   pool->qset = taosOpenQset();
   pool->workers = calloc(sizeof(SWorker), pool->max);
   pthread_mutex_init(&pool->mutex, NULL);
   for (int i = 0; i < pool->max; ++i) {
-    SWorker *pWorker = pool->workers + i;
-    pWorker->id = i;
-    pWorker->pool = pool;
+    SWorker *worker = pool->workers + i;
+    worker->id = i;
+    worker->pool = pool;
   }
 
   uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
@@ -58,16 +37,16 @@ int32_t tWorkerInit(SWorkerPool *pool) {
 
 void tWorkerCleanup(SWorkerPool *pool) {
   for (int i = 0; i < pool->max; ++i) {
-    SWorker *pWorker = pool->workers + i;
-    if(taosCheckPthreadValid(pWorker->thread)) {
+    SWorker *worker = pool->workers + i;
+    if (taosCheckPthreadValid(worker->thread)) {
       taosQsetThreadResume(pool->qset);
     }
   }
 
   for (int i = 0; i < pool->max; ++i) {
-    SWorker *pWorker = pool->workers + i;
-    if (taosCheckPthreadValid(pWorker->thread)) {
-      pthread_join(pWorker->thread, NULL);
+    SWorker *worker = pool->workers + i;
+    if (taosCheckPthreadValid(worker->thread)) {
+      pthread_join(worker->thread, NULL);
     }
   }
 
@@ -78,42 +57,204 @@ void tWorkerCleanup(SWorkerPool *pool) {
   uInfo("worker:%s is closed", pool->name);
 }
 
-void *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle) {
+static void *tWorkerThreadFp(SWorker *worker) {
+  SWorkerPool *pool = worker->pool;
+
+  void *  msg = NULL;
+  void *  ahandle = NULL;
+  int32_t qtype = 0;
+  int32_t code = 0;
+
+  taosBlockSIGPIPE();
+  setThreadName(pool->name);
+  uDebug("worker:%s:%d is running", pool->name, worker->id);
+
+  while (1) {
+    if (taosReadQitemFromQset(pool->qset, &qtype, (void **)&msg, &ahandle) == 0) {
+      uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset);
+      break;
+    }
+
+    code = (*pool->startFp)(ahandle, msg, qtype);
+    (*pool->endFp)(ahandle, msg, qtype, code);
+  }
+
+  return NULL;
+}
+
+taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle) {
   pthread_mutex_lock(&pool->mutex);
-  taos_queue pQueue = taosOpenQueue();
-  if (pQueue == NULL) {
+  taos_queue queue = taosOpenQueue();
+  if (queue == NULL) {
     pthread_mutex_unlock(&pool->mutex);
     return NULL;
   }
 
-  taosAddIntoQset(pool->qset, pQueue, ahandle);
+  taosAddIntoQset(pool->qset, queue, ahandle);
 
   // spawn a thread to process queue
   if (pool->num < pool->max) {
     do {
-      SWorker *pWorker = pool->workers + pool->num;
+      SWorker *worker = pool->workers + pool->num;
 
       pthread_attr_t thAttr;
       pthread_attr_init(&thAttr);
       pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-      if (pthread_create(&pWorker->thread, &thAttr, taosWorkerThreadFp, pWorker) != 0) {
-        uError("workers:%s:%d failed to create thread to process since %s", pool->name, pWorker->id, strerror(errno));
+      if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWorkerThreadFp, worker) != 0) {
+        uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno));
       }
 
       pthread_attr_destroy(&thAttr);
       pool->num++;
-      uDebug("workers:%s:%d is launched, total:%d", pool->name, pWorker->id, pool->num);
+      uDebug("worker:%s:%d is launched, total:%d", pool->name, worker->id, pool->num);
     } while (pool->num < pool->min);
   }
 
   pthread_mutex_unlock(&pool->mutex);
-  uDebug("workers:%s, queue:%p is allocated, ahandle:%p", pool->name, pQueue, ahandle);
+  uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
 
-  return pQueue;
+  return queue;
 }
 
-void tWorkerFreeQueue(SWorkerPool *pool, void *pQueue) {
-  taosCloseQueue(pQueue);
-  uDebug("workers:%s, queue:%p is freed", pool->name, pQueue);
+void tWorkerFreeQueue(SWorkerPool *pool, void *queue) {
+  taosCloseQueue(queue);
+  uDebug("worker:%s, queue:%p is freed", pool->name, queue);
+}
+
+int32_t tWriteWorkerInit(SWriteWorkerPool *pool) {
+  pool->nextId = 0;
+  pool->workers = calloc(sizeof(SWriteWorker), pool->max);
+  if (pool->workers == NULL) return -1;
+
+  pthread_mutex_init(&pool->mutex, NULL);
+  for (int32_t i = 0; i < pool->max; ++i) {
+    SWriteWorker *worker = pool->workers + i;
+    worker->id = i;
+    worker->qall = NULL;
+    worker->qset = NULL;
+    worker->pool = pool;
+  }
+
+  uInfo("worker:%s is initialized, max:%d", pool->name, pool->max);
+  return 0;
+}
+
+void tWriteWorkerCleanup(SWriteWorkerPool *pool) {
+  for (int32_t i = 0; i < pool->max; ++i) {
+    SWriteWorker *worker = pool->workers + i;
+    if (taosCheckPthreadValid(worker->thread)) {
+      if (worker->qset) taosQsetThreadResume(worker->qset);
+    }
+  }
+
+  for (int32_t i = 0; i < pool->max; ++i) {
+    SWriteWorker *worker = pool->workers + i;
+    if (taosCheckPthreadValid(worker->thread)) {
+      pthread_join(worker->thread, NULL);
+      taosFreeQall(worker->qall);
+      taosCloseQset(worker->qset);
+    }
+  }
+
+  free(pool->workers);
+  pthread_mutex_destroy(&pool->mutex);
+
+  uInfo("worker:%s is closed", pool->name);
+}
+
+static void *tWriteWorkerThreadFp(SWriteWorker *worker) {
+  SWriteWorkerPool *pool = worker->pool;
+
+  void *  msg = NULL;
+  void *  ahandle = NULL;
+  int32_t numOfMsgs = 0;
+  int32_t qtype = 0;
+
+  taosBlockSIGPIPE();
+  setThreadName(pool->name);
+  uDebug("worker:%s:%d is running", pool->name, worker->id);
+
+  while (1) {
+    numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &ahandle);
+    if (numOfMsgs == 0) {
+      uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, worker->qset);
+      break;
+    }
+
+    bool fsync = false;
+    for (int32_t i = 0; i < numOfMsgs; ++i) {
+      taosGetQitem(worker->qall, &qtype, (void **)&msg);
+      fsync = fsync | (*pool->startFp)(ahandle, msg, qtype);
+    }
+
+    (*pool->syncFp)(ahandle, fsync);
+
+    // browse all items, and process them one by one
+    taosResetQitems(worker->qall);
+    for (int32_t i = 0; i < numOfMsgs; ++i) {
+      taosGetQitem(worker->qall, &qtype, (void **)&msg);
+      (*pool->endFp)(ahandle, msg, qtype);
+    }
+  }
+
+  return NULL;
+}
+
+taos_queue tWriteWorkerAllocQueue(SWriteWorkerPool *pool, void *ahandle) {
+  pthread_mutex_lock(&pool->mutex);
+  SWriteWorker *worker = pool->workers + pool->nextId;
+
+  taos_queue *queue = taosOpenQueue();
+  if (queue == NULL) {
+    pthread_mutex_unlock(&pool->mutex);
+    return NULL;
+  }
+
+  if (worker->qset == NULL) {
+    worker->qset = taosOpenQset();
+    if (worker->qset == NULL) {
+      taosCloseQueue(queue);
+      pthread_mutex_unlock(&pool->mutex);
+      return NULL;
+    }
+
+    taosAddIntoQset(worker->qset, queue, ahandle);
+    worker->qall = taosAllocateQall();
+    if (worker->qall == NULL) {
+      taosCloseQset(worker->qset);
+      taosCloseQueue(queue);
+      pthread_mutex_unlock(&pool->mutex);
+      return NULL;
+    }
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWriteWorkerThreadFp, worker) != 0) {
+      uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno));
+      taosFreeQall(worker->qall);
+      taosCloseQset(worker->qset);
+      taosCloseQueue(queue);
+      queue = NULL;
+    } else {
+      uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max);
+      pool->nextId = (pool->nextId + 1) % pool->max;
+    }
+
+    pthread_attr_destroy(&thAttr);
+  } else {
+    taosAddIntoQset(worker->qset, queue, ahandle);
+    pool->nextId = (pool->nextId + 1) % pool->max;
+  }
+
+  pthread_mutex_unlock(&pool->mutex);
+  uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
+
+  return queue;
+}
+
+void tWriteWorkerFreeQueue(SWriteWorkerPool *pool, taos_queue queue) {
+  taosCloseQueue(queue);
+  uDebug("worker:%s, queue:%p is freed", pool->name, queue);
 }

src/inc/twal.h

@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_WAL_H_
-#define _TD_WAL_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum {
-  TAOS_WAL_NOLOG = 0,
-  TAOS_WAL_WRITE = 1,
-  TAOS_WAL_FSYNC = 2
-} EWalType;
-
-typedef enum {
-  TAOS_WAL_NOT_KEEP = 0,
-  TAOS_WAL_KEEP = 1
-} EWalKeep;
-
-typedef struct {
-  int8_t   msgType;
-  int8_t   sver;  // sver 2 for WAL SDataRow/SMemRow compatibility
-  int8_t   reserved[2];
-  int32_t  len;
-  uint64_t version;
-  uint32_t signature;
-  uint32_t cksum;
-  char     cont[];
-} SWalHead;
-
-typedef struct {
-  int32_t  vgId;
-  int32_t  fsyncPeriod;  // millisecond
-  EWalType walLevel;     // wal level
-  EWalKeep keep;         // keep the wal file when closed
-} SWalCfg;
-
-typedef void *  twalh;  // WAL HANDLE
-typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
-
-int32_t  walInit();
-void     walCleanUp();
-twalh    walOpen(char *path, SWalCfg *pCfg);
-int32_t  walAlter(twalh pWal, SWalCfg *pCfg);
-void     walStop(twalh);
-void     walClose(twalh);
-int32_t  walRenew(twalh);
-void     walRemoveOneOldFile(twalh);
-void     walRemoveAllOldFiles(twalh);
-int32_t  walWrite(twalh, SWalHead *);
-void     walFsync(twalh, bool forceFsync);
-int32_t  walRestore(twalh, void *pVnode, FWalWrite writeFp);
-int32_t  walGetWalFile(twalh, char *fileName, int64_t *fileId);
-uint64_t walGetVersion(twalh);
-void     walResetVersion(twalh, uint64_t newVer);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // _TD_WAL_H_

src/vnode/CMakeLists.txt

@@ -1,14 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc)
-INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
-INCLUDE_DIRECTORIES(inc)
-AUX_SOURCE_DIRECTORY(src SRC)
-
-ADD_LIBRARY(vnode ${SRC})
-TARGET_LINK_LIBRARIES(vnode tsdb tcq common)

src/vnode/inc/vnodeInt.h

@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_VNODE_INT_H
-#define TDENGINE_VNODE_INT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "tlog.h"
-#include "tsync.h"
-#include "tcq.h"
-#include "tsdb.h"
-#include "vnode.h"
-
-extern int32_t vDebugFlag;
-extern int32_t vNumOfExistedQHandle;   // current initialized and existed query handle in current dnode
-
-#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }}
-#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }}
-#define vWarn(...)  { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }}
-#define vInfo(...)  { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("VND ", 255, __VA_ARGS__); }}
-#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
-#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
-
-typedef struct {
-  int32_t  vgId;      // global vnode group ID
-  int32_t  refCount;  // reference count
-  int64_t  queuedWMsgSize;
-  int32_t  queuedWMsg;
-  int32_t  queuedRMsg;
-  int32_t  flowctrlLevel;
-  int8_t   preClose;  // drop and close switch
-  int8_t   reserved[3];
-  int64_t  sequence;  // for topic
-  int8_t   status;
-  int8_t   role;
-  int8_t   accessState;
-  int8_t   isFull;
-  int8_t   isCommiting;
-  int8_t   dbReplica;
-  int8_t   dropped;
-  int8_t   dbType;
-  uint64_t version;   // current version
-  uint64_t cversion;  // version while commit start
-  uint64_t fversion;  // version on saved data file
-  void *   wqueue;    // write queue
-  void *   qqueue;    // read query queue
-  void *   fqueue;    // read fetch/cancel queue
-  void *   wal;
-  void *   tsdb;
-  int64_t  sync;
-  void *   events;
-  void *   cq;  // continuous query
-  int32_t  dbCfgVersion;
-  int32_t  vgCfgVersion;
-  STsdbCfg tsdbCfg;
-  SSyncCfg syncCfg;
-  SWalCfg  walCfg;
-  void *   qMgmt;
-  char *   rootDir;
-  tsem_t   sem;
-  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
-  pthread_mutex_t statusMutex;
-} SVnodeObj;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/vnode/inc/vnodeSync.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_VNODE_SYNC_H
-#define TDENGINE_VNODE_SYNC_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "vnodeInt.h"
-
-uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver);
-int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
-void     vnodeNotifyRole(int32_t vgId, int8_t role);
-void     vnodeCtrlFlow(int32_t vgId, int32_t level);
-void     vnodeStartSyncFile(int32_t vgId);
-void     vnodeStopSyncFile(int32_t vgId, uint64_t fversion);
-void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
-int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
-int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);
-
-void     vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/vnode/src/vnodeBackup.c

@@ -1,174 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "tglobal.h"
-#include "tfs.h"
-#include "vnodeBackup.h"
-#include "vnodeMain.h"
-
-typedef struct {
-  int32_t vgId;
-} SVBackupMsg;
-
-typedef struct {
-  pthread_t thread;
-  int32_t   workerId;
-} SVBackupWorker;
-
-typedef struct {
-  int32_t    num;
-  SVBackupWorker *worker;
-} SVBackupWorkerPool;
-
-static SVBackupWorkerPool tsVBackupPool;
-static taos_qset     tsVBackupQset;
-static taos_queue    tsVBackupQueue;
-
-static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) {
-  int32_t vgId = pMsg->vgId;
-  char newDir[TSDB_FILENAME_LEN] = {0};
-  char stagingDir[TSDB_FILENAME_LEN] = {0};
-
-  sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
-  sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
-
-  if (tsEnableVnodeBak) {
-    tfsRmdir(newDir);
-    tfsRename(stagingDir, newDir);
-  } else {
-    vInfo("vgId:%d, vnode backup not enabled", vgId);
-
-    tfsRmdir(stagingDir);
-  }
-}
-
-static void *vnodeBackupFunc(void *param) {
-  setThreadName("vnodeBackup");
-
-  while (1) {
-    SVBackupMsg *pMsg = NULL;
-    if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) {
-      vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset);
-      break;
-    }
-
-    vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId);
-    vnodeProcessBackupMsg(pMsg);
-
-    vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId);
-    taosFreeQitem(pMsg);
-  }
-
-  return NULL;
-}
-
-static int32_t vnodeStartBackup() {
-  tsVBackupQueue = taosOpenQueue();
-  if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
-
-  taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL);
-
-  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
-    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
-    pWorker->workerId = i;
-
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-
-    if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) {
-      vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno));
-    }
-
-    pthread_attr_destroy(&thAttr);
-
-    vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num);
-  }
-
-  vDebug("vbackup queue:%p is allocated", tsVBackupQueue);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) {
-  SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg));
-  if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
-
-  pMsg->vgId = vgId;
-
-  int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg);
-  if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS;
-
-  return code;
-}
-
-int32_t vnodeBackup(int32_t vgId) {
-  vTrace("vgId:%d, will backup", vgId);
-  return vnodeWriteIntoBackupWorker(vgId);
-}
-
-int32_t vnodeInitBackup() {
-  tsVBackupQset = taosOpenQset();
-
-  tsVBackupPool.num = 1;
-  tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num);
-
-  if (tsVBackupPool.worker == NULL) return -1;
-  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
-    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
-    pWorker->workerId = i;
-    vDebug("vbackup:%d is created", i);
-  }
-
-  vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset);
-
-  return vnodeStartBackup();
-}
-
-void vnodeCleanupBackup() {
-  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
-    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
-    if (taosCheckPthreadValid(pWorker->thread)) {
-      taosQsetThreadResume(tsVBackupQset);
-    }
-    vDebug("vbackup:%d is closed", i);
-  }
-
-  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
-    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
-    vDebug("vbackup:%d start to join", i);
-    if (taosCheckPthreadValid(pWorker->thread)) {
-      pthread_join(pWorker->thread, NULL);
-    }
-    vDebug("vbackup:%d join success", i);
-  }
-
-  vDebug("vbackup is closed, qset:%p", tsVBackupQset);
-
-  taosCloseQset(tsVBackupQset);
-  tsVBackupQset = NULL;
-
-  tfree(tsVBackupPool.worker);
-
-  vDebug("vbackup queue:%p is freed", tsVBackupQueue);
-  taosCloseQueue(tsVBackupQueue);
-  tsVBackupQueue = NULL;
-}

src/vnode/src/vnodeMgmt.c

@@ -1,214 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "dnode.h"
-#include "vnodeStatus.h"
-#include "vnodeBackup.h"
-#include "vnodeWorker.h"
-#include "vnodeRead.h"
-#include "vnodeWrite.h"
-#include "vnodeMain.h"
-
-static SHashObj *tsVnodesHash = NULL;
-
-static int32_t vnodeInitHash(void);
-static void    vnodeCleanupHash(void);
-static void    vnodeIncRef(void *ptNode);
-
-static SStep tsVnodeSteps[] = {
-  {"vnode-backup", vnodeInitBackup,    vnodeCleanupBackup},
-  {"vnode-worker", vnodeInitMWorker,    vnodeCleanupMWorker},
-  {"vnode-write",  vnodeInitWrite,      vnodeCleanupWrite},
-  {"vnode-read",   vnodeInitRead,       vnodeCleanupRead},
-  {"vnode-hash",   vnodeInitHash,       vnodeCleanupHash},
-  {"tsdb-queue",   tsdbInitCommitQueue, tsdbDestroyCommitQueue}
-};
-
-int32_t vnodeInitMgmt() {
-  int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep);
-  return dnodeStepInit(tsVnodeSteps, stepSize);
-}
-
-void vnodeCleanupMgmt() {
-  int32_t stepSize = sizeof(tsVnodeSteps) / sizeof(SStep);
-  dnodeStepCleanup(tsVnodeSteps, stepSize);
-}
-
-static int32_t vnodeInitHash() {
-  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (tsVnodesHash == NULL) {
-    vError("failed to init vnode mgmt");
-    return -1;
-  }
-
-  return 0;
-}
-
-static void vnodeCleanupHash() {
-  if (tsVnodesHash != NULL) {
-    vDebug("vnode mgmt is cleanup");
-    taosHashCleanup(tsVnodesHash);
-    tsVnodesHash = NULL;
-  }
-}
-
-void *vnodeGetWal(void *pVnode) {
-  return ((SVnodeObj *)pVnode)->wal;
-}
-
-void vnodeAddIntoHash(SVnodeObj *pVnode) {
-  taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
-}
-
-void vnodeRemoveFromHash(SVnodeObj *pVnode) { 
-  taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t));
-}
-
-static void vnodeIncRef(void *ptNode) {
-  assert(ptNode != NULL);
-
-  SVnodeObj **ppVnode = (SVnodeObj **)ptNode;
-  assert(ppVnode);
-  assert(*ppVnode);
-
-  SVnodeObj *pVnode = *ppVnode;
-  atomic_add_fetch_32(&pVnode->refCount, 1);
-  vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-}
-
-void *vnodeAcquire(int32_t vgId) {
-  SVnodeObj *pVnode = NULL;
-  if (tsVnodesHash != NULL) {
-    taosHashGetClone(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, &pVnode);
-  }
-
-  if (pVnode == NULL) {
-    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
-    vDebug("vgId:%d, not exist", vgId);
-    return NULL;
-  }
-
-  return pVnode;
-}
-
-void vnodeRelease(void *vparam) {
-  SVnodeObj *pVnode = vparam;
-  if (vparam == NULL) return;
-
-  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  int32_t vgId = pVnode->vgId;
-
-  vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode);
-  assert(refCount >= 0);
-
-  if (refCount > 0) {
-    if (vnodeInResetStatus(pVnode) && refCount <= 3) {
-      tsem_post(&pVnode->sem);
-    }
-  } else {
-    vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode);
-    vnodeDestroyInMWorker(pVnode);
-    int32_t count = taosHashGetSize(tsVnodesHash);
-    vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
-  }
-}
-
-void *vnodeAcquireNotClose(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode != NULL  && pVnode->preClose == 1) {
-    vnodeRelease(pVnode);
-    terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
-    vDebug("vgId:%d, not exist, pre closing", vgId);
-    return NULL;
-  }
-
-  return pVnode;
-}
-
-static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
-  int64_t totalStorage = 0;
-  int64_t compStorage = 0;
-  int64_t pointsWritten = 0;
-
-  if (vnodeInClosingStatus(pVnode)) return;
-  if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
-
-  if (pVnode->tsdb) {
-    tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
-  }
-
-  SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
-  pLoad->vgId = htonl(pVnode->vgId);
-  pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion);
-  pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion);
-  pLoad->totalStorage = htobe64(totalStorage);
-  pLoad->compStorage = htobe64(compStorage);
-  pLoad->pointsWritten = htobe64(pointsWritten);
-  pLoad->vnodeVersion = htobe64(pVnode->version);
-  pLoad->status = pVnode->status;
-  pLoad->role = pVnode->role;
-  pLoad->replica = pVnode->syncCfg.replica;  
-  pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0; 
-}
-
-int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  void *pIter = taosHashIterate(tsVnodesHash, NULL);
-  while (pIter) {
-    SVnodeObj **pVnode = pIter;
-    if (*pVnode) {
-
-    (*numOfVnodes)++;
-    if (*numOfVnodes >= TSDB_MAX_VNODES) {
-      vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES);
-      continue;
-    } else {
-      vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
-    }
-
-    }
-
-    pIter = taosHashIterate(tsVnodesHash, pIter);    
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-void vnodeBuildStatusMsg(void *param) {
-  SStatusMsg *pStatus = param;
-
-  void *pIter = taosHashIterate(tsVnodesHash, NULL);
-  while (pIter) {
-    SVnodeObj **pVnode = pIter;
-    if (*pVnode) {
-      vnodeBuildVloadMsg(*pVnode, pStatus);
-    }
-    pIter = taosHashIterate(tsVnodesHash, pIter);
-  }
-}
-
-void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    pAccess[i].vgId = htonl(pAccess[i].vgId);
-    SVnodeObj *pVnode = vnodeAcquireNotClose(pAccess[i].vgId);
-    if (pVnode != NULL) {
-      pVnode->accessState = pAccess[i].accessState;
-      if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) {
-        vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState);
-      }
-      vnodeRelease(pVnode);
-    }
-  }
-}

src/vnode/src/vnodeSync.c

@@ -1,180 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taosmsg.h"
-#include "query.h"
-#include "dnode.h"
-#include "vnodeVersion.h"
-#include "vnodeMain.h"
-#include "vnodeStatus.h"
-
-uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while get file info", vgId);
-    return 0;
-  }
-
-  *fver = pVnode->fversion;
-  uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
-
-  vnodeRelease(pVnode);
-  return ret;
-}
-
-int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while get wal info", vgId);
-    return -1;
-  }
-
-  int32_t code = walGetWalFile(pVnode->wal, fileName, fileId);
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-void vnodeNotifyRole(int32_t vgId, int8_t role) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vTrace("vgId:%d, vnode not found while notify role", vgId);
-    return;
-  }
-  if (pVnode->dropped) {
-    vTrace("vgId:%d, vnode dropped while notify role", vgId);
-    vnodeRelease(pVnode);
-    return;
-  }
-
-  vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]);
-  pVnode->role = role;
-  dnodeSendStatusMsgToMnode();
-
-  if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
-    cqStart(pVnode->cq);
-  } else {
-    cqStop(pVnode->cq);
-  }
-
-  vnodeRelease(pVnode);
-}
-
-void vnodeCtrlFlow(int32_t vgId, int32_t level) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vTrace("vgId:%d, vnode not found while flow ctrl", vgId);
-    return;
-  }
-  if (pVnode->dropped) {
-    vTrace("vgId:%d, vnode dropped while flow ctrl", vgId);
-    vnodeRelease(pVnode);
-    return;
-  }
-
-  if (pVnode->flowctrlLevel != level) {
-    vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level);
-    pVnode->flowctrlLevel = level;
-  }
-
-  vnodeRelease(pVnode);
-}
-
-void vnodeStartSyncFile(int32_t vgId) {
-  SVnodeObj *pVnode = vnodeAcquireNotClose(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while start filesync", vgId);
-    return;
-  }
-
-  vInfo("vgId:%d, datafile will be synced", vgId);
-  vnodeSetResetStatus(pVnode);
-
-  vnodeRelease(pVnode);
-}
-
-void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while stop filesync", vgId);
-    return;
-  }
-
-  pVnode->fversion = fversion;
-  pVnode->version = fversion;
-  vnodeSaveVersion(pVnode);
-  walResetVersion(pVnode->wal, fversion);
-
-  vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
-  vnodeSetReadyStatus(pVnode);
-
-  vnodeRelease(pVnode);
-}
-
-void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while confirm forward", vgId);
-  }
-
-  if (code == TSDB_CODE_SYN_CONFIRM_EXPIRED && pVnode->status == TAOS_VN_STATUS_CLOSING) {
-    vDebug("vgId:%d, db:%s, vnode is closing while confirm forward", vgId, pVnode->db);
-    code = TSDB_CODE_VND_IS_CLOSING;
-  }
-
-  dnodeSendRpcVWriteRsp(pVnode, wparam, code);
-  vnodeRelease(pVnode);
-}
-
-int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while write to cache", vgId);
-    vnodeRelease(pVnode);
-    return TSDB_CODE_VND_INVALID_VGROUP_ID;
-  }
-
-  int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam);
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) {
-  SVnodeObj *pVnode = vnodeAcquireNotClose(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while write to cache", vgId);
-    return -1;
-  }
-
-  int32_t code = 0;
-  if (pVnode->isCommiting) {
-    vInfo("vgId:%d, vnode is commiting while get version", vgId);
-    code = -1;
-  } else {
-    *fver = pVnode->fversion;
-    *wver = pVnode->version;
-  }
-
-  vnodeRelease(pVnode);
-  return code;
-}
-
-void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) {
-  SVnodeObj *pVnode = vparam;
-  syncConfirmForward(pVnode->sync, version, code, force);
-}

src/vnode/src/vnodeWorker.c

@@ -1,206 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
-#include "tqueue.h"
-#include "tglobal.h"
-#include "vnodeWorker.h"
-#include "vnodeMain.h"
-
-typedef enum {
-  VNODE_WORKER_ACTION_CLEANUP,
-  VNODE_WORKER_ACTION_DESTROY
-} EVMWorkerAction;
-
-typedef struct {
-  int32_t vgId;
-  int32_t code;
-  void *  rpcHandle;
-  SVnodeObj *pVnode;
-  EVMWorkerAction action;
-} SVMWorkerMsg;
-
-typedef struct {
-  pthread_t thread;
-  int32_t   workerId;
-} SVMWorker;
-
-typedef struct {
-  int32_t    curNum;
-  int32_t    maxNum;
-  SVMWorker *worker;
-} SVMWorkerPool;
-
-static SVMWorkerPool tsVMWorkerPool;
-static taos_qset     tsVMWorkerQset;
-static taos_queue    tsVMWorkerQueue;
-
-static void *vnodeMWorkerFunc(void *param);
-
-static int32_t vnodeStartMWorker() {
-  tsVMWorkerQueue = taosOpenQueue();
-  if (tsVMWorkerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
-
-  taosAddIntoQset(tsVMWorkerQset, tsVMWorkerQueue, NULL);
-
-  for (int32_t i = tsVMWorkerPool.curNum; i < tsVMWorkerPool.maxNum; ++i) {
-    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
-    pWorker->workerId = i;
-
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-
-    if (pthread_create(&pWorker->thread, &thAttr, vnodeMWorkerFunc, pWorker) != 0) {
-      vError("failed to create thread to process vmworker queue, reason:%s", strerror(errno));
-    }
-
-    pthread_attr_destroy(&thAttr);
-
-    tsVMWorkerPool.curNum = i + 1;
-    vDebug("vmworker:%d is launched, total:%d", pWorker->workerId, tsVMWorkerPool.maxNum);
-  }
-
-  vDebug("vmworker queue:%p is allocated", tsVMWorkerQueue);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t vnodeInitMWorker() {
-  tsVMWorkerQset = taosOpenQset();
-
-  tsVMWorkerPool.maxNum = 1;
-  tsVMWorkerPool.curNum = 0;
-  tsVMWorkerPool.worker = calloc(sizeof(SVMWorker), tsVMWorkerPool.maxNum);
-
-  if (tsVMWorkerPool.worker == NULL) return -1;
-  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
-    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
-    pWorker->workerId = i;
-    vDebug("vmworker:%d is created", i);
-  }
-
-  vDebug("vmworker is initialized, num:%d qset:%p", tsVMWorkerPool.maxNum, tsVMWorkerQset);
-
-  return vnodeStartMWorker();
-}
-
-static void vnodeStopMWorker() {
-  vDebug("vmworker queue:%p is freed", tsVMWorkerQueue);
-  taosCloseQueue(tsVMWorkerQueue);
-  tsVMWorkerQueue = NULL;
-}
-
-void vnodeCleanupMWorker() {
-  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
-    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
-    if (taosCheckPthreadValid(pWorker->thread)) {
-      taosQsetThreadResume(tsVMWorkerQset);
-    }
-    vDebug("vmworker:%d is closed", i);
-  }
-
-  for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) {
-    SVMWorker *pWorker = tsVMWorkerPool.worker + i;
-    vDebug("vmworker:%d start to join", i);
-    if (taosCheckPthreadValid(pWorker->thread)) {
-      pthread_join(pWorker->thread, NULL);
-    }
-    vDebug("vmworker:%d join success", i);
-  }
-
-  vDebug("vmworker is closed, qset:%p", tsVMWorkerQset);
-
-  taosCloseQset(tsVMWorkerQset);
-  tsVMWorkerQset = NULL;
-  tfree(tsVMWorkerPool.worker);
-
-  vnodeStopMWorker();
-}
-
-static int32_t vnodeWriteIntoMWorker(SVnodeObj *pVnode, EVMWorkerAction action, void *rpcHandle) {
-  SVMWorkerMsg *pMsg = taosAllocateQitem(sizeof(SVMWorkerMsg));
-  if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
-
-  pMsg->vgId = pVnode->vgId;
-  pMsg->pVnode = pVnode;
-  pMsg->rpcHandle = rpcHandle;
-  pMsg->action = action;
-
-  int32_t code = taosWriteQitem(tsVMWorkerQueue, TAOS_QTYPE_RPC, pMsg);
-  if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS;
-
-  return code;
-}
-
-int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode) {
-  vTrace("vgId:%d, will cleanup in vmworker", pVnode->vgId);
-  return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_CLEANUP, NULL);
-}
-
-int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode) {
-  vTrace("vgId:%d, will destroy in vmworker", pVnode->vgId);
-  return vnodeWriteIntoMWorker(pVnode, VNODE_WORKER_ACTION_DESTROY, NULL);
-}
-
-static void vnodeFreeMWorkerMsg(SVMWorkerMsg *pMsg) {
-  vTrace("vgId:%d, disposed in vmworker", pMsg->vgId);
-  taosFreeQitem(pMsg);
-}
-
-static void vnodeSendVMWorkerRpcRsp(SVMWorkerMsg *pMsg) {
-  if (pMsg->rpcHandle != NULL) {
-    SRpcMsg rpcRsp = {.handle = pMsg->rpcHandle, .code = pMsg->code};
-    rpcSendResponse(&rpcRsp);
-  }
-
-  vnodeFreeMWorkerMsg(pMsg);
-}
-
-static void vnodeProcessMWorkerMsg(SVMWorkerMsg *pMsg) {
-  pMsg->code = 0;
-
-  switch (pMsg->action) {
-    case VNODE_WORKER_ACTION_CLEANUP:
-      vnodeCleanUp(pMsg->pVnode);
-      break;
-    case VNODE_WORKER_ACTION_DESTROY:
-      vnodeDestroy(pMsg->pVnode);
-      break;
-    default:
-      break;
-  }
-}
-
-static void *vnodeMWorkerFunc(void *param) {
-  setThreadName("vnodeMWorker");
-
-  while (1) {
-    SVMWorkerMsg *pMsg = NULL;
-    if (taosReadQitemFromQset(tsVMWorkerQset, NULL, (void **)&pMsg, NULL) == 0) {
-      vDebug("qset:%p, vmworker got no message from qset, exiting", tsVMWorkerQset);
-      break;
-    }
-
-    vTrace("vgId:%d, action:%d will be processed in vmworker queue", pMsg->vgId, pMsg->action);
-    vnodeProcessMWorkerMsg(pMsg);
-    vnodeSendVMWorkerRpcRsp(pMsg);
-  }
-
-  return NULL;
-}

src/vnode/src/vnodeWrite.c

@@ -1,408 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tp.h"
-#include "taosmsg.h"
-#include "taoserror.h"
-#include "tglobal.h"
-#include "tqueue.h"
-#include "ttimer.h"
-#include "dnode.h"
-#include "vnodeStatus.h"
-
-#define MAX_QUEUED_MSG_NUM 100000
-#define MAX_QUEUED_MSG_SIZE 1024*1024*1024  //1GB
-
-extern void *  tsDnodeTmr;
-static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *);
-static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
-static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite);
-
-int32_t vnodeInitWrite(void) {
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT]          = vnodeProcessSubmitMsg;
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = vnodeProcessCreateTableMsg;
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]   = vnodeProcessDropTableMsg;
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = vnodeProcessAlterTableMsg;
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = vnodeProcessDropStableMsg;
-  vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL]  = vnodeProcessUpdateTagValMsg;
-
-  return 0;
-}
-
-void vnodeCleanupWrite() {}
-
-int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) {
-  int32_t    code = 0;
-  SVnodeObj *pVnode = vparam;
-  SWalHead * pHead = wparam;
-  SVWriteMsg*pWrite = rparam;
-
-  SRspRet *pRspRet = NULL;
-  if (pWrite != NULL) pRspRet = &pWrite->rspRet;
-
-  if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
-    vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId,
-           taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version);
-    return TSDB_CODE_VND_MSG_NOT_PROCESSED;
-  }
-
-  vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId,
-         taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version);
-
-  if (pHead->version == 0) {  // from client or CQ
-    if (!vnodeInReadyStatus(pVnode)) {
-      vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId,
-             taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version);
-      return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
-    }
-
-    if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-      vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId,
-             taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version);
-      return TSDB_CODE_APP_NOT_READY;
-    }
-
-    // assign version
-    pHead->version = pVnode->version + 1;
-  } else {  // from wal or forward
-    // for data from WAL or forward, version may be smaller
-    if (pHead->version <= pVnode->version) return 0;
-  }
-
-  // forward to peers, even it is WAL/FWD, it shall be called to update version in sync
-  int32_t syncCode = 0;
-  bool    force = (pWrite == NULL ? false : pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT);
-  syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force);
-  if (syncCode < 0) {
-    pHead->version = 0;
-    return syncCode;
-  }
-
-  // write into WAL
-  code = walWrite(pVnode->wal, pHead);
-  if (code < 0) {
-    if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
-    vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code);
-    pHead->version = 0;
-    return code;
-  }
-
-  pVnode->version = pHead->version;
-
-  // write data locally
-  code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet);
-  if (code < 0) {
-    if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
-    return code;
-  }
-
-  return syncCode;
-}
-
-static int32_t vnodeCheckWrite(SVnodeObj *pVnode) {
-  if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
-    vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-    return TSDB_CODE_VND_NO_WRITE_AUTH;
-  }
-
-  if (pVnode->dbReplica != pVnode->syncCfg.replica &&
-      pVnode->syncCfg.nodeInfo[pVnode->syncCfg.replica - 1].nodeId == dnodeGetDnodeId()) {
-    vDebug("vgId:%d, vnode is balancing and will be dropped, dbReplica:%d vgReplica:%d, refCount:%d pVnode:%p",
-           pVnode->vgId, pVnode->dbReplica, pVnode->syncCfg.replica, pVnode->refCount, pVnode);
-    return TSDB_CODE_VND_IS_BALANCING;
-  }
-
-  // tsdb may be in reset state
-  if (pVnode->tsdb == NULL) {
-    vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode);
-    return TSDB_CODE_APP_NOT_READY;
-  }
-
-  if (pVnode->isFull) {
-    vDebug("vgId:%d, vnode is full, refCount:%d", pVnode->vgId, pVnode->refCount);
-    return TSDB_CODE_VND_IS_FULL;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  vTrace("vgId:%d, submit msg is processed", pVnode->vgId);
-
-  if (pVnode->dbType == TSDB_DB_TYPE_TOPIC && pVnode->role == TAOS_SYNC_ROLE_MASTER) {
-    tpUpdateTs(pVnode->vgId, &pVnode->sequence, pCont);
-  }
-
-  // save insert result into item
-  SShellSubmitRspMsg *pRsp = NULL;
-  if (pRet) {
-    pRet->len = sizeof(SShellSubmitRspMsg);
-    pRet->rsp = rpcMallocCont(pRet->len);
-    pRsp = pRet->rsp;
-  }
-
-  if (tsdbInsertData(pVnode->tsdb, pCont, pRsp) < 0) code = terrno;
-
-  return code;
-}
-
-static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  int code = TSDB_CODE_SUCCESS;
-
-  STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
-  if (pCfg == NULL) {
-    ASSERT(terrno != 0);
-    return terrno;
-  }
-
-  if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) {
-    code = terrno;
-    ASSERT(code != 0);
-  }
-
-  tsdbClearTableCfg(pCfg);
-  return code;
-}
-
-static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  SMDDropTableMsg *pTable = pCont;
-  int32_t          code = TSDB_CODE_SUCCESS;
-
-  vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableFname);
-  STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->tid)};
-
-  if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno;
-
-  return code;
-}
-
-static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  // TODO: disposed in tsdb
-  // STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont);
-  // if (pCfg == NULL) return terrno;
-  // if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno;
-
-  // tsdbClearTableCfg(pCfg);
-  vDebug("vgId:%d, alter table msg is received", pVnode->vgId);
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  SDropSTableMsg *pTable = pCont;
-  int32_t         code = TSDB_CODE_SUCCESS;
-
-  vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableFname);
-
-  STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1};
-
-  if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno;
-
-  vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableFname, tstrerror(code));
-
-  return code;
-}
-
-static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
-  if (tsdbUpdateTableTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont) < 0) {
-    return terrno;
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) {
-  if (pHead->len > TSDB_MAX_WAL_SIZE) {
-    vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version);
-    terrno = TSDB_CODE_WAL_SIZE_LIMIT;
-    return NULL;
-  }
-
-  int32_t size = sizeof(SVWriteMsg) + pHead->len;
-  SVWriteMsg *pWrite = taosAllocateQitem(size);
-  if (pWrite == NULL) {
-    terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  if (pRpcMsg != NULL) {
-    pWrite->rpcMsg = *pRpcMsg;
-  }
-
-  memcpy(&pWrite->walHead, pHead, sizeof(SWalHead) + pHead->len);
-  pWrite->pVnode = pVnode;
-  pWrite->qtype = qtype;
-
-  atomic_add_fetch_32(&pVnode->refCount, 1);
-
-  return pWrite;
-}
-
-static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) {
-  SVnodeObj *pVnode = pWrite->pVnode;
-
-  if (pWrite->qtype == TAOS_QTYPE_RPC) {
-    int32_t code = vnodeCheckWrite(pVnode);
-    if (code != TSDB_CODE_SUCCESS) {
-      vError("vgId:%d, failed to write into vwqueue since %s", pVnode->vgId, tstrerror(code));
-      taosFreeQitem(pWrite);
-      vnodeRelease(pVnode);
-      return code;
-    }
-  }
-
-  if (tsAvailDataDirGB <= tsMinimalDataDirGB) {
-    vError("vgId:%d, failed to write into vwqueue since no diskspace, avail:%fGB", pVnode->vgId, tsAvailDataDirGB);
-    taosFreeQitem(pWrite);
-    vnodeRelease(pVnode);
-    return TSDB_CODE_VND_NO_DISKSPACE;
-  }
-
-  if (!vnodeInReadyOrUpdatingStatus(pVnode)) {
-    vError("vgId:%d, failed to write into vwqueue, vstatus is %s, refCount:%d pVnode:%p", pVnode->vgId,
-           vnodeStatus[pVnode->status], pVnode->refCount, pVnode);
-    taosFreeQitem(pWrite);
-    vnodeRelease(pVnode);
-    return TSDB_CODE_APP_NOT_READY;
-  }
-
-  int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1);
-  int64_t queuedSize = atomic_add_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len);
-
-  if (queued > MAX_QUEUED_MSG_NUM || queuedSize > MAX_QUEUED_MSG_SIZE) {
-    int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3;
-    if (ms > 100) ms = 100;
-    vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms);
-    taosMsleep(ms);
-  }
-
-  vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d size:%" PRId64, pVnode->vgId, pVnode->refCount,
-         pVnode->queuedWMsg, pVnode->queuedWMsgSize);
-
-  taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) {
-  SVnodeObj *pVnode = vparam;
-  if (qtype == TAOS_QTYPE_RPC) {
-   if (!vnodeInReadyStatus(pVnode)) {
-      return TSDB_CODE_APP_NOT_READY;  // it may be in deleting or closing state
-    }
-
-    if (pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-      return TSDB_CODE_APP_NOT_READY;
-    }
-  }
-
-  SVWriteMsg *pWrite = vnodeBuildVWriteMsg(vparam, wparam, qtype, rparam);
-  if (pWrite == NULL) {
-    assert(terrno != 0);
-    return terrno;
-  }
-
-  int32_t code = vnodePerformFlowCtrl(pWrite);
-  if (code != 0) return 0;
-
-  return vnodeWriteToWQueueImp(pWrite);
-}
-
-void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) {
-  SVnodeObj *pVnode = vparam;
-  if (pVnode) {
-    int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1);
-    int64_t queuedSize = atomic_sub_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len);
-
-    vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d size:%" PRId64, pVnode->vgId, pWrite,
-           pWrite->rpcMsg.ahandle, queued, queuedSize);
-  }
-
-  taosFreeQitem(pWrite);
-  vnodeRelease(pVnode);
-}
-
-static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) {
-  SVWriteMsg *pWrite = param;
-  SVnodeObj * pVnode = pWrite->pVnode;
-  int32_t     code = TSDB_CODE_VND_IS_SYNCING;
-
-  if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL;
-
-  pWrite->processedCount++;
-  if (pWrite->processedCount >= 100) {
-    vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code),
-           pWrite->processedCount);
-    void *handle = pWrite->rpcMsg.handle;
-    taosFreeQitem(pWrite);
-    vnodeRelease(pVnode);
-    SRpcMsg rpcRsp = {.handle = handle, .code = code};
-    rpcSendResponse(&rpcRsp);
-  } else {
-    code = vnodePerformFlowCtrl(pWrite);
-    if (code == 0) {
-      vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite,
-             pWrite->processedCount);
-      pWrite->processedCount = 0;
-      void *handle = pWrite->rpcMsg.handle;
-      code = vnodeWriteToWQueueImp(pWrite);
-      if (code != TSDB_CODE_SUCCESS) {
-        SRpcMsg rpcRsp = {.handle = handle, .code = code};
-        rpcSendResponse(&rpcRsp);
-      }
-    }
-  }
-}
-
-static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) {
-  SVnodeObj *pVnode = pWrite->pVnode;
-  if (pWrite->qtype != TAOS_QTYPE_RPC) return 0;
-  if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->queuedWMsgSize < MAX_QUEUED_MSG_SIZE &&
-      pVnode->flowctrlLevel <= 0)
-    return 0;
-
-  if (tsEnableFlowCtrl == 0) {
-    int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2);
-    if (ms > 100) ms = 100;
-    vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms);
-    taosMsleep(ms);
-    return 0;
-  } else {
-    void *unUsedTimerId = NULL;
-    taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsedTimerId);
-
-    vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle,
-           pWrite->processedCount);
-    return TSDB_CODE_VND_ACTION_IN_PROGRESS;
-  }
-}
-
-void vnodeWaitWriteCompleted(SVnodeObj *pVnode) {
-  int32_t extraSleep = 0;
-  while (pVnode->queuedWMsg > 0) {
-    vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg);
-    taosMsleep(10);
-    extraSleep = 1;
-  }
-
-  if (extraSleep)
-    taosMsleep(900);
-}