Merge remote-tracking branch 'origin/3.0' into feature/3.0_liaohj

cmake/cmake.define

@@ -36,11 +36,13 @@ IF (TD_WINDOWS)
     ENDIF ()
 
 ELSE ()
-    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
+    #SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+    #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
 
 MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
-IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
+IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
     ADD_DEFINITIONS("-D_TD_ARM_")
 ELSE ()
     ADD_DEFINITIONS("-msse4.2 -mfma")

cmake/cmake.options

@@ -55,17 +55,6 @@ option(
     OFF
 )
 
-IF(${TD_WINDOWS})
-    MESSAGE("Not build BDB on Windows")
-ELSE ()
-    option(
-            BUILD_WITH_BDB
-            "If build with BerkleyDB"
-            ON
-          )
-
-ENDIF ()
-
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 

cmake/cmake.platform

@@ -42,7 +42,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
 
         SET(TD_DARWIN TRUE)
         SET(OSTYPE "macOS")
-        ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare -Wno-return-type")
+        ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare")
 
         MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
         IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")

contrib/CMakeLists.txt

@@ -63,9 +63,9 @@ if(${BUILD_WITH_UV})
 endif(${BUILD_WITH_UV})
 
 # bdb
-if(${BUILD_WITH_BDB})
+#if(${BUILD_WITH_BDB})
-    cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-endif(${BUILD_WITH_BDB})
+#endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})

contrib/test/CMakeLists.txt

@@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
-if(${BUILD_WITH_BDB})
+#if(${BUILD_WITH_BDB})
-    add_subdirectory(bdb)
+    #add_subdirectory(bdb)
-endif(${BUILD_WITH_BDB})
+#endif(${BUILD_WITH_BDB})
 
 if(${BUILD_WITH_SQLITE})
     add_subdirectory(sqlite)

include/common/tdatablock.h

@@ -127,39 +127,39 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
   pColumnInfoData->hasNull = true;
 }
 
-static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
+static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT ||
          pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int8_t*)p = *(int8_t*)v;
 }
 
-static FORCE_INLINE int32_t colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
+static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int16_t*)p = *(int16_t*)v;
 }
 
-static FORCE_INLINE int32_t colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
+static FORCE_INLINE void colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int32_t*)p = *(int32_t*)v;
 }
 
-static FORCE_INLINE int32_t colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
+static FORCE_INLINE void colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
   int32_t type = pColumnInfoData->info.type;
   ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int64_t*)p = *(int64_t*)v;
 }
 
-static FORCE_INLINE int32_t colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
+static FORCE_INLINE void colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(float*)p = *(float*)v;
 }
 
-static FORCE_INLINE int32_t colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
+static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(double*)p = *(double*)v;
@@ -177,7 +177,7 @@ void    colDataTrim(SColumnInfoData* pColumnInfoData);
 size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
 size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
 
-int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
+int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap);
 int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
                            int32_t pageSize);
 int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);

include/common/tmsg.h

@@ -512,6 +512,7 @@ typedef struct {
   int32_t maxRows;
   int32_t commitTime;
   int32_t fsyncPeriod;
+  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;  // time resolution
   int8_t  compression;
@@ -521,6 +522,7 @@ typedef struct {
   int8_t  cacheLastRow;
   int8_t  ignoreExist;
   int8_t  streamMode;
+  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;  // SRetention
 } SCreateDbReq;
@@ -586,6 +588,41 @@ int32_t tSerializeSUseDbRspImp(SCoder* pEncoder, const SUseDbRsp* pRsp);
 int32_t tDeserializeSUseDbRspImp(SCoder* pDecoder, SUseDbRsp* pRsp);
 void    tFreeSUsedbRsp(SUseDbRsp* pRsp);
 
+typedef struct {
+  char    db[TSDB_DB_FNAME_LEN];
+} SDbCfgReq;
+
+int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
+int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
+
+typedef struct {
+  int32_t numOfVgroups;
+  int32_t cacheBlockSize;
+  int32_t totalBlocks;
+  int32_t daysPerFile;
+  int32_t daysToKeep0;
+  int32_t daysToKeep1;
+  int32_t daysToKeep2;
+  int32_t minRows;
+  int32_t maxRows;
+  int32_t commitTime;
+  int32_t fsyncPeriod;
+  int32_t ttl;
+  int8_t  walLevel;
+  int8_t  precision;
+  int8_t  compression;
+  int8_t  replications;
+  int8_t  quorum;
+  int8_t  update;
+  int8_t  cacheLastRow;
+  int8_t  streamMode;
+  int8_t  singleSTable;
+} SDbCfgRsp;
+
+int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp);
+int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp);
+
+
 typedef struct {
   int32_t rowNum;
 } SQnodeListReq;
@@ -680,6 +717,18 @@ typedef struct {
   char    charset[TD_LOCALE_LEN];     // tsCharset
 } SClusterCfg;
 
+typedef struct {
+  int32_t openVnodes;
+  int32_t totalVnodes;
+  int32_t masterNum;
+  int64_t numOfSelectReqs;
+  int64_t numOfInsertReqs;
+  int64_t numOfInsertSuccessReqs;
+  int64_t numOfBatchInsertReqs;
+  int64_t numOfBatchInsertSuccessReqs;
+  int64_t errors;
+} SVnodesStat;
+
 typedef struct {
   int32_t vgId;
   int8_t  role;
@@ -2273,6 +2322,7 @@ typedef struct {
   char    cgroup[TSDB_CGROUP_LEN];
 
   int64_t currentOffset;
+  uint64_t reqId;
   char    topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
 

include/common/tmsgdef.h

@@ -155,6 +155,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -217,6 +218,15 @@ enum {
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL)
   
+  // Monitor info exchange between processes
+  TD_NEW_MSG_SEG(TDMT_MON_MSG)
+  TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MON_QM_INFO, "monitor-qinfo", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL)
+
 #if defined(TD_MSG_NUMBER_)
   TDMT_MAX
 #endif

include/libs/catalog/catalog.h

@@ -77,6 +77,8 @@ typedef struct SDbVgVersion {
   int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
 } SDbVgVersion;
 
+typedef SDbCfgRsp SDbCfgInfo;
+
 int32_t catalogInit(SCatalogCfg *cfg);
 
 /**
@@ -217,6 +219,8 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable
 
 int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
 
+int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg);
+
 
 /**
  * Destroy catalog and relase all resources

include/libs/monitor/monitor.h

@@ -19,6 +19,7 @@
 #include "tarray.h"
 #include "tdef.h"
 #include "tlog.h"
+#include "tmsg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -29,6 +30,44 @@ extern "C" {
 #define MON_VER_LEN    12
 #define MON_LOG_LEN    1024
 
+typedef struct {
+  int64_t   ts;
+  ELogLevel level;
+  char      content[MON_LOG_LEN];
+} SMonLogItem;
+
+typedef struct {
+  SArray *logs;  // array of SMonLogItem
+  int32_t numOfErrorLogs;
+  int32_t numOfInfoLogs;
+  int32_t numOfDebugLogs;
+  int32_t numOfTraceLogs;
+} SMonLogs;
+
+typedef struct {
+  char      name[TSDB_FILENAME_LEN];
+  int8_t    level;
+  SDiskSize size;
+} SMonDiskDesc;
+
+typedef struct {
+  double  cpu_engine;
+  double  cpu_system;
+  float   cpu_cores;
+  int64_t mem_engine;     // KB
+  int64_t mem_system;     // KB
+  int64_t mem_total;      // KB
+  int64_t disk_engine;    // Byte
+  int64_t disk_used;      // Byte
+  int64_t disk_total;     // Byte
+  int64_t net_in;         // bytes
+  int64_t net_out;        // bytes
+  int64_t io_read;        // bytes
+  int64_t io_write;       // bytes
+  int64_t io_read_disk;   // bytes
+  int64_t io_write_disk;  // bytes
+} SMonSysInfo;
+
 typedef struct {
   int32_t dnode_id;
   char    dnode_ep[TSDB_EP_LEN];
@@ -36,6 +75,19 @@ typedef struct {
   int32_t protocol;
 } SMonBasicInfo;
 
+typedef struct {
+  float        uptime;  // day
+  int8_t       has_mnode;
+  SMonDiskDesc logdir;
+  SMonDiskDesc tempdir;
+} SMonDnodeInfo;
+
+typedef struct {
+  SMonBasicInfo basic;
+  SMonDnodeInfo dnode;
+  SMonSysInfo   sys;
+} SMonDmInfo;
+
 typedef struct {
   int32_t dnode_id;
   char    dnode_ep[TSDB_EP_LEN];
@@ -87,46 +139,65 @@ typedef struct {
 } SMonGrantInfo;
 
 typedef struct {
-  float   uptime;  // day
+  SMonClusterInfo cluster;
-  double  cpu_engine;
+  SMonVgroupInfo  vgroup;
-  double  cpu_system;
+  SMonGrantInfo   grant;
-  float   cpu_cores;
+  SMonSysInfo     sys;
-  int64_t mem_engine;   // KB
+  SMonLogs        log;
-  int64_t mem_system;   // KB
+} SMonMmInfo;
-  int64_t mem_total;    // KB
+
-  int64_t disk_engine;  // Byte
+int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo);
-  int64_t disk_used;    // Byte
+int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo);
-  int64_t disk_total;   // Byte
+void    tFreeSMonMmInfo(SMonMmInfo *pInfo);
-  int64_t net_in;       // bytes
-  int64_t net_out;      // bytes
-  int64_t io_read;      // bytes
-  int64_t io_write;     // bytes
-  int64_t io_read_disk;   // bytes
-  int64_t io_write_disk;  // bytes
-  int64_t req_select;
-  int64_t req_insert;
-  int64_t req_insert_success;
-  int64_t req_insert_batch;
-  int64_t req_insert_batch_success;
-  int32_t errors;
-  int32_t vnodes_num;
-  int32_t masters;
-  int8_t  has_mnode;
-} SMonDnodeInfo;
 
 typedef struct {
-  char      name[TSDB_FILENAME_LEN];
+  SArray *datadirs;  // array of SMonDiskDesc
-  int8_t    level;
-  SDiskSize size;
-} SMonDiskDesc;
-
-typedef struct {
-  SArray      *datadirs;  // array of SMonDiskDesc
-  SMonDiskDesc logdir;
-  SMonDiskDesc tempdir;
 } SMonDiskInfo;
 
-typedef struct SMonInfo SMonInfo;
+typedef struct {
+  SMonDiskInfo tfs;
+  SVnodesStat  vstat;
+  SMonSysInfo  sys;
+  SMonLogs     log;
+} SMonVmInfo;
+
+int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo);
+int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo);
+void    tFreeSMonVmInfo(SMonVmInfo *pInfo);
+
+typedef struct {
+  SMonSysInfo sys;
+  SMonLogs    log;
+} SMonQmInfo;
+
+int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo);
+int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo);
+void    tFreeSMonQmInfo(SMonQmInfo *pInfo);
+
+typedef struct {
+  SMonSysInfo sys;
+  SMonLogs    log;
+} SMonSmInfo;
+
+int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo);
+int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo);
+void    tFreeSMonSmInfo(SMonSmInfo *pInfo);
+typedef struct {
+  SMonSysInfo sys;
+  SMonLogs    log;
+} SMonBmInfo;
+
+int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo);
+int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo);
+void    tFreeSMonBmInfo(SMonBmInfo *pInfo);
+
+typedef struct {
+  SArray *pVloads;  // SVnodeLoad
+} SMonVloadInfo;
+
+int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo);
+int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo);
+void    tFreeSMonVloadInfo(SMonVloadInfo *pInfo);
 
 typedef struct {
   const char *server;
@@ -138,16 +209,14 @@ typedef struct {
 int32_t monInit(const SMonCfg *pCfg);
 void    monCleanup();
 void    monRecordLog(int64_t ts, ELogLevel level, const char *content);
-
+int32_t monGetLogs(SMonLogs *logs);
-SMonInfo *monCreateMonitorInfo();
+void    monSetDmInfo(SMonDmInfo *pInfo);
-void      monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
+void    monSetMmInfo(SMonMmInfo *pInfo);
-void      monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo);
+void    monSetVmInfo(SMonVmInfo *pInfo);
-void      monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo);
+void    monSetQmInfo(SMonQmInfo *pInfo);
-void      monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo);
+void    monSetSmInfo(SMonSmInfo *pInfo);
-void      monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo);
+void    monSetBmInfo(SMonBmInfo *pInfo);
-void      monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo);
+void    monSendReport();
-void      monSendReport(SMonInfo *pMonitor);
-void      monCleanupMonitorInfo(SMonInfo *pMonitor);
 
 #ifdef __cplusplus
 }

include/libs/nodes/nodes.h

@@ -30,11 +30,19 @@ extern "C" {
 #define FOREACH(node, list)	\
   for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
 
-// only be use in FOREACH
-#define ERASE_NODE(list) cell = nodesListErase(list, cell);
-
 #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
 
+#define INSERT_LIST(target, src) nodesListInsertList((target), cell, src)
+
+#define WHERE_EACH(node, list) \
+  SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
+  while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false))
+
+#define WHERE_NEXT cell = cell->pNext
+
+// only be use in WHERE_EACH
+#define ERASE_NODE(list) cell = nodesListErase((list), cell)
+
 #define FORBOTH(node1, list1, node2, list2) \
   for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \
     (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
@@ -202,7 +210,9 @@ int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
+int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
+void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
 SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);
 // Only clear the linked list structure, without releasing the elements inside

include/libs/nodes/querynodes.h

@@ -124,6 +124,7 @@ typedef struct STableNode {
   char dbName[TSDB_DB_NAME_LEN];
   char tableName[TSDB_TABLE_NAME_LEN];
   char tableAlias[TSDB_TABLE_NAME_LEN];
+  uint8_t precision;
 } STableNode;
 
 struct STableMeta;
@@ -236,6 +237,7 @@ typedef struct SSelectStmt {
   SNode* pLimit;
   SNode* pSlimit;
   char stmtName[TSDB_TABLE_NAME_LEN];
+  uint8_t precision;
 } SSelectStmt;
 
 typedef enum ESetOperatorType {

include/os/osSysinfo.h

@@ -44,8 +44,8 @@ int32_t taosGetTotalMemory(int64_t *totalKB);
 int32_t taosGetProcMemory(int64_t *usedKB);
 int32_t taosGetSysMemory(int64_t *usedKB);
 int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
-int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
+void    taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
-int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
+void    taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes);
 
 void    taosKillSystem();
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen);

include/util/taoserror.h

@@ -483,16 +483,17 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
 #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 #define TSDB_CODE_PAR_INVALID_RANGE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x2619)
-#define TSDB_CODE_PAR_INVALID_STR_OPTION        TAOS_DEF_ERROR_CODE(0, 0x2620)
+#define TSDB_CODE_PAR_INVALID_STR_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261A)
-#define TSDB_CODE_PAR_INVALID_ENUM_OPTION       TAOS_DEF_ERROR_CODE(0, 0x2621)
+#define TSDB_CODE_PAR_INVALID_ENUM_OPTION       TAOS_DEF_ERROR_CODE(0, 0x261B)
-#define TSDB_CODE_PAR_INVALID_TTL_OPTION        TAOS_DEF_ERROR_CODE(0, 0x2622)
+#define TSDB_CODE_PAR_INVALID_TTL_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261C)
-#define TSDB_CODE_PAR_INVALID_KEEP_NUM          TAOS_DEF_ERROR_CODE(0, 0x2623)
+#define TSDB_CODE_PAR_INVALID_KEEP_NUM          TAOS_DEF_ERROR_CODE(0, 0x261D)
-#define TSDB_CODE_PAR_INVALID_KEEP_ORDER        TAOS_DEF_ERROR_CODE(0, 0x2624)
+#define TSDB_CODE_PAR_INVALID_KEEP_ORDER        TAOS_DEF_ERROR_CODE(0, 0x261E)
-#define TSDB_CODE_PAR_INVALID_KEEP_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2625)
+#define TSDB_CODE_PAR_INVALID_KEEP_VALUE        TAOS_DEF_ERROR_CODE(0, 0x261F)
-#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION    TAOS_DEF_ERROR_CODE(0, 0x2626)
+#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION    TAOS_DEF_ERROR_CODE(0, 0x2620)
-#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION    TAOS_DEF_ERROR_CODE(0, 0x2627)
+#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION    TAOS_DEF_ERROR_CODE(0, 0x2621)
-#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION     TAOS_DEF_ERROR_CODE(0, 0x2628)
+#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION     TAOS_DEF_ERROR_CODE(0, 0x2622)
-#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2629)
+#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2623)
+#define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2700)

source/client/src/tmq.c

@@ -1044,7 +1044,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
   if (epStatus == 1) {
     int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
-    tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
+    tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
     if (epSkipCnt < 5000) return 0;
   }
   atomic_store_32(&tmq->epSkipCnt, 0);
@@ -1149,6 +1149,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
   pReq->consumerId = tmq->consumerId;
   pReq->epoch = tmq->epoch;
   pReq->currentOffset = reqOffset;
+  pReq->reqId = generateRequestId();
 
   pReq->head.vgId = htonl(pVg->vgId);
   pReq->head.contLen = htonl(sizeof(SMqPollReq));
@@ -1234,7 +1235,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
         int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
-        tscDebug("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
+        tscTrace("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
         continue;
         /*if (vgSkipCnt < 10000) continue;*/
 #if 0
@@ -1279,7 +1280,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
           .len = sizeof(SMqPollReq),
           .handle = NULL,
       };
-      sendInfo->requestId = generateRequestId();
+      sendInfo->requestId = pReq->reqId;
       sendInfo->requestObjRefId = 0;
       sendInfo->param = pParam;
       sendInfo->fp = tmqPollCb;
@@ -1288,7 +1289,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset);
+      tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;

source/common/src/tdatablock.c

@@ -17,6 +17,7 @@
 #include "tdatablock.h"
 #include "tcompare.h"
 #include "tglobal.h"
+#include "tlog.h"
 
 int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
   pEp->port = 0;
@@ -130,44 +131,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
     pColumnInfoData->varmeta.length += varDataTLen(pData);
   } else {
-    char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+    memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
-    switch (type) {
-      case TSDB_DATA_TYPE_BOOL: {
-        *(bool*)p = *(bool*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_TINYINT:
-      case TSDB_DATA_TYPE_UTINYINT: {
-        *(int8_t*)p = *(int8_t*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_SMALLINT:
-      case TSDB_DATA_TYPE_USMALLINT: {
-        *(int16_t*)p = *(int16_t*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_INT:
-      case TSDB_DATA_TYPE_UINT: {
-        *(int32_t*)p = *(int32_t*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_TIMESTAMP:
-      case TSDB_DATA_TYPE_BIGINT:
-      case TSDB_DATA_TYPE_UBIGINT: {
-        *(int64_t*)p = *(int64_t*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_FLOAT: {
-        *(float*)p = *(float*)pData;
-        break;
-      }
-      case TSDB_DATA_TYPE_DOUBLE: {
-        *(double*)p = *(double*)pData;
-        break;
-      }
-      default:
-        assert(0);
-    }
   }
 
   return 0;
@@ -175,6 +139,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
 static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
                           int32_t numOfRow2) {
+  if (numOfRow2 <= 0) return;
+
   uint32_t total = numOfRow1 + numOfRow2;
 
   if (BitmapLen(numOfRow1) < BitmapLen(total)) {
@@ -189,22 +155,32 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
 
   if (remindBits == 0) {  // no need to shift bits of bitmap
     memcpy(pColumnInfoData->nullbitmap + BitmapLen(numOfRow1), pSource->nullbitmap, BitmapLen(numOfRow2));
-  } else {
+    return;
-    int32_t len = BitmapLen(numOfRow2);
+  }
-    int32_t i = 0;
 
-    uint8_t* p = (uint8_t*)pSource->nullbitmap;
+  uint8_t* p = (uint8_t*)pSource->nullbitmap;
-    pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits);
+  pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits);  // copy remind bits
 
-    uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
+  if (BitmapLen(numOfRow1) == BitmapLen(total)) {
-    while (i < len) {
+    return;
-      start[i] |= (p[i] << shiftBits);
+  }
-      i += 1;
 
-      if (i > 1) {
+  int32_t len = BitmapLen(numOfRow2);
-        start[i - 1] |= (p[i] >> remindBits);
+  int32_t i = 0;
-      }
+
+  uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
+  int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1);
+  while (i < len) { // size limit of pSource->nullbitmap
+    if (i >= 1) {
+      start[i - 1] |= (p[i] >> remindBits); //copy remind bits
     }
+
+    if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap
+      return;
+    }
+
+    start[i] |= (p[i] << shiftBits);  //copy shift bits
+    i += 1;
   }
 }
 
@@ -216,6 +192,9 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     return numOfRow1;
   }
 
+  if (pSource->hasNull) {
+    pColumnInfoData->hasNull = pSource->hasNull;
+  }
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     // Handle the bitmap
     char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
@@ -336,13 +315,18 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
   return 0;
 }
 
-int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
+// if pIndexMap = NULL, merger one column by on column
-  assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols);
+int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) {
+  assert(pSrc != NULL && pDest != NULL);
 
-  int32_t numOfCols = pSrc->info.numOfCols;
+  int32_t numOfCols = pDest->info.numOfCols;
   for (int32_t i = 0; i < numOfCols; ++i) {
+    int32_t mapIndex = i;
+    if(pIndexMap) {
+      mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
+    }
     SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
-    SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
+    SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex);
 
     uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows);
     uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows);
@@ -399,49 +383,49 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
   // TODO speedup by checking if the whole page can fit in firstly.
   if (!hasVarCol) {
     size_t  rowSize = blockDataGetRowSize(pBlock);
-    int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8;
+    int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0);
+    ASSERT(capacity > 0);
 
-    *stopIndex = startIndex + capacity;
+    *stopIndex = startIndex + capacity - 1;
     if (*stopIndex >= numOfRows) {
       *stopIndex = numOfRows - 1;
     }
 
     return TSDB_CODE_SUCCESS;
-  } else {
+  }
-    // iterate the rows that can be fit in this buffer page
+  // iterate the rows that can be fit in this buffer page
-    int32_t size = (headerSize + colHeaderSize);
+  int32_t size = (headerSize + colHeaderSize);
-
+  for (int32_t j = startIndex; j < numOfRows; ++j) {
-    for (int32_t j = startIndex; j < numOfRows; ++j) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
-      for (int32_t i = 0; i < numOfCols; ++i) {
+      SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i);
-        SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i);
+      if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-        if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+        if (pColInfoData->varmeta.offset[j] != -1) {
-          if (pColInfoData->varmeta.offset[j] != -1) {
+          char* p = colDataGetData(pColInfoData, j);
-            char* p = colDataGetData(pColInfoData, j);
+          size += varDataTLen(p);
-            size += varDataTLen(p);
-          }
-
-          size += sizeof(pColInfoData->varmeta.offset[0]);
-        } else {
-          size += pColInfoData->info.bytes;
-
-          if (((j - startIndex) & 0x07) == 0) {
-            size += 1;  // the space for null bitmap
-          }
         }
-      }
 
-      if (size > pageSize) {
+        size += sizeof(pColInfoData->varmeta.offset[0]);
-        *stopIndex = j - 1;
+      } else {
-        ASSERT(*stopIndex > startIndex);
+        size += pColInfoData->info.bytes;
 
-        return TSDB_CODE_SUCCESS;
+        if (((j - startIndex) & 0x07) == 0) {
+          size += 1;  // the space for null bitmap
+        }
       }
     }
 
-    // all fit in
+    if (size > pageSize) {  // pageSize must be able to hold one row
-    *stopIndex = numOfRows - 1;
+      *stopIndex = j - 1;
-    return TSDB_CODE_SUCCESS;
+      ASSERT(*stopIndex >= startIndex);
+
+      return TSDB_CODE_SUCCESS;
+    }
   }
+
+  // all fit in
+  *stopIndex = numOfRows - 1;
+  return TSDB_CODE_SUCCESS;
+
 }
 
 SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) {
@@ -546,6 +530,11 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
 
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       size_t metaSize = pBlock->info.rows * sizeof(int32_t);
+      char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize);    // preview calloc is too small
+      if (tmp == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      pCol->varmeta.offset = (int32_t*)tmp;
       memcpy(pCol->varmeta.offset, pStart, metaSize);
       pStart += metaSize;
     } else {
@@ -709,23 +698,13 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
     void* left1 = colDataGetData(pColInfoData, left);
     void* right1 = colDataGetData(pColInfoData, right);
 
-    switch (pColInfoData->info.type) {
+    __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
-      case TSDB_DATA_TYPE_INT: {
-        int32_t leftx = *(int32_t*)left1;
-        int32_t rightx = *(int32_t*)right1;
 
-        if (leftx == rightx) {
+    int ret = fn(left1, right1);
-          break;
+    if (ret == 0) {
-        } else {
+      continue;
-          if (pOrder->order == TSDB_ORDER_ASC) {
+    } else {
-            return (leftx < rightx) ? -1 : 1;
+      return ret;
-          } else {
-            return (leftx < rightx) ? 1 : -1;
-          }
-        }
-      }
-      default:
-        assert(0);
     }
   }
 
@@ -779,61 +758,12 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
         pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]];
       }
     } else {
-      switch (pSrc->info.type) {
+      for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
-        case TSDB_DATA_TYPE_UINT:
+        if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-        case TSDB_DATA_TYPE_INT: {
+          colDataSetNull_f(pDst->nullbitmap, j);
-          for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
+          continue;
-            int32_t* p = (int32_t*)pDst->pData;
-            int32_t* srclist = (int32_t*)pSrc->pData;
-
-            p[j] = srclist[index[j]];
-            if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-              colDataSetNull_f(pDst->nullbitmap, j);
-            }
-          }
-          break;
         }
-        case TSDB_DATA_TYPE_UTINYINT:
+        memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes);
-        case TSDB_DATA_TYPE_TINYINT: {
-          for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
-            int32_t* p = (int32_t*)pDst->pData;
-            int32_t* srclist = (int32_t*)pSrc->pData;
-
-            p[j] = srclist[index[j]];
-            if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-              colDataSetNull_f(pDst->nullbitmap, j);
-            }
-          }
-          break;
-        }
-        case TSDB_DATA_TYPE_USMALLINT:
-        case TSDB_DATA_TYPE_SMALLINT: {
-          for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
-            int32_t* p = (int32_t*)pDst->pData;
-            int32_t* srclist = (int32_t*)pSrc->pData;
-
-            p[j] = srclist[index[j]];
-            if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-              colDataSetNull_f(pDst->nullbitmap, j);
-            }
-          }
-          break;
-        }
-        case TSDB_DATA_TYPE_UBIGINT:
-        case TSDB_DATA_TYPE_BIGINT: {
-          for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
-            int32_t* p = (int32_t*)pDst->pData;
-            int32_t* srclist = (int32_t*)pSrc->pData;
-
-            p[j] = srclist[index[j]];
-            if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
-              colDataSetNull_f(pDst->nullbitmap, j);
-            }
-          }
-          break;
-        }
-        default:
-          assert(0);
       }
     }
   }
@@ -906,33 +836,6 @@ static int32_t* createTupleIndex(size_t rows) {
 
 static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); }
 
-static __compar_fn_t getComparFn(int32_t type, int32_t order) {
-  switch (type) {
-    case TSDB_DATA_TYPE_TINYINT:
-      return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc;
-    case TSDB_DATA_TYPE_SMALLINT:
-      return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc;
-    case TSDB_DATA_TYPE_INT:
-      return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
-    case TSDB_DATA_TYPE_BIGINT:
-      return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc;
-    case TSDB_DATA_TYPE_FLOAT:
-      return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc;
-    case TSDB_DATA_TYPE_DOUBLE:
-      return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc;
-    case TSDB_DATA_TYPE_UTINYINT:
-      return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc;
-    case TSDB_DATA_TYPE_USMALLINT:
-      return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc;
-    case TSDB_DATA_TYPE_UINT:
-      return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc;
-    case TSDB_DATA_TYPE_UBIGINT:
-      return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc;
-    default:
-      return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
-  }
-}
-
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
   ASSERT(pDataBlock != NULL && pOrderInfo != NULL);
   if (pDataBlock->info.rows <= 1) {
@@ -966,11 +869,11 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
 
         int64_t p0 = taosGetTimestampUs();
 
-        __compar_fn_t fn = getComparFn(pColInfoData->info.type, pOrder->order);
+        __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
         qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
 
         int64_t p1 = taosGetTimestampUs();
-        printf("sort:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
+        uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
 
         return TSDB_CODE_SUCCESS;
       } else {  // var data type
@@ -999,24 +902,21 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
 
   SColumnInfoData* pCols = createHelpColInfoData(pDataBlock);
   if (pCols == NULL) {
+    destroyTupleIndex(index);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return terrno;
   }
 
   int64_t p2 = taosGetTimestampUs();
 
-  int32_t code = blockDataAssign(pCols, pDataBlock, index);
+  blockDataAssign(pCols, pDataBlock, index);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    return code;
-  }
 
   int64_t p3 = taosGetTimestampUs();
 
   copyBackToBlock(pDataBlock, pCols);
   int64_t p4 = taosGetTimestampUs();
 
-  printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
+  uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
          p3 - p2, p4 - p3, rows);
   destroyTupleIndex(index);
 
@@ -1252,6 +1152,9 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
 }
 
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
+  if(pDataBlock == NULL){
+    return NULL;
+  }
   int32_t numOfCols = pDataBlock->info.numOfCols;
 
   SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));

source/common/src/tglobal.c

@@ -394,7 +394,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeQueryThreads = tsNumOfCores / 2;
-  tsNumOfVnodeQueryThreads = TMIN(tsNumOfVnodeQueryThreads, 1);
+  tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1);
   if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeFetchThreads = tsNumOfCores / 2;
@@ -402,11 +402,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeWriteThreads = tsNumOfCores;
-  tsNumOfVnodeWriteThreads = TMIN(tsNumOfVnodeWriteThreads, 1);
+  tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
   if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeSyncThreads = tsNumOfCores / 2;
-  tsNumOfVnodeSyncThreads = TMIN(tsNumOfVnodeSyncThreads, 1);
+  tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1);
   if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeMergeThreads = tsNumOfCores / 8;
@@ -414,7 +414,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfQnodeQueryThreads = tsNumOfCores / 2;
-  tsNumOfQnodeQueryThreads = TMIN(tsNumOfQnodeQueryThreads, 1);
+  tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1);
   if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfQnodeFetchThreads = tsNumOfCores / 2;

source/common/src/tmsg.c

@@ -1515,6 +1515,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
   if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
@@ -1524,6 +1525,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->singleSTable) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1;
   for (int32_t i = 0; i < pReq->numOfRetensions; ++i) {
     SRetention *pRetension = taosArrayGet(pReq->pRetensions, i);
@@ -1556,6 +1558,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
   if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
@@ -1565,6 +1568,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->singleSTable) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1;
   pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention));
   if (pReq->pRetensions == NULL) {
@@ -1942,6 +1946,94 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) {
   taosArrayDestroy(pRsp->pArray);
 }
 
+int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->cacheBlockSize) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->totalBlocks) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->commitTime) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->fsyncPeriod) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->quorum) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->update) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->cacheBlockSize) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->totalBlocks) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->commitTime) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->fsyncPeriod) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->quorum) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1;
+
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+
 int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -2553,6 +2645,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
 int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
   taosArrayDestroy(pReq->pRetensions);
   pReq->pRetensions = NULL;
+  return 0;
 }
 
 int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {

source/dnode/mgmt/bm/bmHandle.c

@@ -16,6 +16,34 @@
 #define _DEFAULT_SOURCE
 #include "bmInt.h"
 
+void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo) {
+}
+
+int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonBmInfo bmInfo = {0};
+  bmGetMonitorInfo(pWrapper, &bmInfo);
+  dmGetMonitorSysInfo(&bmInfo.sys);
+  monGetLogs(&bmInfo.log);
+
+  int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonBmInfo(pRsp, rspLen, &bmInfo);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonBmInfo(&bmInfo);
+  return 0;
+}
+
 int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnode  *pDnode = pWrapper->pDnode;
   SRpcMsg *pReq = &pMsg->rpcMsg;
@@ -54,4 +82,6 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   }
 }
 
-void bmInitMsgHandle(SMgmtWrapper *pWrapper) {}
+void bmInitMsgHandle(SMgmtWrapper *pWrapper) {
+  dndSetMsgHandle(pWrapper, TDMT_MON_BM_INFO, bmProcessMonitorMsg, DEFAULT_HANDLE);
+}

source/dnode/mgmt/bm/bmWorker.c

@@ -33,7 +33,37 @@ static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t num
   }
 }
 
-static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+static inline void bmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
+                 .ahandle = pMsg->rpcMsg.ahandle,
+                 .code = code,
+                 .pCont = pMsg->pRsp,
+                 .contLen = pMsg->rspLen};
+  tmsgSendRsp(&rsp);
+}
+
+static void bmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
+  SBnodeMgmt *pMgmt = pInfo->ahandle;
+
+  dTrace("msg:%p, get from bnode monitor queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = -1;
+
+  if (pMsg->rpcMsg.msgType == TDMT_MON_BM_INFO) {
+    code = bmProcessGetMonBmInfoReq(pMgmt->pWrapper, pMsg);
+  }
+
+  if (pRpc->msgType & 1U) {
+    if (code != 0 && terrno != 0) code = terrno;
+    bmSendRsp(pMgmt->pWrapper, pMsg, code);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
+static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SBnodeMgmt   *pMgmt = pInfo->ahandle;
   SMgmtWrapper *pWrapper = pMgmt->pWrapper;
 
@@ -72,18 +102,37 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
+int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SBnodeMgmt    *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
 int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
-  SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt};
+  SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessWriteQueue, .param = pMgmt};
   if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) {
-    dError("failed to start bnode write worker since %s", terrstr());
+    dError("failed to start bnode-write worker since %s", terrstr());
     return -1;
   }
 
+  if (tsMultiProcess) {
+    SSingleWorkerCfg mCfg = {
+        .min = 1, .max = 1, .name = "bnode-monitor", .fp = (FItem)bmProcessMonQueue, .param = pMgmt};
+    if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
+      dError("failed to start bnode-monitor worker since %s", terrstr());
+      return -1;
+    }
+   }
+
   dDebug("bnode workers are initialized");
   return 0;
 }
 
 void bmStopWorker(SBnodeMgmt *pMgmt) {
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tMultiWorkerCleanup(&pMgmt->writeWorker);
   dDebug("bnode workers are closed");
 }

source/dnode/mgmt/dm/dmHandle.c

@@ -42,8 +42,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
 
   SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
   if (pWrapper != NULL) {
-    req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
+    SMonVloadInfo info = {0};
-    vmMonitorVnodeLoads(pWrapper, req.pVloads);
+    dmGetVnodeLoads(pWrapper, &info);
+    req.pVloads = info.pVloads;
     dndReleaseWrapper(pWrapper);
   }
 
@@ -96,6 +97,7 @@ int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
   }
 
   pMgmt->statusSent = 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
@@ -117,7 +119,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
   return TSDB_CODE_OPS_NOT_SUPPORT;
 }
 
-
 static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, EDndType ntype, SNodeMsg *pMsg) {
   SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
   if (pWrapper != NULL) {
@@ -209,7 +210,7 @@ void dmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, DEFAULT_HANDLE);
 
   // Requests handled by MNODE
-  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessStatusMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMonitorMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/dm/dmMonitor.c

@@ -14,21 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dndInt.h"
+#include "dmInt.h"
-
-static int32_t dmGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
-  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
-  pInfo->logdir.size = tsLogSpace.size;
-  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
-  pInfo->tempdir.size = tsTempSpace.size;
-
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
-  if (pWrapper != NULL) {
-    vmMonitorTfsInfo(pWrapper, pInfo);
-    dndReleaseWrapper(pWrapper);
-  }
-  return 0;
-}
 
 static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   pInfo->protocol = 1;
@@ -39,6 +25,163 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
 
 static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
+  if (pWrapper != NULL) {
+    pInfo->has_mnode = pWrapper->required;
+    dndReleaseWrapper(pWrapper);
+  }
+  tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
+  pInfo->logdir.size = tsLogSpace.size;
+  tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
+  pInfo->tempdir.size = tsTempSpace.size;
+}
+
+static void dmGetMonitorInfo(SDnode *pDnode, SMonDmInfo *pInfo) {
+  dmGetMonitorBasicInfo(pDnode, &pInfo->basic);
+  dmGetMonitorSysInfo(&pInfo->sys);
+  dmGetMonitorDnodeInfo(pDnode, &pInfo->dnode);
+}
+
+void dmSendMonitorReport(SDnode *pDnode) {
+  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+  SMonDmInfo dmInfo = {0};
+  SMonMmInfo mmInfo = {0};
+  SMonVmInfo vmInfo = {0};
+  SMonQmInfo qmInfo = {0};
+  SMonSmInfo smInfo = {0};
+  SMonBmInfo bmInfo = {0};
+
+  SRpcMsg req = {0};
+  SRpcMsg rsp;
+  SEpSet  epset = {.inUse = 0, .numOfEps = 1};
+  tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+  epset.eps[0].port = tsServerPort;
+
+  SMgmtWrapper *pWrapper = NULL;
+  dmGetMonitorInfo(pDnode, &dmInfo);
+
+  bool getFromAPI = !tsMultiProcess;
+  pWrapper = &pDnode->wrappers[MNODE];
+  if (getFromAPI) {
+    if (dndMarkWrapper(pWrapper) != 0) {
+      mmGetMonitorInfo(pWrapper, &mmInfo);
+      dndReleaseWrapper(pWrapper);
+    }
+  } else {
+    if (pWrapper->required) {
+      req.msgType = TDMT_MON_MM_INFO;
+      dndSendRecv(pDnode, &epset, &req, &rsp);
+      if (rsp.code == 0 && rsp.contLen > 0) {
+        tDeserializeSMonMmInfo(rsp.pCont, rsp.contLen, &mmInfo);
+      }
+      rpcFreeCont(rsp.pCont);
+    }
+  }
+
+  pWrapper = &pDnode->wrappers[VNODES];
+  if (getFromAPI) {
+    if (dndMarkWrapper(pWrapper) != 0) {
+      vmGetMonitorInfo(pWrapper, &vmInfo);
+      dndReleaseWrapper(pWrapper);
+    }
+  } else {
+    if (pWrapper->required) {
+      req.msgType = TDMT_MON_VM_INFO;
+      dndSendRecv(pDnode, &epset, &req, &rsp);
+      if (rsp.code == 0 && rsp.contLen > 0) {
+        tDeserializeSMonVmInfo(rsp.pCont, rsp.contLen, &vmInfo);
+      }
+      rpcFreeCont(rsp.pCont);
+    }
+  }
+
+  pWrapper = &pDnode->wrappers[QNODE];
+  if (getFromAPI) {
+    if (dndMarkWrapper(pWrapper) != 0) {
+      qmGetMonitorInfo(pWrapper, &qmInfo);
+      dndReleaseWrapper(pWrapper);
+    }
+  } else {
+    if (pWrapper->required) {
+      req.msgType = TDMT_MON_QM_INFO;
+      dndSendRecv(pDnode, &epset, &req, &rsp);
+      if (rsp.code == 0 && rsp.contLen > 0) {
+        tDeserializeSMonQmInfo(rsp.pCont, rsp.contLen, &qmInfo);
+      }
+      rpcFreeCont(rsp.pCont);
+    }
+  }
+
+  pWrapper = &pDnode->wrappers[SNODE];
+  if (getFromAPI) {
+    if (dndMarkWrapper(pWrapper) != 0) {
+      smGetMonitorInfo(pWrapper, &smInfo);
+      dndReleaseWrapper(pWrapper);
+    }
+  } else {
+    if (pWrapper->required) {
+      req.msgType = TDMT_MON_SM_INFO;
+      dndSendRecv(pDnode, &epset, &req, &rsp);
+      if (rsp.code == 0 && rsp.contLen > 0) {
+        tDeserializeSMonSmInfo(rsp.pCont, rsp.contLen, &smInfo);
+      }
+      rpcFreeCont(rsp.pCont);
+    }
+  }
+
+  pWrapper = &pDnode->wrappers[BNODE];
+  if (getFromAPI) {
+    if (dndMarkWrapper(pWrapper) != 0) {
+      bmGetMonitorInfo(pWrapper, &bmInfo);
+      dndReleaseWrapper(pWrapper);
+    }
+  } else {
+    if (pWrapper->required) {
+      req.msgType = TDMT_MON_BM_INFO;
+      dndSendRecv(pDnode, &epset, &req, &rsp);
+      if (rsp.code == 0 && rsp.contLen > 0) {
+        tDeserializeSMonBmInfo(rsp.pCont, rsp.contLen, &bmInfo);
+      }
+      rpcFreeCont(rsp.pCont);
+    }
+  }
+
+  monSetDmInfo(&dmInfo);
+  monSetMmInfo(&mmInfo);
+  monSetVmInfo(&vmInfo);
+  monSetQmInfo(&qmInfo);
+  monSetSmInfo(&smInfo);
+  monSetBmInfo(&bmInfo);
+  tFreeSMonMmInfo(&mmInfo);
+  tFreeSMonVmInfo(&vmInfo);
+  tFreeSMonQmInfo(&qmInfo);
+  tFreeSMonSmInfo(&smInfo);
+  tFreeSMonBmInfo(&bmInfo);
+  monSendReport();
+}
+
+void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) {
+  bool getFromAPI = !tsMultiProcess;
+  if (getFromAPI) {
+    vmGetVnodeLoads(pWrapper, pInfo);
+  } else {
+    SRpcMsg req = {.msgType = TDMT_MON_VM_LOAD};
+    SRpcMsg rsp = {0};
+    SEpSet  epset = {.inUse = 0, .numOfEps = 1};
+    tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
+    epset.eps[0].port = tsServerPort;
+
+    dndSendRecv(pWrapper->pDnode, &epset, &req, &rsp);
+    if (rsp.code == 0 && rsp.contLen > 0) {
+      tDeserializeSMonVloadInfo(rsp.pCont, rsp.contLen, pInfo);
+    }
+    rpcFreeCont(rsp.pCont);
+  }
+}
+
+void dmGetMonitorSysInfo(SMonSysInfo *pInfo) {
   taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
   taosGetCpuCores(&pInfo->cpu_cores);
   taosGetProcMemory(&pInfo->mem_engine);
@@ -47,61 +190,6 @@ static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   pInfo->disk_engine = 0;
   pInfo->disk_used = tsDataSpace.size.used;
   pInfo->disk_total = tsDataSpace.size.total;
-  taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
+  taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out);
-  taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
+  taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
-
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
-  if (pWrapper != NULL) {
-    vmMonitorVnodeReqs(pWrapper, pInfo);
-    dndReleaseWrapper(pWrapper);
-  }
-
-  pWrapper = dndAcquireWrapper(pDnode, MNODE);
-  if (pWrapper != NULL) {
-    pInfo->has_mnode = pWrapper->required;
-    dndReleaseWrapper(pWrapper);
-  }
 }
-
-void dmSendMonitorReport(SDnode *pDnode) {
-  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
-  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
-
-  SMonInfo *pMonitor = monCreateMonitorInfo();
-  if (pMonitor == NULL) return;
-
-  SMonBasicInfo basicInfo = {0};
-  dmGetMonitorBasicInfo(pDnode, &basicInfo);
-  monSetBasicInfo(pMonitor, &basicInfo);
-
-  SMonClusterInfo clusterInfo = {0};
-  SMonVgroupInfo  vgroupInfo = {0};
-  SMonGrantInfo   grantInfo = {0};
-
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
-  if (pWrapper != NULL) {
-    if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
-      monSetClusterInfo(pMonitor, &clusterInfo);
-      monSetVgroupInfo(pMonitor, &vgroupInfo);
-      monSetGrantInfo(pMonitor, &grantInfo);
-    }
-    dndReleaseWrapper(pWrapper);
-  }
-
-  SMonDnodeInfo dnodeInfo = {0};
-  dmGetMonitorDnodeInfo(pDnode, &dnodeInfo);
-  monSetDnodeInfo(pMonitor, &dnodeInfo);
-
-  SMonDiskInfo diskInfo = {0};
-  if (dmGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
-    monSetDiskInfo(pMonitor, &diskInfo);
-  }
-
-  taosArrayDestroy(clusterInfo.dnodes);
-  taosArrayDestroy(clusterInfo.mnodes);
-  taosArrayDestroy(vgroupInfo.vgroups);
-  taosArrayDestroy(diskInfo.datadirs);
-
-  monSendReport(pMonitor);
-  monCleanupMonitorInfo(pMonitor);
-}

source/dnode/mgmt/dm/dmWorker.c

@@ -44,6 +44,7 @@ static void *dmThreadRoutine(void *param) {
       lastMonitorTime = curTime;
     }
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t dmStartThread(SDnodeMgmt *pMgmt) {
@@ -101,9 +102,9 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
     return -1;
   }
 
-  SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
+  SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-monitor", .fp = (FItem)dmProcessQueue, .param = pMgmt};
-  if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) {
+  if (tSingleWorkerInit(&pMgmt->monitorWorker, &scfg) != 0) {
-    dError("failed to start dnode status worker since %s", terrstr());
+    dError("failed to start dnode monitor worker since %s", terrstr());
     return -1;
   }
 
@@ -113,7 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
 
 void dmStopWorker(SDnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
-  tSingleWorkerCleanup(&pMgmt->statusWorker);
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
 
   if (pMgmt->threadId != NULL) {
     taosDestoryThread(pMgmt->threadId);
@@ -131,9 +132,9 @@ int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
-int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnodeMgmt    *pMgmt = pWrapper->pMgmt;
-  SSingleWorker *pWorker = &pMgmt->statusWorker;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
 
   dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
   taosWriteQitem(pWorker->queue, pMsg);

source/dnode/mgmt/inc/bmInt.h

@@ -17,6 +17,7 @@
 #define _TD_DND_BNODE_INT_H_
 
 #include "dndInt.h"
+
 #include "bnode.h"
 
 #ifdef __cplusplus
@@ -29,6 +30,7 @@ typedef struct SBnodeMgmt {
   SMgmtWrapper *pWrapper;
   const char   *path;
   SMultiWorker  writeWorker;
+  SSingleWorker monitorWorker;
 } SBnodeMgmt;
 
 // bmInt.c
@@ -39,11 +41,13 @@ int32_t bmDrop(SMgmtWrapper *pWrapper);
 void    bmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
 
 // bmWorker.c
 int32_t bmStartWorker(SBnodeMgmt *pMgmt);
 void    bmStopWorker(SBnodeMgmt *pMgmt);
 int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/inc/dmInt.h

@@ -32,7 +32,7 @@ typedef struct SDnodeMgmt {
   TdThread     *threadId;
   SRWLatch      latch;
   SSingleWorker mgmtWorker;
-  SSingleWorker statusWorker;
+  SSingleWorker monitorWorker;
   SMsgCb        msgCb;
   const char   *path;
   SDnode       *pDnode;
@@ -54,6 +54,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
 int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg);
 
 // dmMonitor.c
+void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo);
 void dmSendMonitorReport(SDnode *pDnode);
 
 // dmWorker.c
@@ -61,7 +62,7 @@ int32_t dmStartThread(SDnodeMgmt *pMgmt);
 int32_t dmStartWorker(SDnodeMgmt *pMgmt);
 void    dmStopWorker(SDnodeMgmt *pMgmt);
 int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
-int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/inc/dndInt.h

@@ -169,6 +169,7 @@ void     dndCleanupTrans(SDnode *pDnode);
 SMsgCb   dndCreateMsgcb(SMgmtWrapper *pWrapper);
 SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper);
 int32_t  dndInitMsgHandle(SDnode *pDnode);
+void     dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
 
 // mgmt
 void dmSetMgmtFp(SMgmtWrapper *pWrapper);
@@ -182,22 +183,13 @@ void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
 void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
 void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
 
-typedef struct {
+void dmGetMonitorSysInfo(SMonSysInfo *pInfo);
-  int32_t openVnodes;
+void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo);
-  int32_t totalVnodes;
+void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo);
-  int32_t masterNum;
+void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo);
-  int64_t numOfSelectReqs;
+void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo);
-  int64_t numOfInsertReqs;
+void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo);
-  int64_t numOfInsertSuccessReqs;
+void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo);
-  int64_t numOfBatchInsertReqs;
-  int64_t numOfBatchInsertSuccessReqs;
-} SVnodesStat;
-
-void    vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads);
-int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo);
-void    vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo);
-int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                           SMonGrantInfo *pGrantInfo);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/inc/mmInt.h

@@ -32,6 +32,7 @@ typedef struct SMnodeMgmt {
   SSingleWorker readWorker;
   SSingleWorker writeWorker;
   SSingleWorker syncWorker;
+  SSingleWorker monitorWorker;
   SReplica      replicas[TSDB_MAX_REPLICA];
   int8_t        replica;
   int8_t        selfIndex;
@@ -51,6 +52,7 @@ void    mmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
 
 // mmWorker.c
 int32_t mmStartWorker(SMnodeMgmt *pMgmt);
@@ -59,6 +61,7 @@ int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
 int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
 int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);

source/dnode/mgmt/inc/qmInt.h

@@ -30,6 +30,7 @@ typedef struct SQnodeMgmt {
   const char   *path;
   SSingleWorker queryWorker;
   SSingleWorker fetchWorker;
+  SSingleWorker monitorWorker;
 } SQnodeMgmt;
 
 // qmInt.c
@@ -40,6 +41,7 @@ int32_t qmDrop(SMgmtWrapper *pWrapper);
 void    qmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
 
 // qmWorker.c
 int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
@@ -50,6 +52,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt);
 void    qmStopWorker(SQnodeMgmt *pMgmt);
 int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/inc/smInt.h

@@ -32,6 +32,7 @@ typedef struct SSnodeMgmt {
   int8_t        uniqueWorkerInUse;
   SArray       *uniqueWorkers;  // SArray<SMultiWorker*>
   SSingleWorker sharedWorker;
+  SSingleWorker monitorWorker;
 } SSnodeMgmt;
 
 // smInt.c
@@ -42,6 +43,7 @@ int32_t smDrop(SMgmtWrapper *pWrapper);
 void    smInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
 
 // smWorker.c
 int32_t smStartWorker(SSnodeMgmt *pMgmt);
@@ -50,6 +52,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/inc/vmInt.h

@@ -28,6 +28,7 @@ typedef struct SVnodesMgmt {
   SHashObj     *hash;
   SRWLatch      latch;
   SVnodesStat   state;
+  SVnodesStat   lastState;
   STfs         *pTfs;
   SQWorkerPool  queryPool;
   SQWorkerPool  fetchPool;
@@ -38,6 +39,7 @@ typedef struct SVnodesMgmt {
   SDnode       *pDnode;
   SMgmtWrapper *pWrapper;
   SSingleWorker mgmtWorker;
+  SSingleWorker monitorWorker;
 } SVnodesMgmt;
 
 typedef struct {
@@ -91,6 +93,8 @@ int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
 int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
 int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
 int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
+int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
+int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
 
 // vmFile.c
 int32_t     vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
@@ -114,6 +118,7 @@ int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrappert, SNodeMsg *pMsg);
+int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/main/dndTransport.c

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-  
+
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 
@@ -482,4 +482,8 @@ SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
                   .parent = pWrapper,
                   .name = pWrapper->name};
   return cfg;
+}
+
+void dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) {
+  rpcSendRecv(pDnode->trans.clientRpc, pEpSet, pReq, pRsp);
 }

source/dnode/mgmt/mm/mmHandle.c

@@ -16,6 +16,36 @@
 #define _DEFAULT_SOURCE
 #include "mmInt.h"
 
+void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo) {
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant);
+}
+
+int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonMmInfo mmInfo = {0};
+  mmGetMonitorInfo(pWrapper, &mmInfo);
+  dmGetMonitorSysInfo(&mmInfo.sys);
+  monGetLogs(&mmInfo.log);
+
+  int32_t rspLen = tSerializeSMonMmInfo(NULL, 0, &mmInfo);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonMmInfo(pRsp, rspLen, &mmInfo);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonMmInfo(&mmInfo);
+  return 0;
+}
+
 int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnode  *pDnode = pWrapper->pDnode;
   SRpcMsg *pReq = &pMsg->rpcMsg;
@@ -74,6 +104,8 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
 }
 
 void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
+  dndSetMsgHandle(pWrapper, TDMT_MON_MM_INFO, mmProcessMonitorMsg, DEFAULT_HANDLE);
+
   // Requests handled by DNODE
   dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
@@ -146,6 +178,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);
 
   // Requests handled by VNODE
   dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
@@ -162,5 +195,4 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE);
-
 }

source/dnode/mgmt/mm/mmInt.c

@@ -241,8 +241,3 @@ void mmSetMgmtFp(SMgmtWrapper *pWrapper) {
   pWrapper->fp = mgmtFp;
 }
 
-int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                           SMonGrantInfo *pGrantInfo) {
-  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
-  return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
-}

source/dnode/mgmt/mm/mmWorker.c

@@ -23,19 +23,18 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   int32_t  code = -1;
 
-  if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) {
+  if (pMsg->rpcMsg.msgType == TDMT_DND_ALTER_MNODE) {
+    code = mmProcessAlterReq(pMgmt, pMsg);
+  } else if (pMsg->rpcMsg.msgType == TDMT_MON_MM_INFO) {
+    code = mmProcessGetMonMmInfoReq(pMgmt->pWrapper, pMsg);
+  } else {
     pMsg->pNode = pMgmt->pMnode;
     code = mndProcessMsg(pMsg);
-  } else {
-    code = mmProcessAlterReq(pMgmt, pMsg);
   }
 
   if (pRpc->msgType & 1U) {
     if (pRpc->handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-      if (code != 0) {
+      if (code != 0 && terrno != 0) code = terrno;
-        code = terrno;
-        dError("msg:%p, failed to process since %s", pMsg, terrstr());
-      }
       SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
       tmsgSendRsp(&rsp);
     }
@@ -98,6 +97,15 @@ int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
+int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SMnodeMgmt    *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
 static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) {
   SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
   if (pMsg == NULL) return -1;
@@ -157,15 +165,24 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
 
   SSingleWorkerCfg sCfg = {.min = 1, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt};
   if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
-    dError("failed to start mnode sync-worker since %s", terrstr());
+    dError("failed to start mnode mnode-sync worker since %s", terrstr());
     return -1;
   }
 
+  if (tsMultiProcess) {
+    SSingleWorkerCfg mCfg = {.min = 1, .max = 1, .name = "mnode-monitor", .fp = (FItem)mmProcessQueue, .param = pMgmt};
+    if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
+      dError("failed to start mnode mnode-monitor worker since %s", terrstr());
+      return -1;
+    }
+  }
+
   dDebug("mnode workers are initialized");
   return 0;
 }
 
 void mmStopWorker(SMnodeMgmt *pMgmt) {
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->queryWorker);
   tSingleWorkerCleanup(&pMgmt->readWorker);
   tSingleWorkerCleanup(&pMgmt->writeWorker);

source/dnode/mgmt/qm/qmHandle.c

@@ -16,6 +16,34 @@
 #define _DEFAULT_SOURCE
 #include "qmInt.h"
 
+void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo) {
+}
+
+int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonQmInfo qmInfo = {0};
+  qmGetMonitorInfo(pWrapper, &qmInfo);
+  dmGetMonitorSysInfo(&qmInfo.sys);
+  monGetLogs(&qmInfo.log);
+
+  int32_t rspLen = tSerializeSMonQmInfo(NULL, 0, &qmInfo);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonQmInfo(pRsp, rspLen, &qmInfo);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonQmInfo(&qmInfo);
+  return 0;
+}
+
 int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnode  *pDnode = pWrapper->pDnode;
   SRpcMsg *pReq = &pMsg->rpcMsg;
@@ -55,6 +83,8 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
 }
 
 void qmInitMsgHandle(SMgmtWrapper *pWrapper) {
+  dndSetMsgHandle(pWrapper, TDMT_MON_QM_INFO, qmProcessMonitorMsg, DEFAULT_HANDLE);
+
   // Requests handled by VNODE
   dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE);

source/dnode/mgmt/qm/qmWorker.c

@@ -16,11 +16,36 @@
 #define _DEFAULT_SOURCE
 #include "qmInt.h"
 
-static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+static inline void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
-  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
+  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
+                 .ahandle = pMsg->rpcMsg.ahandle,
+                 .code = code,
+                 .pCont = pMsg->pRsp,
+                 .contLen = pMsg->rspLen};
   tmsgSendRsp(&rsp);
 }
 
+static void qmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
+  SQnodeMgmt *pMgmt = pInfo->ahandle;
+
+  dTrace("msg:%p, get from qnode monitor queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = -1;
+
+  if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) {
+    code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg);
+  }
+
+  if (pRpc->msgType & 1U) {
+    if (code != 0 && terrno != 0) code = terrno;
+    qmSendRsp(pMgmt->pWrapper, pMsg, code);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
 static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SQnodeMgmt *pMgmt = pInfo->ahandle;
 
@@ -66,6 +91,15 @@ int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
+int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SQnodeMgmt    *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
 static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
   SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
   if (pMsg == NULL) {
@@ -128,11 +162,21 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
     return -1;
   }
 
+  if (tsMultiProcess) {
+    SSingleWorkerCfg mCfg = {
+        .min = 1, .max = 1, .name = "qnode-monitor", .fp = (FItem)qmProcessMonQueue, .param = pMgmt};
+    if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
+      dError("failed to start qnode-monitor worker since %s", terrstr());
+      return -1;
+    }
+  }
+
   dDebug("qnode workers are initialized");
   return 0;
 }
 
 void qmStopWorker(SQnodeMgmt *pMgmt) {
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->queryWorker);
   tSingleWorkerCleanup(&pMgmt->fetchWorker);
   dDebug("qnode workers are closed");

source/dnode/mgmt/sm/smHandle.c

@@ -16,6 +16,33 @@
 #define _DEFAULT_SOURCE
 #include "smInt.h"
 
+void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo) {}
+
+int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonSmInfo smInfo = {0};
+  smGetMonitorInfo(pWrapper, &smInfo);
+  dmGetMonitorSysInfo(&smInfo.sys);
+  monGetLogs(&smInfo.log);
+
+  int32_t rspLen = tSerializeSMonSmInfo(NULL, 0, &smInfo);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonSmInfo(pRsp, rspLen, &smInfo);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonSmInfo(&smInfo);
+  return 0;
+}
+
 int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnode  *pDnode = pWrapper->pDnode;
   SRpcMsg *pReq = &pMsg->rpcMsg;
@@ -55,6 +82,8 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
 }
 
 void smInitMsgHandle(SMgmtWrapper *pWrapper) {
+  dndSetMsgHandle(pWrapper, TDMT_MON_SM_INFO, smProcessMonitorMsg, DEFAULT_HANDLE);
+
   // Requests handled by SNODE
   dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE);

source/dnode/mgmt/sm/smWorker.c

@@ -16,6 +16,36 @@
 #define _DEFAULT_SOURCE
 #include "smInt.h"
 
+static inline void smSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
+                 .ahandle = pMsg->rpcMsg.ahandle,
+                 .code = code,
+                 .pCont = pMsg->pRsp,
+                 .contLen = pMsg->rspLen};
+  tmsgSendRsp(&rsp);
+}
+
+static void smProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
+  SSnodeMgmt *pMgmt = pInfo->ahandle;
+
+  dTrace("msg:%p, get from snode monitor queue", pMsg);
+  SRpcMsg *pRpc = &pMsg->rpcMsg;
+  int32_t  code = -1;
+
+  if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) {
+    code = smProcessGetMonSmInfoReq(pMgmt->pWrapper, pMsg);
+  }
+
+  if (pRpc->msgType & 1U) {
+    if (code != 0 && terrno != 0) code = terrno;
+    smSendRsp(pMgmt->pWrapper, pMsg, code);
+  }
+
+  dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
+  rpcFreeCont(pRpc->pCont);
+  taosFreeQitem(pMsg);
+}
+
 static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SSnodeMgmt *pMgmt = pInfo->ahandle;
 
@@ -80,11 +110,21 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
     return -1;
   }
 
+  if (tsMultiProcess) {
+    SSingleWorkerCfg mCfg = {
+        .min = 1, .max = 1, .name = "snode-monitor", .fp = (FItem)smProcessMonitorQueue, .param = pMgmt};
+    if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
+      dError("failed to start snode-monitor worker since %s", terrstr());
+      return -1;
+    }
+  }
+
   dDebug("snode workers are initialized");
   return 0;
 }
 
 void smStopWorker(SSnodeMgmt *pMgmt) {
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
   for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
     SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i);
     tMultiWorkerCleanup(pWorker);
@@ -120,6 +160,15 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
+int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SSnodeMgmt    *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
 int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SSnodeMgmt   *pMgmt = pWrapper->pMgmt;
   int32_t       index = smGetSWIdFromMsg(&pMsg->rpcMsg);

source/dnode/mgmt/vm/vmHandle.c

@@ -16,6 +16,71 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
+void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo) {
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  tfsGetMonitorInfo(pMgmt->pTfs, &vmInfo->tfs);
+
+  taosWLockLatch(&pMgmt->latch);
+  vmInfo->vstat.totalVnodes = pMgmt->state.totalVnodes;
+  vmInfo->vstat.masterNum = pMgmt->state.masterNum;
+  vmInfo->vstat.numOfSelectReqs = pMgmt->state.numOfSelectReqs - pMgmt->lastState.numOfSelectReqs;
+  vmInfo->vstat.numOfInsertReqs = pMgmt->state.numOfInsertReqs - pMgmt->lastState.numOfInsertReqs;
+  vmInfo->vstat.numOfInsertSuccessReqs = pMgmt->state.numOfInsertSuccessReqs - pMgmt->lastState.numOfInsertSuccessReqs;
+  vmInfo->vstat.numOfBatchInsertReqs = pMgmt->state.numOfBatchInsertReqs - pMgmt->lastState.numOfBatchInsertReqs;
+  vmInfo->vstat.numOfBatchInsertSuccessReqs =
+      pMgmt->state.numOfBatchInsertSuccessReqs - pMgmt->lastState.numOfBatchInsertSuccessReqs;
+  pMgmt->lastState = pMgmt->state;
+  taosWUnLockLatch(&pMgmt->latch);
+}
+
+int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonVmInfo vmInfo = {0};
+  vmGetMonitorInfo(pWrapper, &vmInfo);
+  dmGetMonitorSysInfo(&vmInfo.sys);
+  monGetLogs(&vmInfo.log);
+
+  int32_t rspLen = tSerializeSMonVmInfo(NULL, 0, &vmInfo);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonVmInfo(pRsp, rspLen, &vmInfo);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonVmInfo(&vmInfo);
+  return 0;
+}
+
+int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
+  SMonVloadInfo vloads = {0};
+  vmGetVnodeLoads(pWrapper, &vloads);
+
+  int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads);
+  if (rspLen < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  void *pRsp = rpcMallocCont(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  tSerializeSMonVloadInfo(pRsp, rspLen, &vloads);
+  pReq->pRsp = pRsp;
+  pReq->rspLen = rspLen;
+  tFreeSMonVloadInfo(&vloads);
+  return 0;
+}
+
 static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->vgId = pCreate->vgId;
   pCfg->wsize = pCreate->cacheBlockSize;
@@ -239,6 +304,9 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
 }
 
 void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
+  dndSetMsgHandle(pWrapper, TDMT_MON_VM_INFO, vmProcessMonitorMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MON_VM_LOAD, vmProcessMonitorMsg, DEFAULT_HANDLE);
+
   // Requests handled by VNODE
   dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);

source/dnode/mgmt/vm/vmInt.c

@@ -344,38 +344,21 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
   pWrapper->fp = mgmtFp;
 }
 
-int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) {
+void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) {
-  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
-  if (pMgmt == NULL) return -1;
-
-  return tfsGetMonitorInfo(pMgmt->pTfs, pInfo);
-}
-
-void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) {
-  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
-  if (pMgmt == NULL) return;
-
-  SVnodesStat *pStat = &pMgmt->state;
-  pInfo->req_select = pStat->numOfSelectReqs;
-  pInfo->req_insert = pStat->numOfInsertReqs;
-  pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
-  pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
-  pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
-  pInfo->errors = tsNumOfErrorLogs;
-  pInfo->vnodes_num = pStat->totalVnodes;
-  pInfo->masters = pStat->masterNum;
-}
-
-void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
   SVnodesMgmt *pMgmt = pWrapper->pMgmt;
   SVnodesStat *pStat = &pMgmt->state;
-  int32_t      totalVnodes = 0;
+  SArray      *pLoads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad));
-  int32_t      masterNum = 0;
+
-  int64_t      numOfSelectReqs = 0;
+  int32_t totalVnodes = 0;
-  int64_t      numOfInsertReqs = 0;
+  int32_t masterNum = 0;
-  int64_t      numOfInsertSuccessReqs = 0;
+  int64_t numOfSelectReqs = 0;
-  int64_t      numOfBatchInsertReqs = 0;
+  int64_t numOfInsertReqs = 0;
-  int64_t      numOfBatchInsertSuccessReqs = 0;
+  int64_t numOfInsertSuccessReqs = 0;
+  int64_t numOfBatchInsertReqs = 0;
+  int64_t numOfBatchInsertSuccessReqs = 0;
+
+  pInfo->pVloads = pLoads;
+  if (pLoads == NULL) return;
 
   taosRLockLatch(&pMgmt->latch);
 
@@ -402,6 +385,7 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
 
   taosRUnLockLatch(&pMgmt->latch);
 
+  taosWLockLatch(&pMgmt->latch);
   pStat->totalVnodes = totalVnodes;
   pStat->masterNum = masterNum;
   pStat->numOfSelectReqs = numOfSelectReqs;
@@ -409,4 +393,5 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
   pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
   pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
   pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
+  taosWUnLockLatch(&pMgmt->latch);
 }

source/dnode/mgmt/vm/vmWorker.c

@@ -16,8 +16,12 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
-static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
+static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
-  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
+  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
+                 .ahandle = pMsg->rpcMsg.ahandle,
+                 .code = code,
+                 .pCont = pMsg->pRsp,
+                 .contLen = pMsg->rspLen};
   tmsgSendRsp(&rsp);
 }
 
@@ -26,9 +30,15 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 
   int32_t code = -1;
   tmsg_t  msgType = pMsg->rpcMsg.msgType;
-  dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg);
+  dTrace("msg:%p, will be processed in vnode-m queue", pMsg);
 
   switch (msgType) {
+    case TDMT_MON_VM_INFO:
+      code = vmProcessGetMonVmInfoReq(pMgmt->pWrapper, pMsg);
+      break;
+    case TDMT_MON_VM_LOAD:
+      code = vmProcessGetVnodeLoadsReq(pMgmt->pWrapper, pMsg);
+      break;
     case TDMT_DND_CREATE_VNODE:
       code = vmProcessCreateVnodeReq(pMgmt, pMsg);
       break;
@@ -255,6 +265,15 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   return 0;
 }
 
+int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SVnodesMgmt   *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->monitorWorker;
+
+  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
 static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) {
   SVnodesMgmt *pMgmt = pWrapper->pMgmt;
   SMsgHead    *pHead = pRpc->pCont;
@@ -412,11 +431,21 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
     return -1;
   }
 
+  if (tsMultiProcess) {
+    SSingleWorkerCfg mCfg = {
+        .min = 1, .max = 1, .name = "vnode-monitor", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
+    if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
+      dError("failed to start mnode vnode-monitor worker since %s", terrstr());
+      return -1;
+    }
+  }
+
   dDebug("vnode workers are initialized");
   return 0;
 }
 
 void vmStopWorker(SVnodesMgmt *pMgmt) {
+  tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
   tQWorkerCleanup(&pMgmt->fetchPool);
   tQWorkerCleanup(&pMgmt->queryPool);

source/dnode/mnode/impl/inc/mndDef.h

@@ -260,6 +260,7 @@ typedef struct {
   int32_t maxRows;
   int32_t commitTime;
   int32_t fsyncPeriod;
+  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
@@ -268,6 +269,7 @@ typedef struct {
   int8_t  update;
   int8_t  cacheLastRow;
   int8_t  streamMode;
+  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;
 } SDbCfg;

source/dnode/mnode/impl/src/mndDb.c

@@ -40,6 +40,7 @@ static int32_t  mndProcessCompactDbReq(SNodeMsg *pReq);
 static int32_t  mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
 static int32_t  mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
 static void     mndCancelGetNextDb(SMnode *pMnode, void *pIter);
+static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq);
 
 int32_t mndInitDb(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_DB,
@@ -56,6 +57,7 @@ int32_t mndInitDb(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq);
   mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq);
   mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
 
   mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta);
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
@@ -268,6 +270,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->minRows > pCfg->maxRows) return -1;
   if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
   if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
+  if (pCfg->ttl < TSDB_MIN_DB_TTL_OPTION && pCfg->ttl != TSDB_DEFAULT_DB_TTL_OPTION) return -1;
   if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
   if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
   if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
@@ -278,6 +281,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
   if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
   if (pCfg->streamMode < TSDB_MIN_DB_STREAM_MODE || pCfg->streamMode > TSDB_MAX_DB_STREAM_MODE) return -1;
+  if (pCfg->singleSTable < TSDB_MIN_DB_SINGLE_STABLE_OPTION || pCfg->streamMode > TSDB_MAX_DB_SINGLE_STABLE_OPTION) return -1;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -293,6 +297,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK;
   if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
   if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
+  if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
   if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
   if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
   if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
@@ -301,6 +306,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION;
   if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
   if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE;
+  if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION;
   if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
 }
 
@@ -437,6 +443,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
       .maxRows = pCreate->maxRows,
       .commitTime = pCreate->commitTime,
       .fsyncPeriod = pCreate->fsyncPeriod,
+      .ttl = pCreate->ttl,
       .walLevel = pCreate->walLevel,
       .precision = pCreate->precision,
       .compression = pCreate->compression,
@@ -445,6 +452,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
       .update = pCreate->update,
       .cacheLastRow = pCreate->cacheLastRow,
       .streamMode = pCreate->streamMode,
+      .singleSTable = pCreate->singleSTable,
   };
 
   dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
@@ -730,6 +738,71 @@ ALTER_DB_OVER:
   return code;
 }
 
+static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
+  SMnode     *pMnode = pReq->pNode;
+  int32_t     code = -1;
+  SDbObj     *pDb = NULL;
+  SDbCfgReq   cfgReq = {0};
+  SDbCfgRsp   cfgRsp = {0};
+
+  if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto GET_DB_CFG_OVER;
+  }
+
+  pDb = mndAcquireDb(pMnode, cfgReq.db);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_EXIST;
+    goto GET_DB_CFG_OVER;
+  }
+
+  cfgRsp.numOfVgroups   = pDb->cfg.numOfVgroups;
+  cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize;
+  cfgRsp.totalBlocks    = pDb->cfg.totalBlocks;
+  cfgRsp.daysPerFile    = pDb->cfg.daysPerFile;
+  cfgRsp.daysToKeep0    = pDb->cfg.daysToKeep0;
+  cfgRsp.daysToKeep1    = pDb->cfg.daysToKeep1;
+  cfgRsp.daysToKeep2    = pDb->cfg.daysToKeep2;
+  cfgRsp.minRows        = pDb->cfg.minRows;
+  cfgRsp.maxRows        = pDb->cfg.maxRows;
+  cfgRsp.commitTime     = pDb->cfg.commitTime;
+  cfgRsp.fsyncPeriod    = pDb->cfg.fsyncPeriod;
+  cfgRsp.ttl            = pDb->cfg.ttl;
+  cfgRsp.walLevel       = pDb->cfg.walLevel;
+  cfgRsp.precision      = pDb->cfg.precision;
+  cfgRsp.compression    = pDb->cfg.compression;
+  cfgRsp.replications   = pDb->cfg.replications;
+  cfgRsp.quorum         = pDb->cfg.quorum;
+  cfgRsp.update         = pDb->cfg.update;
+  cfgRsp.cacheLastRow   = pDb->cfg.cacheLastRow;
+  cfgRsp.streamMode     = pDb->cfg.streamMode;
+  cfgRsp.singleSTable   = pDb->cfg.singleSTable;
+
+  int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp);
+  void   *pRsp = rpcMallocCont(contLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = -1;
+    goto GET_DB_CFG_OVER;
+  }
+
+  tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp);
+
+  pReq->pRsp = pRsp;
+  pReq->rspLen = contLen;
+
+GET_DB_CFG_OVER:
+
+  if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr());
+  }
+
+  mndReleaseDb(pMnode, pDb);
+
+  return code;
+}
+
+
 static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
   if (pRedoRaw == NULL) return -1;
@@ -1509,6 +1582,23 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
   cols++;
 
+  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
+  *(int32_t *)pWrite = pDb->cfg.ttl;
+  cols++;
+
+  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
+  *(int8_t *)pWrite = pDb->cfg.singleSTable;
+  cols++;
+
+  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
+  *(int8_t *)pWrite = pDb->cfg.streamMode;
+  cols++;
+
+  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
+  char *status = "ready";
+  STR_WITH_SIZE_TO_VARSTR(pWrite, status, strlen(status));
+  cols++;
+
   //  pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
   //  *(int8_t *)pWrite = pDb->cfg.update;
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
 
 bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
   int64_t interval = TABS(pDnode->lastAccessTime - curMs);
-  if (interval > 3500 * tsStatusInterval) {
+  if (interval > 30000 * tsStatusInterval) {
     if (pDnode->rebootTime > 0) {
       pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
     }
@@ -630,6 +630,7 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) {
 
 static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
   mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle);
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -63,7 +63,11 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     //                                                 {.name = "update",           .bytes = 1,   .type =
     //                                                 TSDB_DATA_TYPE_TINYINT},  // disable update
 };

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -60,7 +60,7 @@ static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp);
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
+static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName);
 static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName);
 
 int32_t mndInitSubscribe(SMnode *pMnode) {
@@ -102,12 +102,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
   return pSub;
 }
 
-static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) {
+static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, const char* topicName) {
   SMqMVRebReq req = {
       .vgId = pConsumerEp->vgId,
       .oldConsumerId = pConsumerEp->oldConsumerId,
       .newConsumerId = pConsumerEp->consumerId,
   };
+  req.topic = strdup(topicName);
 
   int32_t tlen = tEncodeSMqMVRebReq(NULL, &req);
   void   *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
@@ -122,6 +123,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
 
   void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
   tEncodeSMqMVRebReq(&abuf, &req);
+  taosMemoryFree(req.topic);
 
   *pBuf = buf;
   *pLen = tlen;
@@ -129,12 +131,12 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
   return 0;
 }
 
-static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) {
+static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName) {
   ASSERT(pConsumerEp->oldConsumerId != -1);
 
   void   *buf;
   int32_t tlen;
-  if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp) < 0) {
+  if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) {
     return -1;
   }
 
@@ -502,10 +504,10 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
             pConsumerEp->epoch = 0;
             taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
 
+            char topic[TSDB_TOPIC_FNAME_LEN];
+            char cgroup[TSDB_CGROUP_LEN];
+            mndSplitSubscribeKey(pSub->key, topic, cgroup);
             if (pConsumerEp->oldConsumerId == -1) {
-              char topic[TSDB_TOPIC_FNAME_LEN];
-              char cgroup[TSDB_CGROUP_LEN];
-              mndSplitSubscribeKey(pSub->key, topic, cgroup);
               SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
 
               mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId,
@@ -517,7 +519,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
               mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "",
                     pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
 
-              mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
+              mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic);
             }
           }
         }
@@ -849,7 +851,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
                 pConsumerEp->consumerId);
           mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
         } else {
-          mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
+          mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName);
         }
         // to trigger rebalance at once, do not set status active
         /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/

source/dnode/mnode/impl/src/mnode.c

@@ -502,7 +502,11 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
 
     SMonVgroupDesc desc = {0};
     desc.vgroup_id = pVgroup->vgId;
-    strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name));
+
+    SName name = {0};
+    tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+    tNameGetDbName(&name, desc.database_name);
+
     desc.tables_num = pVgroup->numOfTables;
     pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
     tstrncpy(desc.status, "unsynced", sizeof(desc.status));

source/dnode/vnode/CMakeLists.txt

@@ -1,54 +1,76 @@
-set(META_DB_IMPL_LIST "BDB" "TDB")
+# vnode
-set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation")
+add_library(vnode STATIC "")
-set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
+target_sources(
+    vnode
+    PRIVATE
+    # vnode
+    "src/vnd/vnodeArenaMAImpl.c"
+    "src/vnd/vnodeBufferPool.c"
+    "src/vnd/vnodeCfg.c"
+    "src/vnd/vnodeCommit.c"
+    "src/vnd/vnodeInt.c"
+    "src/vnd/vnodeMain.c"
+    "src/vnd/vnodeMgr.c"
+    "src/vnd/vnodeQuery.c"
+    "src/vnd/vnodeStateMgr.c"
+    "src/vnd/vnodeWrite.c"
 
-if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
+    # meta
-    message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
+    # "src/meta/metaBDBImpl.c"
-else()
+    "src/meta/metaCache.c"
-    message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
+    "src/meta/metaCfg.c"
-endif()
+    "src/meta/metaIdx.c"
+    "src/meta/metaMain.c"
+    "src/meta/metaQuery.c"
+    "src/meta/metaTable.c"
+    "src/meta/metaTbCfg.c"
+    "src/meta/metaTbTag.c"
+    "src/meta/metaTbUid.c"
+    "src/meta/metaTDBImpl.c"
 
-aux_source_directory(src/meta META_SRC)
+    # tsdb
-if(${META_DB_IMPL} STREQUAL "BDB")
+    # "src/tsdb/tsdbBDBImpl.c"
-    list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c")
+    "src/tsdb/tsdbCommit.c"
-elseif(${META_DB_IMPL} STREQUAL "TDB")
+    "src/tsdb/tsdbCompact.c"
-    list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c")
+    "src/tsdb/tsdbFile.c"
-endif()
+    "src/tsdb/tsdbFS.c"
+    "src/tsdb/tsdbMain.c"
+    "src/tsdb/tsdbMemTable.c"
+    "src/tsdb/tsdbOptions.c"
+    "src/tsdb/tsdbRead.c"
+    "src/tsdb/tsdbReadImpl.c"
+    "src/tsdb/tsdbScan.c"
+    # "src/tsdb/tsdbSma.c"
+    "src/tsdb/tsdbWrite.c"
 
-aux_source_directory(src/tq TQ_SRC)
+    # tq
-aux_source_directory(src/tsdb TSDB_SRC)
+    "src/tq/tq.c"
-aux_source_directory(src/vnd VND_SRC)
+    "src/tq/tqCommit.c"
-
+    "src/tq/tqMetaStore.c"
-list(APPEND
+    "src/tq/tqOffset.c"
-    VNODE_SRC
+    "src/tq/tqPush.c"
-    ${META_SRC}
+    "src/tq/tqRead.c"
-    ${TQ_SRC}
-    ${TSDB_SRC}
-    ${VND_SRC}
 )
-
-add_library(vnode STATIC ${VNODE_SRC})
 target_include_directories(
     vnode
-    PUBLIC inc
+    PUBLIC "inc"
-    PRIVATE src/inc
+    PRIVATE "src/inc"
 )
 target_link_libraries(
     vnode
     PUBLIC os
     PUBLIC util
     PUBLIC common
-    PUBLIC transport
     PUBLIC tfs
     PUBLIC wal
-    PUBLIC scheduler
-    PUBLIC executor
-    PUBLIC stream
     PUBLIC qworker
     PUBLIC sync
-    # TODO: get rid of BDB
+    PUBLIC executor
-    PUBLIC bdb
+    PUBLIC scheduler
     PUBLIC tdb
+    #PUBLIC bdb
+    PUBLIC transport
+    PUBLIC stream
 )
 
 if(${BUILD_TEST})

source/dnode/vnode/inc/meta.h

@@ -1,86 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_H_
-#define _TD_META_H_
-
-#include "tmallocator.h"
-#include "tmsg.h"
-#include "trow.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define META_SUPER_TABLE  TD_SUPER_TABLE
-#define META_CHILD_TABLE  TD_CHILD_TABLE
-#define META_NORMAL_TABLE TD_NORMAL_TABLE
-
-// Types exported
-typedef struct SMeta SMeta;
-
-typedef struct SMetaCfg {
-  /// LRU cache size
-  uint64_t lruSize;
-} SMetaCfg;
-
-typedef struct SMTbCursor  SMTbCursor;
-typedef struct SMCtbCursor SMCtbCursor;
-typedef struct SMSmaCursor SMSmaCursor;
-
-typedef SVCreateTbReq   STbCfg;
-typedef SVCreateTSmaReq SSmaCfg;
-
-// SMeta operations
-SMeta * metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
-void    metaClose(SMeta *pMeta);
-void    metaRemove(const char *path);
-int     metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
-int     metaDropTable(SMeta *pMeta, tb_uid_t uid);
-int     metaCommit(SMeta *pMeta);
-int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
-int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid);
-
-// For Query
-STbCfg *        metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
-STbCfg *        metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
-SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
-STSchema *      metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
-STSma *         metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid);
-STSmaWrapper *  metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid);
-SArray *        metaGetSmaTbUids(SMeta *pMeta, bool isDup);
-int             metaGetTbNum(SMeta *pMeta);
-
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
-void        metaCloseTbCursor(SMTbCursor *pTbCur);
-char *      metaTbCursorNext(SMTbCursor *pTbCur);
-
-SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
-void         metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
-tb_uid_t     metaCtbCursorNext(SMCtbCursor *pCtbCur);
-
-SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
-void         metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
-const char * metaSmaCursorNext(SMSmaCursor *pSmaCur);
-
-// Options
-void metaOptionsInit(SMetaCfg *pMetaCfg);
-void metaOptionsClear(SMetaCfg *pMetaCfg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_H_*/

source/dnode/vnode/inc/tsdb.h

@@ -1,288 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_H_
-#define _TD_TSDB_H_
-
-#include "tmallocator.h"
-#include "meta.h"
-#include "tcommon.h"
-#include "tfs.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SDataStatis {
-  int16_t colId;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-} SDataStatis;
-
-typedef struct STable {
-  uint64_t  tid;
-  uint64_t  uid;
-  STSchema *pSchema;
-} STable;
-
-#define BLOCK_LOAD_OFFSET_SEQ_ORDER   1
-#define BLOCK_LOAD_TABLE_SEQ_ORDER    2
-#define BLOCK_LOAD_TABLE_RR_ORDER     3
-
-#define TABLE_TID(t) (t)->tid
-#define TABLE_UID(t) (t)->uid
-
-// TYPES EXPOSED
-typedef struct STsdb STsdb;
-
-typedef struct STsdbCfg {
-  int8_t   precision;
-  int8_t   update;
-  int8_t   compression;
-  int32_t  daysPerFile;
-  int32_t  minRowsPerFileBlock;
-  int32_t  maxRowsPerFileBlock;
-  int32_t  keep;
-  int32_t  keep1;
-  int32_t  keep2;
-  uint64_t lruCacheSize;
-  SArray  *retentions;
-} STsdbCfg;
-
-// query condition to build multi-table data block iterator
-typedef struct STsdbQueryCond {
-  STimeWindow  twindow;
-  int32_t      order;             // desc|asc order to iterate the data block
-  int32_t      numOfCols;
-  SColumnInfo *colList;
-  bool         loadExternalRows;  // load external rows or not
-  int32_t      type;              // data block load type:
-} STsdbQueryCond;
-
-typedef struct {
-  TSKEY    lastKey;
-  uint64_t uid;
-} STableKeyInfo;
-
-// STsdb
-STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
-void   tsdbClose(STsdb *);
-void   tsdbRemove(const char *path);
-int    tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
-int    tsdbPrepareCommit(STsdb *pTsdb);
-int    tsdbCommit(STsdb *pTsdb);
-
-
-int32_t tsdbInitSma(STsdb *pTsdb);
-int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
-int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
-/**
- * @brief When submit msg received, update the relative expired window synchronously.
- *
- * @param pTsdb
- * @param msg
- * @return int32_t
- */
-int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg);
-
-/**
- * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
- *
- * @param pTsdb
- * @param indexUid
- * @param msg
- * @return int32_t
- */
-int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
-
-/**
- * @brief Drop tSma data and local cache.
- * 
- * @param pTsdb 
- * @param indexUid 
- * @return int32_t 
- */
-int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
-
-/**
- * @brief Insert RSma(Rollup SMA) data.
- *
- * @param pTsdb
- * @param msg
- * @return int32_t
- */
-int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
-
-// TODO: This is the basic params, and should wrap the params to a queryHandle.
-/**
- * @brief Get tSma(Time-range-wise SMA) data.
- * 
- * @param pTsdb 
- * @param pData 
- * @param indexUid 
- * @param querySKey 
- * @param nMaxResult 
- * @return int32_t 
- */
-int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
-
-// STsdbCfg
-int  tsdbOptionsInit(STsdbCfg *);
-void tsdbOptionsClear(STsdbCfg *);
-
-typedef void* tsdbReaderT;
-
-/**
- * Get the data block iterator, starting from position according to the query condition
- *
- * @param tsdb       tsdb handle
- * @param pCond      query condition, including time window, result set order, and basic required columns for each block
- * @param tableInfoGroup  table object list in the form of set, grouped into different sets according to the
- *                        group by condition
- * @param qinfo      query info handle from query processor
- * @return
- */
-tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId);
-
-/**
- * Get the last row of the given query time window for all the tables in STableGroupInfo object.
- * Note that only one data block with only row will be returned while invoking retrieve data block function for
- * all tables in this group.
- *
- * @param tsdb   tsdb handle
- * @param pCond  query condition, including time window, result set order, and basic required columns for each block
- * @param tableInfo  table list.
- * @return
- */
-//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
-//                                  SMemRef *pRef);
-
-
-tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
-
-int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo);
-
-bool isTsdbCacheLastRow(tsdbReaderT* pReader);
-
-/**
- *
- * @param tsdb
- * @param uid
- * @param skey
- * @param pTagCond
- * @param len
- * @param tagNameRelType
- * @param tbnameCond
- * @param pGroupInfo
- * @param pColIndex
- * @param numOfCols
- * @param reqId
- * @return
- */
-int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
-                                 int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
-                                 SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId);
-/**
- * get num of rows in mem table
- *
- * @param pHandle
- * @return row size
- */
-
-int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle);
-
-/**
- * move to next block if exists
- *
- * @param pTsdbReadHandle
- * @return
- */
-bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
-
-/**
- * Get current data block information
- *
- * @param pTsdbReadHandle
- * @param pBlockInfo
- * @return
- */
-void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
-
-/**
- *
- * Get the pre-calculated information w.r.t. current data block.
- *
- * In case of data block in cache, the pBlockStatis will always be NULL.
- * If a block is not completed loaded from disk, the pBlockStatis will be NULL.
-
- * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
- * @return
- */
-int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis);
-
-/**
- *
- * The query condition with primary timestamp is passed to iterator during its constructor function,
- * the returned data block must be satisfied with the time window condition in any cases,
- * which means the SData data block is not actually the completed disk data blocks.
- *
- * @param pTsdbReadHandle      query handle
- * @param pColumnIdList     required data columns id list
- * @return
- */
-SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
-
-/**
- * destroy the created table group list, which is generated by tag query
- * @param pGroupList
- */
-void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
-
-/**
- * create the table group result including only one table, used to handle the normal table query
- *
- * @param tsdb        tsdbHandle
- * @param uid         table uid
- * @param pGroupInfo  the generated result
- * @return
- */
-int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
-
-/**
- *
- * @param tsdb
- * @param pTableIdList
- * @param pGroupInfo
- * @return
- */
-int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
-
-/**
- * clean up the query handle
- * @param queryHandle
- */
-void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
-
-int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_H_*/

source/dnode/vnode/inc/vnode.h

@@ -21,12 +21,17 @@
 #include "tqueue.h"
 #include "trpc.h"
 
-#include "meta.h"
 #include "tarray.h"
 #include "tfs.h"
-#include "tsdb.h"
 #include "wal.h"
 
+#include "tmallocator.h"
+#include "tmsg.h"
+#include "trow.h"
+#include "tmallocator.h"
+#include "tcommon.h"
+#include "tfs.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -34,6 +39,82 @@ extern "C" {
 /* ------------------------ TYPES EXPOSED ------------------------ */
 typedef struct SMgmtWrapper SMgmtWrapper;
 typedef struct SVnode       SVnode;
+
+#define META_SUPER_TABLE  TD_SUPER_TABLE
+#define META_CHILD_TABLE  TD_CHILD_TABLE
+#define META_NORMAL_TABLE TD_NORMAL_TABLE
+
+// Types exported
+typedef struct SMeta SMeta;
+
+typedef struct SMetaCfg {
+  /// LRU cache size
+  uint64_t lruSize;
+} SMetaCfg;
+
+typedef struct SMTbCursor  SMTbCursor;
+typedef struct SMCtbCursor SMCtbCursor;
+typedef struct SMSmaCursor SMSmaCursor;
+
+typedef SVCreateTbReq   STbCfg;
+typedef SVCreateTSmaReq SSmaCfg;
+
+typedef struct SDataStatis {
+  int16_t colId;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+} SDataStatis;
+
+typedef struct STsdbQueryCond {
+  STimeWindow  twindow;
+  int32_t      order;             // desc|asc order to iterate the data block
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} STsdbQueryCond;
+
+typedef struct {
+  TSKEY    lastKey;
+  uint64_t uid;
+} STableKeyInfo;
+
+
+typedef struct STable {
+  uint64_t  tid;
+  uint64_t  uid;
+  STSchema *pSchema;
+} STable;
+
+#define BLOCK_LOAD_OFFSET_SEQ_ORDER   1
+#define BLOCK_LOAD_TABLE_SEQ_ORDER    2
+#define BLOCK_LOAD_TABLE_RR_ORDER     3
+
+#define TABLE_TID(t) (t)->tid
+#define TABLE_UID(t) (t)->uid
+
+// TYPES EXPOSED
+typedef struct STsdb STsdb;
+
+typedef struct STsdbCfg {
+  int8_t   precision;
+  int8_t   update;
+  int8_t   compression;
+  int32_t  daysPerFile;
+  int32_t  minRowsPerFileBlock;
+  int32_t  maxRowsPerFileBlock;
+  int32_t  keep;
+  int32_t  keep1;
+  int32_t  keep2;
+  uint64_t lruCacheSize;
+  SArray  *retentions;
+} STsdbCfg;
+
+
 typedef struct {
   // TODO
   int32_t reserved;
@@ -263,6 +344,241 @@ int     tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockIn
 // return SArray<SColumnInfoData>
 SArray *tqRetrieveDataBlock(STqReadHandle *pHandle);
 
+// meta.h
+SMeta          *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
+void            metaClose(SMeta *pMeta);
+void            metaRemove(const char *path);
+int             metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
+int             metaDropTable(SMeta *pMeta, tb_uid_t uid);
+int             metaCommit(SMeta *pMeta);
+int32_t         metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
+int32_t         metaDropTSma(SMeta *pMeta, int64_t indexUid);
+STbCfg         *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
+STbCfg         *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
+STSchema       *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
+STSma          *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid);
+STSmaWrapper   *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid);
+SArray         *metaGetSmaTbUids(SMeta *pMeta, bool isDup);
+int             metaGetTbNum(SMeta *pMeta);
+SMTbCursor     *metaOpenTbCursor(SMeta *pMeta);
+void            metaCloseTbCursor(SMTbCursor *pTbCur);
+char           *metaTbCursorNext(SMTbCursor *pTbCur);
+SMCtbCursor    *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
+void            metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
+tb_uid_t        metaCtbCursorNext(SMCtbCursor *pCtbCur);
+
+SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
+void         metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
+const char  *metaSmaCursorNext(SMSmaCursor *pSmaCur);
+
+// Options
+void metaOptionsInit(SMetaCfg *pMetaCfg);
+void metaOptionsClear(SMetaCfg *pMetaCfg);
+
+// query condition to build multi-table data block iterator
+// STsdb
+STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
+void   tsdbClose(STsdb *);
+void   tsdbRemove(const char *path);
+int    tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
+int    tsdbPrepareCommit(STsdb *pTsdb);
+int    tsdbCommit(STsdb *pTsdb);
+
+
+int32_t tsdbInitSma(STsdb *pTsdb);
+int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
+int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
+/**
+ * @brief When submit msg received, update the relative expired window synchronously.
+ *
+ * @param pTsdb
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg);
+
+/**
+ * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
+ *
+ * @param pTsdb
+ * @param indexUid
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
+
+/**
+ * @brief Drop tSma data and local cache.
+ * 
+ * @param pTsdb 
+ * @param indexUid 
+ * @return int32_t 
+ */
+int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
+
+/**
+ * @brief Insert RSma(Rollup SMA) data.
+ *
+ * @param pTsdb
+ * @param msg
+ * @return int32_t
+ */
+int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
+
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
+/**
+ * @brief Get tSma(Time-range-wise SMA) data.
+ * 
+ * @param pTsdb 
+ * @param pData 
+ * @param indexUid 
+ * @param querySKey 
+ * @param nMaxResult 
+ * @return int32_t 
+ */
+int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
+
+// STsdbCfg
+int  tsdbOptionsInit(STsdbCfg *);
+void tsdbOptionsClear(STsdbCfg *);
+
+typedef void* tsdbReaderT;
+
+/**
+ * Get the data block iterator, starting from position according to the query condition
+ *
+ * @param tsdb       tsdb handle
+ * @param pCond      query condition, including time window, result set order, and basic required columns for each block
+ * @param tableInfoGroup  table object list in the form of set, grouped into different sets according to the
+ *                        group by condition
+ * @param qinfo      query info handle from query processor
+ * @return
+ */
+tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId);
+
+/**
+ * Get the last row of the given query time window for all the tables in STableGroupInfo object.
+ * Note that only one data block with only row will be returned while invoking retrieve data block function for
+ * all tables in this group.
+ *
+ * @param tsdb   tsdb handle
+ * @param pCond  query condition, including time window, result set order, and basic required columns for each block
+ * @param tableInfo  table list.
+ * @return
+ */
+//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
+//                                  SMemRef *pRef);
+
+
+tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
+
+int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo);
+
+bool isTsdbCacheLastRow(tsdbReaderT* pReader);
+
+/**
+ *
+ * @param tsdb
+ * @param uid
+ * @param skey
+ * @param pTagCond
+ * @param len
+ * @param tagNameRelType
+ * @param tbnameCond
+ * @param pGroupInfo
+ * @param pColIndex
+ * @param numOfCols
+ * @param reqId
+ * @return
+ */
+int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
+                                 int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
+                                 SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId);
+/**
+ * get num of rows in mem table
+ *
+ * @param pHandle
+ * @return row size
+ */
+
+int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle);
+
+/**
+ * move to next block if exists
+ *
+ * @param pTsdbReadHandle
+ * @return
+ */
+bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
+
+/**
+ * Get current data block information
+ *
+ * @param pTsdbReadHandle
+ * @param pBlockInfo
+ * @return
+ */
+void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
+
+/**
+ *
+ * Get the pre-calculated information w.r.t. current data block.
+ *
+ * In case of data block in cache, the pBlockStatis will always be NULL.
+ * If a block is not completed loaded from disk, the pBlockStatis will be NULL.
+
+ * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
+ * @return
+ */
+int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis);
+
+/**
+ *
+ * The query condition with primary timestamp is passed to iterator during its constructor function,
+ * the returned data block must be satisfied with the time window condition in any cases,
+ * which means the SData data block is not actually the completed disk data blocks.
+ *
+ * @param pTsdbReadHandle      query handle
+ * @param pColumnIdList     required data columns id list
+ * @return
+ */
+SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
+
+/**
+ * destroy the created table group list, which is generated by tag query
+ * @param pGroupList
+ */
+void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
+
+/**
+ * create the table group result including only one table, used to handle the normal table query
+ *
+ * @param tsdb        tsdbHandle
+ * @param uid         table uid
+ * @param pGroupInfo  the generated result
+ * @return
+ */
+int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+
+/**
+ *
+ * @param tsdb
+ * @param pTableIdList
+ * @param pGroupInfo
+ * @return
+ */
+int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
+
+/**
+ * clean up the query handle
+ * @param queryHandle
+ */
+void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
+
+int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
+
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/meta.h

@@ -13,12 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_META_DEF_H_
+#ifndef _TD_VNODE_META_H_
-#define _TD_META_DEF_H_
+#define _TD_VNODE_META_H_
-
-#include "tmallocator.h"
-
-#include "meta.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -78,4 +74,4 @@ struct SMeta {
 }
 #endif
 
-#endif /*_TD_META_DEF_H_*/
+#endif /*_TD_VNODE_META_H_*/

source/dnode/vnode/src/inc/tq.h

@@ -13,18 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TQ_INT_H_
+#ifndef _TD_VNODE_TQ_H_
-#define _TD_TQ_INT_H_
+#define _TD_VNODE_TQ_H_
-
-#include "meta.h"
-#include "tlog.h"
-#include "tqPush.h"
-#include "vnd.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// tqInt.h
 #define tqFatal(...)                                             \
   {                                                              \
     if (tqDebugFlag & DEBUG_FATAL) {                             \
@@ -106,6 +102,9 @@ static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
 
 typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
 
+typedef struct STqOffsetCfg   STqOffsetCfg;
+typedef struct STqOffsetStore STqOffsetStore;
+
 typedef struct {
   int16_t ver;
   int16_t action;
@@ -211,13 +210,93 @@ typedef struct {
   SArray* topics;  // SArray<STqTopic>
 } STqConsumer;
 
+enum {
+  TQ_PUSHER_TYPE__CLIENT = 1,
+  TQ_PUSHER_TYPE__STREAM,
+};
+
+typedef struct {
+  int8_t   type;
+  int8_t   reserved[3];
+  int32_t  ttl;
+  int64_t  consumerId;
+  SRpcMsg* pMsg;
+  // SMqPollRsp* rsp;
+} STqClientPusher;
+
+typedef struct {
+  int8_t      type;
+  int8_t      nodeType;
+  int8_t      reserved[6];
+  int64_t     streamId;
+  qTaskInfo_t task;
+  // TODO sync function
+} STqStreamPusher;
+
+typedef struct {
+  int8_t type;  // mq or stream
+} STqPusher;
+
+typedef struct {
+  SHashObj* pHash;  // <id, STqPush*>
+} STqPushMgr;
+
+typedef struct {
+  int8_t inited;
+  tmr_h  timer;
+} STqPushMgmt;
+
+static STqPushMgmt tqPushMgmt;
+
+
 int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
 int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);
 
 static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }
 
+// tqMetaStore.h
+STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
+                          FTqDelete pDeleter, int32_t tqConfigFlag);
+int32_t       tqStoreClose(STqMetaStore*);
+// int32_t       tqStoreDelete(TqMetaStore*);
+// int32_t       tqStoreCommitAll(TqMetaStore*);
+int32_t tqStorePersist(STqMetaStore*);
+// clean deleted idx and data from persistent file
+int32_t tqStoreCompact(STqMetaStore*);
+
+void* tqHandleGet(STqMetaStore*, int64_t key);
+// make it unpersist
+void*   tqHandleTouchGet(STqMetaStore*, int64_t key);
+int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
+int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
+// delete committed kv pair
+// notice that a delete action still needs to be committed
+int32_t tqHandleDel(STqMetaStore*, int64_t key);
+int32_t tqHandlePurge(STqMetaStore*, int64_t key);
+int32_t tqHandleCommit(STqMetaStore*, int64_t key);
+int32_t tqHandleAbort(STqMetaStore*, int64_t key);
+
+// tqOffset
+STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
+void            STqOffsetClose(STqOffsetStore*);
+
+int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey);
+int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset);
+int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
+int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
+
+// tqPush
+int32_t tqPushMgrInit();
+void    tqPushMgrCleanUp();
+
+STqPushMgr* tqPushMgrOpen();
+void        tqPushMgrClose(STqPushMgr* pushMgr);
+
+STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
+STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TQ_INT_H_*/
+#endif /*_TD_VNODE_TQ_H_*/

source/dnode/vnode/src/inc/tqCommit.h

@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */

source/dnode/vnode/src/inc/tqMetaStore.h

@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TQ_META_STORE_H_
-#define _TQ_META_STORE_H_
-
-#include "os.h"
-#include "tqInt.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
-                          FTqDelete pDeleter, int32_t tqConfigFlag);
-int32_t       tqStoreClose(STqMetaStore*);
-// int32_t       tqStoreDelete(TqMetaStore*);
-// int32_t       tqStoreCommitAll(TqMetaStore*);
-int32_t tqStorePersist(STqMetaStore*);
-// clean deleted idx and data from persistent file
-int32_t tqStoreCompact(STqMetaStore*);
-
-void* tqHandleGet(STqMetaStore*, int64_t key);
-// make it unpersist
-void*   tqHandleTouchGet(STqMetaStore*, int64_t key);
-int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
-int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
-// delete committed kv pair
-// notice that a delete action still needs to be committed
-int32_t tqHandleDel(STqMetaStore*, int64_t key);
-int32_t tqHandlePurge(STqMetaStore*, int64_t key);
-int32_t tqHandleCommit(STqMetaStore*, int64_t key);
-int32_t tqHandleAbort(STqMetaStore*, int64_t key);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* ifndef _TQ_META_STORE_H_ */

source/dnode/vnode/src/inc/tqOffset.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TQ_OFFSET_H_
-#define _TD_TQ_OFFSET_H_
-
-#include "tqInt.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct STqOffsetCfg   STqOffsetCfg;
-typedef struct STqOffsetStore STqOffsetStore;
-
-STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
-void            STqOffsetClose(STqOffsetStore*);
-
-int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey);
-int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset);
-int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
-int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TQ_OFFSET_H_*/

source/dnode/vnode/src/inc/tqPush.h

@@ -1,80 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TQ_PUSH_H_
-#define _TQ_PUSH_H_
-
-#include "executor.h"
-#include "thash.h"
-#include "trpc.h"
-#include "ttimer.h"
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum {
-  TQ_PUSHER_TYPE__CLIENT = 1,
-  TQ_PUSHER_TYPE__STREAM,
-};
-
-typedef struct {
-  int8_t   type;
-  int8_t   reserved[3];
-  int32_t  ttl;
-  int64_t  consumerId;
-  SRpcMsg* pMsg;
-  // SMqPollRsp* rsp;
-} STqClientPusher;
-
-typedef struct {
-  int8_t      type;
-  int8_t      nodeType;
-  int8_t      reserved[6];
-  int64_t     streamId;
-  qTaskInfo_t task;
-  // TODO sync function
-} STqStreamPusher;
-
-typedef struct {
-  int8_t type;  // mq or stream
-} STqPusher;
-
-typedef struct {
-  SHashObj* pHash;  // <id, STqPush*>
-} STqPushMgr;
-
-typedef struct {
-  int8_t inited;
-  tmr_h  timer;
-} STqPushMgmt;
-
-static STqPushMgmt tqPushMgmt;
-
-int32_t tqPushMgrInit();
-void    tqPushMgrCleanUp();
-
-STqPushMgr* tqPushMgrOpen();
-void        tqPushMgrClose(STqPushMgr* pushMgr);
-
-STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
-STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TQ_PUSH_H_*/

source/dnode/vnode/src/inc/tsdb.h

@@ -0,0 +1,936 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_TSDB_H_
+#define _TD_VNODE_TSDB_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SSmaStat SSmaStat;
+typedef struct SSmaEnv  SSmaEnv;
+typedef struct SSmaEnvs SSmaEnvs;
+
+typedef enum {
+  TSDB_FILE_HEAD = 0,  // .head
+  TSDB_FILE_DATA,      // .data
+  TSDB_FILE_LAST,      // .last
+  TSDB_FILE_SMAD,      // .smad(Block-wise SMA)
+  TSDB_FILE_SMAL,      // .smal(Block-wise SMA)
+  TSDB_FILE_MAX,       //
+  TSDB_FILE_META,      // meta
+  TSDB_FILE_TSMA,      // v2t100.${sma_index_name}, Time-range-wise SMA
+  TSDB_FILE_RSMA,      // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
+} E_TSDB_FILE_T;
+
+typedef struct {
+  uint32_t magic;
+  uint32_t fver;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo   info;
+  STfsFile  f;
+  TdFilePtr pFile;
+  uint8_t   state;
+} SDFile;
+
+struct SSmaEnvs {
+  int16_t  nTSma;
+  int16_t  nRSma;
+  SSmaEnv *pTSmaEnv;
+  SSmaEnv *pRSmaEnv;
+};
+
+typedef struct {
+  int      fid;
+  int8_t   state;  // -128~127
+  uint8_t  ver;    // 0~255, DFileSet version
+  uint16_t reserve;
+  SDFile   files[TSDB_FILE_MAX];
+} SDFileSet;
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct STbData {
+  tb_uid_t   uid;
+  TSKEY      keyMin;
+  TSKEY      keyMax;
+  int64_t    nrows;
+  SSkipList *pData;
+} STbData;
+
+typedef struct STsdbMemTable {
+  T_REF_DECLARE()
+  SRWLatch       latch;
+  TSKEY          keyMin;
+  TSKEY          keyMax;
+  uint64_t       nRow;
+  SMemAllocator *pMA;
+  // Container
+  SSkipList *pSlIdx;  // SSkiplist<STbData>
+  SHashObj * pHashIdx;
+} STsdbMemTable;
+
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SArray *    df;    // data file array
+  SArray *    sf;    // sma data file array    v2f1900.index_name_1
+} SFSStatus;
+
+typedef struct {
+  TdThreadRwlock lock;
+
+  SFSStatus *cstatus;        // current status
+  SHashObj * metaCache;      // meta cache
+  SHashObj * metaCacheComp;  // meta cache for compact
+  bool       intxn;
+  SFSStatus *nstatus;  // new status
+} STsdbFS;
+
+struct STsdb {
+  int32_t               vgId;
+  bool                  repoLocked;
+  TdThreadMutex         mutex;
+  char *                path;
+  STsdbCfg              config;
+  STsdbMemTable *       mem;
+  STsdbMemTable *       imem;
+  SRtn                  rtn;
+  SMemAllocatorFactory *pmaf;
+  STsdbFS *             fs;
+  SMeta *               pMeta;
+  STfs *                pTfs;
+  SSmaEnvs              smaEnvs;
+};
+
+#define REPO_ID(r)        ((r)->vgId)
+#define REPO_CFG(r)       (&(r)->config)
+#define REPO_FS(r)        ((r)->fs)
+#define REPO_META(r)      ((r)->pMeta)
+#define REPO_TFS(r)       ((r)->pTfs)
+#define IS_REPO_LOCKED(r) ((r)->repoLocked)
+#define REPO_TSMA_NUM(r)  ((r)->smaEnvs.nTSma)
+#define REPO_RSMA_NUM(r)  ((r)->smaEnvs.nRSma)
+#define REPO_TSMA_ENV(r)  ((r)->smaEnvs.pTSmaEnv)
+#define REPO_RSMA_ENV(r)  ((r)->smaEnvs.pRSmaEnv)
+
+int tsdbLockRepo(STsdb *pTsdb);
+int tsdbUnlockRepo(STsdb *pTsdb);
+
+static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) {
+  return pTable->pSchema;
+}
+
+// tsdbLog
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+// tsdbMemTable.h
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
+
+static void * taosTMalloc(size_t size);
+static void * taosTCalloc(size_t nmemb, size_t size);
+static void * taosTRealloc(void *ptr, size_t size);
+static void * taosTZfree(void *ptr);
+static size_t taosTSizeof(void *ptr);
+static void   taosTMemset(void *ptr, int c);
+
+STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
+void           tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
+int            tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
+int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
+                          TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
+
+static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
+  if (pIter == NULL) return NULL;
+
+  SSkipListNode *node = tSkipListIterGet(pIter);
+  if (node == NULL) return NULL;
+
+  return (STSRow *)SL_GET_NODE_DATA(node);
+}
+
+static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
+  STSRow *row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
+
+  return TD_ROW_KEY(row);
+}
+
+// tsdbOptions
+extern const STsdbCfg defautlTsdbOptions;
+
+int  tsdbValidateOptions(const STsdbCfg *);
+void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
+
+// tsdbReadImpl
+typedef struct SReadH SReadH;
+
+typedef struct {
+  uint32_t len;
+  uint32_t offset;
+  uint32_t hasLast : 2;
+  uint32_t numOfBlocks : 30;
+  uint64_t uid;
+  TSKEY    maxKey;
+} SBlockIdx;
+
+#ifdef TD_REFACTOR_3
+typedef struct {
+  int64_t last : 1;
+  int64_t offset : 63;
+  int32_t algorithm : 8;
+  int32_t numOfRows : 24;
+  int32_t len;
+  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  int16_t numOfSubBlocks;
+  int16_t numOfCols;  // not including timestamp column
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+} SBlock;
+
+#else
+
+typedef enum {
+  TSDB_SBLK_VER_0 = 0,
+  TSDB_SBLK_VER_MAX,
+} ESBlockVer;
+
+#define SBlockVerLatest TSDB_SBLK_VER_0
+
+typedef struct {
+  uint8_t  last : 1;
+  uint8_t  blkVer : 7;
+  uint8_t  numOfSubBlocks;
+  col_id_t numOfCols;    // not including timestamp column
+  uint32_t len;          // data block length
+  uint32_t keyLen : 20;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  uint32_t algorithm : 4;
+  uint32_t reserve : 8;
+  col_id_t numOfBSma;
+  uint16_t numOfRows;
+  int64_t  offset;
+  uint64_t aggrStat : 1;
+  uint64_t aggrOffset : 63;
+  TSKEY    keyFirst;
+  TSKEY    keyLast;
+} SBlockV0;
+
+#define SBlock SBlockV0  // latest SBlock definition
+
+#endif
+
+typedef struct {
+  int32_t  delimiter;  // For recovery usage
+  int32_t  tid;
+  uint64_t uid;
+  SBlock   blocks[];
+} SBlockInfo;
+
+#ifdef TD_REFACTOR_3
+typedef struct {
+  int16_t  colId;
+  uint16_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint16_t reserve : 15;
+  int32_t  len;
+  uint32_t type : 8;
+  uint32_t offset : 24;
+  int64_t  sum;
+  int64_t  max;
+  int64_t  min;
+  int16_t  maxIndex;
+  int16_t  minIndex;
+  int16_t  numOfNull;
+  uint8_t  offsetH;
+  char     padding[1];
+} SBlockCol;
+#else
+typedef struct {
+  int16_t  colId;
+  uint16_t type : 6;
+  uint16_t blen : 10;   // bitmap length(TODO: full UT for the bitmap compress of various data input)
+  uint32_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint32_t len : 31;    // data length + bitmap length
+  uint32_t offset;
+} SBlockColV0;
+
+#define SBlockCol SBlockColV0  // latest SBlockCol definition
+
+typedef struct {
+  int16_t colId;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+} SAggrBlkColV0;
+
+#define SAggrBlkCol SAggrBlkColV0  // latest SAggrBlkCol definition
+
+#endif
+
+// Code here just for back-ward compatibility
+static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
+#ifdef TD_REFACTOR_3
+  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
+  pBlockCol->offsetH = (uint8_t)(offset >> 24);
+#else
+  pBlockCol->offset = offset;
+#endif
+}
+
+static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
+#ifdef TD_REFACTOR_3
+  uint32_t offset1 = pBlockCol->offset;
+  uint32_t offset2 = pBlockCol->offsetH;
+  return (offset1 | (offset2 << 24));
+#else
+  return pBlockCol->offset;
+#endif
+}
+
+typedef struct {
+  int32_t   delimiter;  // For recovery usage
+  int32_t   numOfCols;  // For recovery usage
+  uint64_t  uid;        // For recovery usage
+  SBlockCol cols[];
+} SBlockData;
+
+typedef void SAggrBlkData;  // SBlockCol cols[];
+
+struct SReadH {
+  STsdb        *pRepo;
+  SDFileSet     rSet;     // FSET to read
+  SArray       *aBlkIdx;  // SBlockIdx array
+  STable       *pTable;   // table to read
+  SBlockIdx    *pBlkIdx;  // current reading table SBlockIdx
+  int           cidx;
+  SBlockInfo   *pBlkInfo;
+  SBlockData   *pBlkData;      // Block info
+  SAggrBlkData *pAggrBlkData;  // Aggregate Block info
+  SDataCols    *pDCols[2];
+  void         *pBuf;    // buffer
+  void         *pCBuf;   // compression buffer
+  void         *pExBuf;  // extra buffer
+};
+
+#define TSDB_READ_REPO(rh)      ((rh)->pRepo)
+#define TSDB_READ_REPO_ID(rh)   REPO_ID(TSDB_READ_REPO(rh))
+#define TSDB_READ_FSET(rh)      (&((rh)->rSet))
+#define TSDB_READ_TABLE(rh)     ((rh)->pTable)
+#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
+#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
+#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
+#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
+#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
+#define TSDB_READ_BUF(rh)       ((rh)->pBuf)
+#define TSDB_READ_COMP_BUF(rh)  ((rh)->pCBuf)
+#define TSDB_READ_EXBUF(rh)     ((rh)->pExBuf)
+
+#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
+  (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+    default:
+      return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
+  }
+}
+
+#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
+
+static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
+  switch (blkVer) {
+    case TSDB_SBLK_VER_0:
+    default:
+      return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
+  }
+}
+
+int   tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
+void  tsdbDestroyReadH(SReadH *pReadh);
+int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
+void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
+int   tsdbLoadBlockIdx(SReadH *pReadh);
+int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
+int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
+                            int numOfColsIds);
+int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
+int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
+void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
+
+static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
+  void  *pBuf = *ppBuf;
+  size_t tsize = taosTSizeof(pBuf);
+
+  if (tsize < size) {
+    if (tsize == 0) tsize = 1024;
+
+    while (tsize < size) {
+      tsize *= 2;
+    }
+
+    *ppBuf = taosTRealloc(pBuf, tsize);
+    if (*ppBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+// tsdbMemory
+static FORCE_INLINE void *taosTMalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = taosMemoryMalloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = taosTMalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  taosTMemset(ret, 0);
+  return ret;
+}
+
+static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
+
+static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return taosTMalloc(size);
+
+  if (size <= taosTSizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  void* tptr1 = taosMemoryRealloc(tptr, tsize);
+  if (tptr1 == NULL) return NULL;
+  tptr = tptr1;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+static FORCE_INLINE void* taosTZfree(void* ptr) {
+  if (ptr) {
+    taosMemoryFree((void*)((char*)ptr - sizeof(size_t)));
+  }
+  return NULL;
+}
+
+// tsdbCommit
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
+
+static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
+  if (key < 0) {
+    return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
+  } else {
+    return (int)((key / tsTickPerDay[precision] / days));
+  }
+}
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
+}
+
+// tsdbDBDef
+// typedef struct SDBFile SDBFile;
+// typedef DB_ENV*        TDBEnv;
+
+// struct SDBFile {
+//   int32_t fid;
+//   DB*     pDB;
+//   char*   path;
+// };
+
+// int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
+// void    tsdbCloseDBF(SDBFile* pDBF);
+// int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path);
+// void    tsdbCloseBDBEnv(DB_ENV* pEnv);
+// int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize);
+// void*   tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize);
+
+// tsdbFile
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+#define TSDB_FILE_STATE_OK 0
+#define TSDB_FILE_STATE_BAD 1
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
+#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
+#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
+#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
+#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
+#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
+#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
+#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
+#define TSDB_FILE_STATE(tf) ((tf)->state)
+#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
+#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
+#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
+
+typedef int32_t TSDB_FILE_T;
+typedef enum {
+  TSDB_FS_VER_0 = 0,
+  TSDB_FS_VER_MAX,
+} ETsdbFsVer;
+
+#define TSDB_LATEST_FVER    TSDB_FS_VER_0  // latest version for DFile
+#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0  // latest version for 'current' file
+
+static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest version for DFile
+  switch (fType) {
+    case TSDB_FILE_HEAD:  // .head
+    case TSDB_FILE_DATA:  // .data
+    case TSDB_FILE_LAST:  // .last
+    case TSDB_FILE_SMAD:  // .smad(Block-wise SMA)
+    case TSDB_FILE_SMAL:  // .smal(Block-wise SMA)
+    default:
+      return TSDB_LATEST_FVER;
+  }
+}
+
+
+void  tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
+int   tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
+  if (pDFile->pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    taosCloseFile(&pDFile->pFile);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  // ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+
+typedef struct {
+  int      fid;
+  int8_t   state;
+  uint8_t  ver;
+  uint16_t reserve;
+#if 0
+  SDFInfo   info;
+#endif
+  STfsFile  f;
+  TdFilePtr pFile;
+
+} SSFile;  // files split by days with fid
+
+#define TSDB_LATEST_FSET_VER 0
+
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_FSET_STATE(s) ((s)->state)
+#define TSDB_FSET_VER(s) ((s)->ver)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                     \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
+    }                                                                          \
+  } while (0);
+
+void  tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsTickPerDay[precision];
+  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
+}
+
+static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+// tsdbFS
+// ================== TSDB global config
+extern bool tsdbForceKeepFile;
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+
+/**
+ * @brief Directory structure of .tsma data files.
+ *
+ *  /vnode2/tsdb $ tree tsma/
+ *   tsma/
+ *   ├── v2f100.index_name_1
+ *   ├── v2f101.index_name_1
+ *   ├── v2f102.index_name_1
+ *   ├── v2f1900.index_name_3
+ *   ├── v2f1901.index_name_3
+ *   ├── v2f1902.index_name_3
+ *   ├── v2f200.index_name_2
+ *   ├── v2f201.index_name_2
+ *   └── v2f202.index_name_2
+ *
+ *   0 directories, 9 files
+ */
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS *  pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet *pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(const STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdb *pRepo);
+void     tsdbCloseFS(STsdb *pRepo);
+void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdb *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+// void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockRdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockWrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
+  int code = taosThreadRwlockUnlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+// tsdbSma
+// #define TSDB_SMA_TEST // remove after test finished
+
+
+// struct SSmaEnv {
+//   TdThreadRwlock lock;
+//   SDiskID        did;
+//   TDBEnv         dbEnv;  // TODO: If it's better to put it in smaIndex level?
+//   char          *path;   // relative path
+//   SSmaStat      *pStat;
+// };
+
+// #define SMA_ENV_LOCK(env)       ((env)->lock)
+// #define SMA_ENV_DID(env)        ((env)->did)
+// #define SMA_ENV_ENV(env)        ((env)->dbEnv)
+// #define SMA_ENV_PATH(env)       ((env)->path)
+// #define SMA_ENV_STAT(env)       ((env)->pStat)
+// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
+
+
+// void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
+// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
+// #if 0
+// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
+// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
+// #endif
+
+// // internal func
+// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
+//   int32_t len = 0;
+//   len += taosEncodeFixedI64(pData, tsKey);
+//   len += taosEncodeFixedI64(pData, groupId);
+//   return len;
+// }
+
+// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
+//   int code = taosThreadRwlockRdlock(&(pEnv->lock));
+//   if (code != 0) {
+//     terrno = TAOS_SYSTEM_ERROR(code);
+//     return -1;
+//   }
+//   return 0;
+// }
+
+// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
+//   int code = taosThreadRwlockWrlock(&(pEnv->lock));
+//   if (code != 0) {
+//     terrno = TAOS_SYSTEM_ERROR(code);
+//     return -1;
+//   }
+//   return 0;
+// }
+
+// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
+//   int code = taosThreadRwlockUnlock(&(pEnv->lock));
+//   if (code != 0) {
+//     terrno = TAOS_SYSTEM_ERROR(code);
+//     return -1;
+//   }
+//   return 0;
+// }
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /*_TD_VNODE_TSDB_H_*/

source/dnode/vnode/src/inc/tsdbCommit.h

@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_COMMIT_H_
-#define _TD_TSDB_COMMIT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  int   minFid;
-  int   midFid;
-  int   maxFid;
-  TSKEY minKey;
-} SRtn;
-
-typedef struct {
-  uint64_t uid;
-  int64_t  offset;
-  int64_t  size;
-} SKVRecord;
-
-void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
-
-static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
-  if (key < 0) {
-    return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
-  } else {
-    return (int)((key / tsTickPerDay[precision] / days));
-  }
-}
-
-static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
-  if (fid >= pRtn->maxFid) {
-    return 0;
-  } else if (fid >= pRtn->midFid) {
-    return 1;
-  } else if (fid >= pRtn->minFid) {
-    return 2;
-  } else {
-    return -1;
-  }
-}
-
-#if 0
-#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
-
-int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
-void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
-void *tsdbCommitData(STsdbRepo *pRepo);
-int   tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
-int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx);
-int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
-int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock,
-                       bool isLast, bool isSuper, void **ppBuf, void **ppCBuf);
-int   tsdbApplyRtn(STsdbRepo *pRepo);
-
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TD_TSDB_COMMIT_H_ */

source/dnode/vnode/src/inc/tsdbCompact.h

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#if 0
-
-#ifndef _TD_TSDB_COMPACT_H_
-#define _TD_TSDB_COMPACT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void *tsdbCompactImpl(STsdbRepo *pRepo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TD_TSDB_COMPACT_H_ */
-
-#endif

source/dnode/vnode/src/inc/tsdbDBDef.h

@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_DB_DEF_H_
-#define _TD_TSDB_DB_DEF_H_
-
-#include "db.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SDBFile SDBFile;
-typedef DB_ENV*        TDBEnv;
-
-struct SDBFile {
-  int32_t fid;
-  DB*     pDB;
-  char*   path;
-};
-
-int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
-void    tsdbCloseDBF(SDBFile* pDBF);
-int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path);
-void    tsdbCloseBDBEnv(DB_ENV* pEnv);
-int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize);
-void*   tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_DB_DEF_H_*/

source/dnode/vnode/src/inc/tsdbDef.h

@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_DEF_H_
-#define _TD_TSDB_DEF_H_
-
-#include "tsdbDBDef.h"
-#include "tmallocator.h"
-#include "meta.h"
-#include "tcompression.h"
-#include "tglobal.h"
-#include "thash.h"
-#include "tlist.h"
-#include "tmsg.h"
-#include "tskiplist.h"
-#include "ttime.h"
-
-#include "tsdb.h"
-#include "tsdbCommit.h"
-#include "tsdbFS.h"
-#include "tsdbFile.h"
-#include "tsdbLog.h"
-#include "tsdbMemTable.h"
-#include "tsdbMemory.h"
-#include "tsdbOptions.h"
-#include "tsdbReadImpl.h"
-#include "tsdbSma.h"
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct STsdb {
-  int32_t               vgId;
-  bool                  repoLocked;
-  TdThreadMutex         mutex;
-  char *                path;
-  STsdbCfg              config;
-  STsdbMemTable *       mem;
-  STsdbMemTable *       imem;
-  SRtn                  rtn;
-  SMemAllocatorFactory *pmaf;
-  STsdbFS *             fs;
-  SMeta *               pMeta;
-  STfs *                pTfs;
-  SSmaEnvs              smaEnvs;
-};
-
-#define REPO_ID(r)        ((r)->vgId)
-#define REPO_CFG(r)       (&(r)->config)
-#define REPO_FS(r)        ((r)->fs)
-#define REPO_META(r)      ((r)->pMeta)
-#define REPO_TFS(r)       ((r)->pTfs)
-#define IS_REPO_LOCKED(r) ((r)->repoLocked)
-#define REPO_TSMA_NUM(r)  ((r)->smaEnvs.nTSma)
-#define REPO_RSMA_NUM(r)  ((r)->smaEnvs.nRSma)
-#define REPO_TSMA_ENV(r)  ((r)->smaEnvs.pTSmaEnv)
-#define REPO_RSMA_ENV(r)  ((r)->smaEnvs.pRSmaEnv)
-
-int tsdbLockRepo(STsdb *pTsdb);
-int tsdbUnlockRepo(STsdb *pTsdb);
-
-static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) {
-  return pTable->pSchema;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_DEF_H_*/

source/dnode/vnode/src/inc/tsdbFS.h

@@ -1,141 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_FS_H_
-#define _TD_TSDB_FS_H_
-
-#include "tsdbFile.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// ================== TSDB global config
-extern bool tsdbForceKeepFile;
-
-// ================== CURRENT file header info
-typedef struct {
-  uint32_t version;  // Current file system version (relating to code)
-  uint32_t len;      // Encode content length (including checksum)
-} SFSHeader;
-
-// ================== TSDB File System Meta
-typedef struct {
-  uint32_t version;       // Commit version from 0 to increase
-  int64_t  totalPoints;   // total points
-  int64_t  totalStorage;  // Uncompressed total storage
-} STsdbFSMeta;
-
-// ==================
-typedef struct {
-  STsdbFSMeta meta;  // FS meta
-  SArray *    df;    // data file array
-  SArray *    sf;    // sma data file array    v2f1900.index_name_1
-} SFSStatus;
-
-/**
- * @brief Directory structure of .tsma data files.
- *
- *  /vnode2/tsdb $ tree tsma/
- *   tsma/
- *   ├── v2f100.index_name_1
- *   ├── v2f101.index_name_1
- *   ├── v2f102.index_name_1
- *   ├── v2f1900.index_name_3
- *   ├── v2f1901.index_name_3
- *   ├── v2f1902.index_name_3
- *   ├── v2f200.index_name_2
- *   ├── v2f201.index_name_2
- *   └── v2f202.index_name_2
- *
- *   0 directories, 9 files
- */
-
-typedef struct {
-  TdThreadRwlock lock;
-
-  SFSStatus *cstatus;        // current status
-  SHashObj * metaCache;      // meta cache
-  SHashObj * metaCacheComp;  // meta cache for compact
-  bool       intxn;
-  SFSStatus *nstatus;  // new status
-} STsdbFS;
-
-#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
-#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
-#define FS_IN_TXN(pfs) (pfs)->intxn
-#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
-#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
-
-typedef struct {
-  int        direction;
-  uint64_t   version;  // current FS version
-  STsdbFS *  pfs;
-  int        index;  // used to position next fset when version the same
-  int        fid;    // used to seek when version is changed
-  SDFileSet *pSet;
-} SFSIter;
-
-#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
-
-STsdbFS *tsdbNewFS(const STsdbCfg *pCfg);
-void *   tsdbFreeFS(STsdbFS *pfs);
-int      tsdbOpenFS(STsdb *pRepo);
-void     tsdbCloseFS(STsdb *pRepo);
-void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
-int      tsdbEndFSTxn(STsdb *pRepo);
-int      tsdbEndFSTxnWithError(STsdbFS *pfs);
-void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
-// void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
-int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
-
-void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
-void       tsdbFSIterSeek(SFSIter *pIter, int fid);
-SDFileSet *tsdbFSIterNext(SFSIter *pIter);
-int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
-
-static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockRdlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockWrlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
-  int code = taosThreadRwlockUnlock(&(pFs->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TD_TSDB_FS_H_ */

source/dnode/vnode/src/inc/tsdbFile.h

@@ -1,435 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TS_TSDB_FILE_H_
-#define _TS_TSDB_FILE_H_
-
-#include "tchecksum.h"
-#include "tfs.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TSDB_FILE_HEAD_SIZE 512
-#define TSDB_FILE_DELIMITER 0xF00AFA0F
-#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
-#define TSDB_IVLD_FID INT_MIN
-#define TSDB_FILE_STATE_OK 0
-#define TSDB_FILE_STATE_BAD 1
-
-#define TSDB_FILE_INFO(tf) (&((tf)->info))
-#define TSDB_FILE_F(tf) (&((tf)->f))
-#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
-#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
-#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
-#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
-#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
-#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
-#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
-#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
-#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
-#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
-#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
-#define TSDB_FILE_STATE(tf) ((tf)->state)
-#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
-#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
-#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
-
-typedef enum {
-  TSDB_FILE_HEAD = 0,  // .head
-  TSDB_FILE_DATA,      // .data
-  TSDB_FILE_LAST,      // .last
-  TSDB_FILE_SMAD,      // .smad(Block-wise SMA)
-  TSDB_FILE_SMAL,      // .smal(Block-wise SMA)
-  TSDB_FILE_MAX,       //
-  TSDB_FILE_META,      // meta
-  TSDB_FILE_TSMA,      // v2t100.${sma_index_name}, Time-range-wise SMA
-  TSDB_FILE_RSMA,      // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
-} E_TSDB_FILE_T;
-
-typedef int32_t TSDB_FILE_T;
-typedef enum {
-  TSDB_FS_VER_0 = 0,
-  TSDB_FS_VER_MAX,
-} ETsdbFsVer;
-
-#define TSDB_LATEST_FVER    TSDB_FS_VER_0  // latest version for DFile
-#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0  // latest version for 'current' file
-
-static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest version for DFile
-  switch (fType) {
-    case TSDB_FILE_HEAD:  // .head
-    case TSDB_FILE_DATA:  // .data
-    case TSDB_FILE_LAST:  // .last
-    case TSDB_FILE_SMAD:  // .smad(Block-wise SMA)
-    case TSDB_FILE_SMAL:  // .smal(Block-wise SMA)
-    default:
-      return TSDB_LATEST_FVER;
-  }
-}
-
-#if 0
-// =============== SMFile
-typedef struct {
-  int64_t  size;
-  int64_t  tombSize;
-  int64_t  nRecords;
-  int64_t  nDels;
-  uint32_t magic;
-} SMFInfo;
-
-typedef struct {
-  SMFInfo  info;
-  STfsFile f;
-  int      fd;
-  uint8_t  state;
-} SMFile;
-
-void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
-void  tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
-int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
-void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
-int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
-void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
-int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
-int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
-int   tsdbUpdateMFileHeader(SMFile* pMFile);
-int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
-int   tsdbScanAndTryFixMFile(STsdb* pRepo);
-int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
-void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
-
-static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
-
-static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
-  ASSERT(TSDB_FILE_CLOSED(pMFile));
-
-  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
-  if (pMFile->fd < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
-  if (TSDB_FILE_OPENED(pMFile)) {
-    close(pMFile->fd);
-    TSDB_FILE_SET_CLOSED(pMFile);
-  }
-}
-
-static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pMFile), offset, whence);
-  if (loffset < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return loffset;
-}
-
-static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  int64_t nwrite = taosWriteFile(pMFile->fd, buf, nbyte);
-  if (nwrite < nbyte) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nwrite;
-}
-
-static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
-  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
-}
-
-static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  int64_t toffset;
-
-  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
-    return -1;
-  }
-
-  ASSERT(pMFile->info.size == toffset);
-
-  if (offset) {
-    *offset = toffset;
-  }
-
-  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
-    return -1;
-  }
-
-  pMFile->info.size += nbyte;
-
-  return (int)nbyte;
-}
-
-static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
-
-static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  int64_t nread = taosReadFile(pMFile->fd, buf, nbyte);
-  if (nread < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nread;
-}
-
-#endif
-
-// =============== SDFile
-typedef struct {
-  uint32_t magic;
-  uint32_t fver;
-  uint32_t len;
-  uint32_t totalBlocks;
-  uint32_t totalSubBlocks;
-  uint32_t offset;
-  uint64_t size;
-  uint64_t tombSize;
-} SDFInfo;
-
-typedef struct {
-  SDFInfo   info;
-  STfsFile  f;
-  TdFilePtr pFile;
-  uint8_t   state;
-} SDFile;
-
-void  tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
-void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
-int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
-void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
-int   tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType);
-int   tsdbUpdateDFileHeader(SDFile* pDFile);
-int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
-int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
-
-static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
-
-static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
-  ASSERT(!TSDB_FILE_OPENED(pDFile));
-
-  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
-  if (pDFile->pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
-  if (TSDB_FILE_OPENED(pDFile)) {
-    taosCloseFile(&pDFile->pFile);
-    TSDB_FILE_SET_CLOSED(pDFile);
-  }
-}
-
-static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
-  // ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
-  if (loffset < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return loffset;
-}
-
-static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
-  if (nwrite < nbyte) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nwrite;
-}
-
-static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
-  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
-}
-
-static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t toffset;
-
-  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
-    return -1;
-  }
-
-  ASSERT(pDFile->info.size == toffset);
-
-  if (offset) {
-    *offset = toffset;
-  }
-
-  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
-    return -1;
-  }
-
-  pDFile->info.size += nbyte;
-
-  return (int)nbyte;
-}
-
-static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
-
-static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
-  ASSERT(TSDB_FILE_OPENED(pDFile));
-
-  int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
-  if (nread < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return nread;
-}
-
-static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
-  if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
-  return 0;
-}
-
-// =============== SDFileSet
-typedef struct {
-  int      fid;
-  int8_t   state;  // -128~127
-  uint8_t  ver;    // 0~255, DFileSet version
-  uint16_t reserve;
-  SDFile   files[TSDB_FILE_MAX];
-} SDFileSet;
-
-typedef struct {
-  int      fid;
-  int8_t   state;
-  uint8_t  ver;
-  uint16_t reserve;
-#if 0
-  SDFInfo   info;
-#endif
-  STfsFile  f;
-  TdFilePtr pFile;
-
-} SSFile;  // files split by days with fid
-
-#define TSDB_LATEST_FSET_VER 0
-
-#define TSDB_FSET_FID(s) ((s)->fid)
-#define TSDB_FSET_STATE(s) ((s)->state)
-#define TSDB_FSET_VER(s) ((s)->ver)
-#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
-#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
-#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
-#define TSDB_FSET_SET_CLOSED(s)                                                \
-  do {                                                                         \
-    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
-      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
-    }                                                                          \
-  } while (0);
-#define TSDB_FSET_FSYNC(s)                                                     \
-  do {                                                                         \
-    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
-      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
-    }                                                                          \
-  } while (0);
-
-void  tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver);
-void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
-int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
-void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet);
-int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
-void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
-int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
-int   tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader);
-int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
-int   tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
-
-static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
-  }
-}
-
-static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
-      tsdbCloseDFileSet(pSet);
-      return -1;
-    }
-  }
-  return 0;
-}
-
-static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
-  }
-}
-
-static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
-      tsdbRemoveDFileSet(pDest);
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
-  *minKey = fid * days * tsTickPerDay[precision];
-  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
-}
-
-static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
-  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
-    if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TS_TSDB_FILE_H_ */

source/dnode/vnode/src/inc/tsdbLog.h

@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_LOG_H_
-#define _TD_TSDB_LOG_H_
-
-#include "tlog.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int32_t tsdbDebugFlag;
-
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TD_TSDB_LOG_H_ */

source/dnode/vnode/src/inc/tsdbMemTable.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_MEM_TABLE_H_
-#define _TD_TSDB_MEM_TABLE_H_
-
-#include "tsdb.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  int   rowsInserted;
-  int   rowsUpdated;
-  int   rowsDeleteSucceed;
-  int   rowsDeleteFailed;
-  int   nOperations;
-  TSKEY keyFirst;
-  TSKEY keyLast;
-} SMergeInfo;
-
-typedef struct STbData {
-  tb_uid_t   uid;
-  TSKEY      keyMin;
-  TSKEY      keyMax;
-  int64_t    nrows;
-  SSkipList *pData;
-} STbData;
-
-typedef struct STsdbMemTable {
-  T_REF_DECLARE()
-  SRWLatch       latch;
-  TSKEY          keyMin;
-  TSKEY          keyMax;
-  uint64_t       nRow;
-  SMemAllocator *pMA;
-  // Container
-  SSkipList *pSlIdx;  // SSkiplist<STbData>
-  SHashObj * pHashIdx;
-} STsdbMemTable;
-
-STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
-void           tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
-int            tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
-int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
-                          TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
-
-static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
-  if (pIter == NULL) return NULL;
-
-  SSkipListNode *node = tSkipListIterGet(pIter);
-  if (node == NULL) return NULL;
-
-  return (STSRow *)SL_GET_NODE_DATA(node);
-}
-
-static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
-  STSRow *row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
-
-  return TD_ROW_KEY(row);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_MEM_TABLE_H_*/

source/dnode/vnode/src/inc/tsdbMemory.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_MEMORY_H_
-#define _TD_TSDB_MEMORY_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-static void * taosTMalloc(size_t size);
-static void * taosTCalloc(size_t nmemb, size_t size);
-static void * taosTRealloc(void *ptr, size_t size);
-static void * taosTZfree(void *ptr);
-static size_t taosTSizeof(void *ptr);
-static void   taosTMemset(void *ptr, int c);
-
-static FORCE_INLINE void *taosTMalloc(size_t size) {
-  if (size <= 0) return NULL;
-
-  void *ret = taosMemoryMalloc(size + sizeof(size_t));
-  if (ret == NULL) return NULL;
-
-  *(size_t *)ret = size;
-
-  return (void *)((char *)ret + sizeof(size_t));
-}
-
-static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
-  size_t tsize = nmemb * size;
-  void * ret = taosTMalloc(tsize);
-  if (ret == NULL) return NULL;
-
-  taosTMemset(ret, 0);
-  return ret;
-}
-
-static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
-
-static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
-
-static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
-  if (ptr == NULL) return taosTMalloc(size);
-
-  if (size <= taosTSizeof(ptr)) return ptr;
-
-  void * tptr = (void *)((char *)ptr - sizeof(size_t));
-  size_t tsize = size + sizeof(size_t);
-  void* tptr1 = taosMemoryRealloc(tptr, tsize);
-  if (tptr1 == NULL) return NULL;
-  tptr = tptr1;
-
-  *(size_t *)tptr = size;
-
-  return (void *)((char *)tptr + sizeof(size_t));
-}
-
-static FORCE_INLINE void* taosTZfree(void* ptr) {
-  if (ptr) {
-    taosMemoryFree((void*)((char*)ptr - sizeof(size_t)));
-  }
-  return NULL;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _TD_TSDB_MEMORY_H_ */

source/dnode/vnode/src/inc/tsdbOptions.h

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_OPTIONS_H_
-#define _TD_TSDB_OPTIONS_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern const STsdbCfg defautlTsdbOptions;
-
-int  tsdbValidateOptions(const STsdbCfg *);
-void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_OPTIONS_H_*/

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -1,256 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_READ_IMPL_H_
-#define _TD_TSDB_READ_IMPL_H_
-
-#include "os.h"
-#include "tcommon.h"
-#include "tfs.h"
-#include "tsdb.h"
-#include "tsdbFile.h"
-#include "tsdbMemory.h"
-#include "tskiplist.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SReadH SReadH;
-
-typedef struct {
-  uint32_t len;
-  uint32_t offset;
-  uint32_t hasLast : 2;
-  uint32_t numOfBlocks : 30;
-  uint64_t uid;
-  TSKEY    maxKey;
-} SBlockIdx;
-
-#ifdef TD_REFACTOR_3
-typedef struct {
-  int64_t last : 1;
-  int64_t offset : 63;
-  int32_t algorithm : 8;
-  int32_t numOfRows : 24;
-  int32_t len;
-  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
-  int16_t numOfSubBlocks;
-  int16_t numOfCols;  // not including timestamp column
-  TSKEY   keyFirst;
-  TSKEY   keyLast;
-} SBlock;
-
-#else
-
-typedef enum {
-  TSDB_SBLK_VER_0 = 0,
-  TSDB_SBLK_VER_MAX,
-} ESBlockVer;
-
-#define SBlockVerLatest TSDB_SBLK_VER_0
-
-typedef struct {
-  uint8_t  last : 1;
-  uint8_t  blkVer : 7;
-  uint8_t  numOfSubBlocks;
-  col_id_t numOfCols;    // not including timestamp column
-  uint32_t len;          // data block length
-  uint32_t keyLen : 20;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
-  uint32_t algorithm : 4;
-  uint32_t reserve : 8;
-  col_id_t numOfBSma;
-  uint16_t numOfRows;
-  int64_t  offset;
-  uint64_t aggrStat : 1;
-  uint64_t aggrOffset : 63;
-  TSKEY    keyFirst;
-  TSKEY    keyLast;
-} SBlockV0;
-
-#define SBlock SBlockV0  // latest SBlock definition
-
-#endif
-
-typedef struct {
-  int32_t  delimiter;  // For recovery usage
-  int32_t  tid;
-  uint64_t uid;
-  SBlock   blocks[];
-} SBlockInfo;
-
-#ifdef TD_REFACTOR_3
-typedef struct {
-  int16_t  colId;
-  uint16_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
-  uint16_t reserve : 15;
-  int32_t  len;
-  uint32_t type : 8;
-  uint32_t offset : 24;
-  int64_t  sum;
-  int64_t  max;
-  int64_t  min;
-  int16_t  maxIndex;
-  int16_t  minIndex;
-  int16_t  numOfNull;
-  uint8_t  offsetH;
-  char     padding[1];
-} SBlockCol;
-#else
-typedef struct {
-  int16_t  colId;
-  uint16_t type : 6;
-  uint16_t blen : 10;   // bitmap length(TODO: full UT for the bitmap compress of various data input)
-  uint32_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
-  uint32_t len : 31;    // data length + bitmap length
-  uint32_t offset;
-} SBlockColV0;
-
-#define SBlockCol SBlockColV0  // latest SBlockCol definition
-
-typedef struct {
-  int16_t colId;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-} SAggrBlkColV0;
-
-#define SAggrBlkCol SAggrBlkColV0  // latest SAggrBlkCol definition
-
-#endif
-
-// Code here just for back-ward compatibility
-static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
-#ifdef TD_REFACTOR_3
-  pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
-  pBlockCol->offsetH = (uint8_t)(offset >> 24);
-#else
-  pBlockCol->offset = offset;
-#endif
-}
-
-static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
-#ifdef TD_REFACTOR_3
-  uint32_t offset1 = pBlockCol->offset;
-  uint32_t offset2 = pBlockCol->offsetH;
-  return (offset1 | (offset2 << 24));
-#else
-  return pBlockCol->offset;
-#endif
-}
-
-typedef struct {
-  int32_t   delimiter;  // For recovery usage
-  int32_t   numOfCols;  // For recovery usage
-  uint64_t  uid;        // For recovery usage
-  SBlockCol cols[];
-} SBlockData;
-
-typedef void SAggrBlkData;  // SBlockCol cols[];
-
-struct SReadH {
-  STsdb        *pRepo;
-  SDFileSet     rSet;     // FSET to read
-  SArray       *aBlkIdx;  // SBlockIdx array
-  STable       *pTable;   // table to read
-  SBlockIdx    *pBlkIdx;  // current reading table SBlockIdx
-  int           cidx;
-  SBlockInfo   *pBlkInfo;
-  SBlockData   *pBlkData;      // Block info
-  SAggrBlkData *pAggrBlkData;  // Aggregate Block info
-  SDataCols    *pDCols[2];
-  void         *pBuf;    // buffer
-  void         *pCBuf;   // compression buffer
-  void         *pExBuf;  // extra buffer
-};
-
-#define TSDB_READ_REPO(rh)      ((rh)->pRepo)
-#define TSDB_READ_REPO_ID(rh)   REPO_ID(TSDB_READ_REPO(rh))
-#define TSDB_READ_FSET(rh)      (&((rh)->rSet))
-#define TSDB_READ_TABLE(rh)     ((rh)->pTable)
-#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
-#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
-#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
-#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
-#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
-#define TSDB_READ_BUF(rh)       ((rh)->pBuf)
-#define TSDB_READ_COMP_BUF(rh)  ((rh)->pCBuf)
-#define TSDB_READ_EXBUF(rh)     ((rh)->pExBuf)
-
-#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
-  (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
-
-static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
-  switch (blkVer) {
-    case TSDB_SBLK_VER_0:
-    default:
-      return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
-  }
-}
-
-#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
-
-static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
-  switch (blkVer) {
-    case TSDB_SBLK_VER_0:
-    default:
-      return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
-  }
-}
-
-int   tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
-void  tsdbDestroyReadH(SReadH *pReadh);
-int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
-void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
-int   tsdbLoadBlockIdx(SReadH *pReadh);
-int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
-int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
-int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
-int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
-                            int numOfColsIds);
-int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
-int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
-void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
-void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
-
-static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
-  void  *pBuf = *ppBuf;
-  size_t tsize = taosTSizeof(pBuf);
-
-  if (tsize < size) {
-    if (tsize == 0) tsize = 1024;
-
-    while (tsize < size) {
-      tsize *= 2;
-    }
-
-    *ppBuf = taosTRealloc(pBuf, tsize);
-    if (*ppBuf == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TSDB_READ_IMPL_H_*/

source/dnode/vnode/src/inc/tsdbSma.h

@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_SMA_H_
-#define _TD_TSDB_SMA_H_
-
-#define TSDB_SMA_TEST // remove after test finished
-
-typedef struct SSmaStat SSmaStat;
-typedef struct SSmaEnv  SSmaEnv;
-typedef struct SSmaEnvs SSmaEnvs;
-
-struct SSmaEnv {
-  TdThreadRwlock lock;
-  SDiskID        did;
-  TDBEnv         dbEnv;  // TODO: If it's better to put it in smaIndex level?
-  char          *path;   // relative path
-  SSmaStat      *pStat;
-};
-
-#define SMA_ENV_LOCK(env)       ((env)->lock)
-#define SMA_ENV_DID(env)        ((env)->did)
-#define SMA_ENV_ENV(env)        ((env)->dbEnv)
-#define SMA_ENV_PATH(env)       ((env)->path)
-#define SMA_ENV_STAT(env)       ((env)->pStat)
-#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
-
-struct SSmaEnvs {
-  int16_t  nTSma;
-  int16_t  nRSma;
-  SSmaEnv *pTSmaEnv;
-  SSmaEnv *pRSmaEnv;
-};
-
-void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-#if 0
-int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
-int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-#endif
-
-// internal func
-static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
-  int32_t len = 0;
-  len += taosEncodeFixedI64(pData, tsKey);
-  len += taosEncodeFixedI64(pData, groupId);
-  return len;
-}
-
-static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
-  int code = taosThreadRwlockRdlock(&(pEnv->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
-  int code = taosThreadRwlockWrlock(&(pEnv->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
-  int code = taosThreadRwlockUnlock(&(pEnv->lock));
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}
-
-#endif /* _TD_TSDB_SMA_H_ */

source/dnode/vnode/src/inc/vnodeInt.h

@@ -16,17 +16,23 @@
 #ifndef _TD_VNODE_DEF_H_
 #define _TD_VNODE_DEF_H_
 
-#include "tmallocator.h"
+#include "executor.h"
-// #include "sync.h"
+#include "tchecksum.h"
 #include "tcoding.h"
+#include "tcompression.h"
 #include "tdatablock.h"
 #include "tfs.h"
+#include "tglobal.h"
 #include "tlist.h"
 #include "tlockfree.h"
 #include "tmacro.h"
+#include "tmallocator.h"
+#include "tskiplist.h"
+#include "ttime.h"
+#include "ttimer.h"
 #include "vnode.h"
-#include "vnodeQuery.h"
 #include "wal.h"
+#include "qworker.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,8 +40,9 @@ extern "C" {
 
 typedef struct STQ STQ;
 
-typedef struct SVState   SVState;
+typedef struct SVState      SVState;
-typedef struct SVBufPool SVBufPool;
+typedef struct SVBufPool    SVBufPool;
+typedef struct SQWorkerMgmt SQHandle;
 
 typedef struct SVnodeTask {
   TD_DLIST_NODE(SVnodeTask);
@@ -95,7 +102,9 @@ struct SVnode {
   STfs*      pTfs;
 };
 
-int vnodeScheduleTask(SVnodeTask* task);
+int  vnodeScheduleTask(SVnodeTask* task);
+int  vnodeQueryOpen(SVnode* pVnode);
+void vnodeQueryClose(SVnode* pVnode);
 
 #define vFatal(...)                                              \
   do {                                                           \
@@ -205,6 +214,12 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
+#include "meta.h"
+
+#include "tsdb.h"
+
+#include "tq.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeQuery.h

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_READ_H_
-#define _TD_VNODE_READ_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "qworker.h"
-#include "vnode.h"
-
-
-typedef struct SQWorkerMgmt SQHandle;
-
-int vnodeQueryOpen(SVnode *pVnode);
-void vnodeQueryClose(SVnode *pVnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_READ_H_*/

source/dnode/vnode/src/meta/metaCache.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "meta.h"
+#include "vnodeInt.h"
-#include "metaDef.h"
 
 struct SMetaCache {
   // TODO

source/dnode/vnode/src/meta/metaCfg.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "metaDef.h"
+#include "vnodeInt.h"
 
 const SMetaCfg defaultMetaOptions = {.lruSize = 0};
 

source/dnode/vnode/src/meta/metaIdx.c

@@ -16,7 +16,7 @@
 #ifdef USE_INVERTED_INDEX
 #include "index.h"
 #endif
-#include "metaDef.h"
+#include "vnodeInt.h"
 
 struct SMetaIdx {
 #ifdef USE_INVERTED_INDEX

source/dnode/vnode/src/meta/metaMain.c

@@ -15,7 +15,7 @@
 
 #include "tcoding.h"
 
-#include "metaDef.h"
+#include "vnodeInt.h"
 
 static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
 static void   metaFree(SMeta *pMeta);

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "metaDef.h"
+#include "vnodeInt.h"
 
 #include "tdbInt.h"
 typedef struct SPoolMem {

source/dnode/vnode/src/meta/metaTable.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "metaDef.h"
+#include "vnodeInt.h"
 
 int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) {
   // Validate the tbOptions

source/dnode/vnode/src/meta/metaTbCfg.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "metaDef.h"
+#include "vnodeInt.h"
 #include "tcoding.h"
 
 int metaValidateTbCfg(SMeta *pMeta, const STbCfg *pTbOptions) {

source/dnode/vnode/src/meta/metaTbUid.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "meta.h"
+#include "vnodeInt.h"
-#include "metaDef.h"
 
 int metaOpenUidGnrt(SMeta *pMeta) {
   // Init a generator

source/dnode/vnode/src/tq/tq.c

@@ -15,9 +15,8 @@
 
 #include "tcompare.h"
 #include "tdatablock.h"
-#include "tqInt.h"
-#include "tqMetaStore.h"
 #include "tstream.h"
+#include "vnodeInt.h"
 
 int32_t tqInit() { return tqPushMgrInit(); }
 
@@ -83,9 +82,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
   memcpy(data, msg, msgLen);
 
   if (msgType == TDMT_VND_SUBMIT) {
-    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
+    // if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
-      return -1;
+    //   return -1;
-    }
+    // }
   }
 
   SRpcMsg req = {
@@ -272,7 +271,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
+  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+         pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
 
   SMqPollRsp rsp = {
       /*.consumerId = consumerId,*/
@@ -296,10 +296,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 
   STqTopic* pTopic = NULL;
-  int sz = taosArrayGetSize(pConsumer->topics);
+  int       sz = taosArrayGetSize(pConsumer->topics);
   for (int32_t i = 0; i < sz; i++) {
     STqTopic* topic = taosArrayGet(pConsumer->topics, i);
-    //TODO race condition
+    // TODO race condition
     ASSERT(pConsumer->consumerId == consumerId);
     if (strcmp(topic->topicName, pReq->topic) == 0) {
       pTopic = topic;
@@ -307,7 +307,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     }
   }
   if (pTopic == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, pTq->pVnode->vgId);
+    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
+          pTq->pVnode->vgId);
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
@@ -315,17 +316,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     return 0;
   }
 
-  vDebug("poll topic %s from consumer %ld (epoch %d)", pTopic->topicName, consumerId, pReq->epoch);
+  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, pTq->pVnode->vgId);
 
   rsp.reqOffset = pReq->currentOffset;
   rsp.skipLogNum = 0;
 
   while (1) {
     /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
-    //TODO
+    // TODO
     consumerEpoch = atomic_load_32(&pConsumer->epoch);
-    if (consumerEpoch > pReq->epoch) {
+    if (consumerEpoch > reqEpoch) {
-      //TODO: return
+      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
     SWalReadHead* pHead;

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "tqMetaStore.h"
+#include "vnodeInt.h"
 // TODO:replace by an abstract file layer
 #include <fcntl.h>
 #include <string.h>

source/dnode/vnode/src/tq/tqOffset.c

@@ -14,7 +14,7 @@
  */
 #define _DEFAULT_SOURCE
 
-#include "tqOffset.h"
+#include "vnodeInt.h"
 
 enum ETqOffsetPersist {
   TQ_OFFSET_PERSIST__LAZY = 1,
@@ -39,4 +39,3 @@ STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) {
   pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
   return pStore;
 }
-

source/dnode/vnode/src/tq/tqPush.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tqPush.h"
+#include "vnodeInt.h"
 
 int32_t tqPushMgrInit() {
   //

source/dnode/vnode/src/tq/tqRead.c

@@ -167,8 +167,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {
+      /*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/
-        /*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/
+      if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
         taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
         return NULL;
       }

source/dnode/vnode/src/tsdb/tsdbBDBImpl.c

@@ -19,8 +19,6 @@
 #include "taoserror.h"
 #include "tcoding.h"
 #include "thash.h"
-#include "tsdbDBDef.h"
-#include "tsdbLog.h"
 
 #define IMPL_WITH_LOCK 1
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 #define TSDB_MAX_SUBBLOCKS 8
 

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -14,7 +14,7 @@
  */
 
 #include <regex.h>
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 #include "os.h"
 
 typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 static const char *TSDB_FNAME_SUFFIX[] = {
     "head",  // TSDB_FILE_HEAD

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF,
                       SMeta *pMeta, STfs *pTfs);
@@ -87,8 +87,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
 
 static void tsdbFree(STsdb *pTsdb) {
   if (pTsdb) {
-    tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
+    // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
-    tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
+    // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
     tsdbFreeFS(pTsdb->fs);
     taosMemoryFreeClear(pTsdb->path);
     taosMemoryFree(pTsdb);
@@ -98,7 +98,7 @@ static void tsdbFree(STsdb *pTsdb) {
 static int tsdbOpenImpl(STsdb *pTsdb) {
   tsdbOpenFS(pTsdb);
 
-  tsdbInitSma(pTsdb);
+  // tsdbInitSma(pTsdb);
   // TODO
 
   return 0;

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 static int      tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
 static int      tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows);

source/dnode/vnode/src/tsdb/tsdbOptions.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 const STsdbCfg defautlTsdbOptions = {.precision = 0,
                                      .lruCacheSize = 0,

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -13,25 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 #include "tdatablock.h"
 #include "os.h"
 #include "talgo.h"
 #include "tcompare.h"
 #include "tdataformat.h"
 #include "texception.h"
-#include "tsdb.h"
-#include "tsdbFS.h"
-#include "tsdbLog.h"
-#include "tsdbReadImpl.h"
-#include "tskiplist.h"
-#include "ttime.h"
 
 #include "taosdef.h"
 #include "tlosertree.h"
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 #include "tmsg.h"
-#include "tsdbCommit.h"
 
 #define EXTRA_BYTES 2
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 #define TSDB_KEY_COL_OFFSET 0
 

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 static const char *TSDB_SMA_DNAME[] = {
     "",      // TSDB_SMA_TYPE_BLOCK
@@ -346,6 +346,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
     }
     taosHashCleanup(pSmaStat->smaStatItems);
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
@@ -450,8 +451,10 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t
              winSKey);
     return TSDB_CODE_FAILED;
   }
+
   tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", REPO_ID(pTsdb), indexUid,
             winSKey);
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -1099,6 +1102,7 @@ static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) {
   }
   // clear sma data files
   // TODO:
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) {
@@ -1221,6 +1225,7 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv
   pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true);
   pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit);
   pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel);
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -1240,6 +1245,7 @@ static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey)
   pSmaH->dFile.path = strdup(tSmaFile);
   pSmaH->smaFsIter.iter = 0;
   pSmaH->smaFsIter.fid = fid;
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -1431,6 +1437,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) {
 
   tdDestroyTSma(&vCreateSmaReq.tSma);
   // TODO: return directly or go on follow steps?
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
@@ -1460,6 +1467,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
   tsdbTSmaSub(pTsdb, 1);  
 
   // TODO: return directly or go on follow steps?
+  return TSDB_CODE_SUCCESS;
 }
 
 #if 0

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbDef.h"
+#include "vnodeInt.h"
 
 /**
  * @brief insert TS data

source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
+#include "vnodeInt.h"
 
 static SVArenaNode *vArenaNodeNew(uint64_t capacity);
 static void         vArenaNodeFree(SVArenaNode *pNode);

source/dnode/vnode/src/vnd/vnodeBufferPool.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
+#include "vnodeInt.h"
 
 /* ------------------------ STRUCTURES ------------------------ */
 #define VNODE_BUF_POOL_SHARDS 3

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
+#include "vnodeInt.h"
 
 const SVnodeCfg defaultVnodeOptions = {
     .wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */