Merge branch '3.0' into feature/tq

include/common/taosmsg.h

@@ -221,8 +221,7 @@ typedef struct SBuildTableMetaInput {
 
 typedef struct SBuildUseDBInput {
   char    db[TSDB_TABLE_FNAME_LEN];
-  int32_t vgroupVersion;
+  int32_t vgVersion;
-  int32_t dbGroupVersion;
 } SBuildUseDBInput;
 
 
@@ -627,8 +626,7 @@ typedef struct {
 typedef struct {
   char    db[TSDB_TABLE_FNAME_LEN];
   int8_t  ignoreNotExists;
-  int32_t vgroupVersion;
+  int32_t vgVersion;
-  int32_t dbGroupVersion;
   int32_t reserve[8];
 } SUseDbMsg;
 
@@ -808,6 +806,9 @@ typedef struct SSTableVgroupMsg {
 
 typedef struct SVgroupInfo {
   int32_t    vgId;
+  int32_t    hashBegin;
+  int32_t    hashEnd;
+  int8_t     inUse;
   int8_t     numOfEps;
   SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
 } SVgroupInfo;
@@ -841,7 +842,7 @@ typedef struct {
   int32_t    tversion;
   uint64_t   tuid;
   uint64_t   suid;
-  SVgroupMsg vgroup;
+  int32_t    vgId;
   SSchema    pSchema[];
 } STableMetaMsg;
 
@@ -863,15 +864,12 @@ typedef struct {
 } STagData;
 
 typedef struct {
-  int32_t     vgroupNum;
+  char        db[TSDB_FULL_DB_NAME_LEN];
-  int32_t     vgroupVersion;
+  int32_t     vgVersion;
-  char        db[TSDB_TABLE_FNAME_LEN];
+  int32_t     vgNum;  
-  int32_t     dbVgroupVersion;
+  int8_t      hashMethod;
-  int32_t     dbVgroupNum;
-  int32_t     dbHashRange;
   SVgroupInfo vgroupInfo[];
-//int32_t     vgIdList[];
+} SUseDbRsp;
-} SUseDbRspMsg;
 
 
 

include/common/tglobal.h

@@ -29,12 +29,7 @@ extern char     tsLocalFqdn[];
 extern char     tsLocalEp[];
 extern uint16_t tsServerPort;
 extern int32_t  tsStatusInterval;
-extern int32_t  tsNumOfMnodes;
-extern int8_t   tsEnableVnodeBak;
 extern int8_t   tsEnableTelemetryReporting;
-extern char     tsArbitrator[];
-extern int8_t   tsArbOnline;
-extern int64_t  tsArbOnlineTimestamp;
 
 // common
 extern int      tsRpcTimer;
@@ -60,7 +55,6 @@ extern int      tsCompatibleModel;     // 2.0 compatible model
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer size in MB for each data node during query processing
 extern int64_t  tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing
 extern int32_t  tsRetrieveBlockingModel;// retrieve threads will be blocked
-
 extern int8_t   tsKeepOriginalColumnName;
 
 // client
@@ -78,66 +72,15 @@ extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the
 extern int32_t tsProjectExecInterval;
 extern int64_t tsMaxRetentWindow;
 
-// db parameters in client
-extern int32_t tsCacheBlockSize;
-extern int32_t tsBlocksPerVnode;
-extern int32_t tsTableIncStepPerVnode;
-extern int32_t tsMaxVgroupsPerDb;
-extern int16_t tsDaysPerFile;
-extern int32_t tsDaysToKeep;
-extern int32_t tsMinRowsInFileBlock;
-extern int32_t tsMaxRowsInFileBlock;
-extern int16_t tsCommitTime;  // seconds
-extern int32_t tsTimePrecision;
-extern int8_t  tsCompression;
-extern int8_t  tsWAL;
-extern int32_t tsFsyncPeriod;
-extern int32_t tsReplications;
-extern int16_t tsPartitons;
-extern int32_t tsQuorum;
-extern int8_t  tsUpdate;
-extern int8_t  tsCacheLastRow;
-
-//tsdb 
-extern bool tsdbForceKeepFile;
-
 // balance
-extern int8_t  tsEnableBalance;
-extern int8_t  tsAlternativeRole;
-extern int32_t tsBalanceInterval;
-extern int32_t tsOfflineThreshold;
-extern int8_t  tsEnableFlowCtrl;
 extern int8_t  tsEnableSlaveQuery;
-extern int8_t  tsEnableAdjustMaster;
 
-// restful
-extern int32_t  tsRestRowLimit;
-extern int8_t   tsTelegrafUseFieldNum;
 
-// mqtt
+// interna
-extern int8_t tsEnableMqttModule;
-extern char   tsMqttHostName[];
-extern char   tsMqttPort[];
-extern char   tsMqttUser[];
-extern char   tsMqttPass[];
-extern char   tsMqttClientId[];
-extern char   tsMqttTopic[];
-
-// monitor
-extern int8_t  tsEnableMonitorModule;
-extern char    tsMonitorDbName[];
-extern char    tsInternalPass[];
-extern int32_t tsMonitorInterval;
-
-// stream
-extern int8_t tsEnableStream;
-
-// internal
 extern int8_t  tsPrintAuth;
 extern char    tsVnodeDir[];
 extern char    tsMnodeDir[];
 extern int64_t tsTickPerDay[3];
-extern int32_t tsTopicBianryLen;
 
 // system info
 extern float   tsTotalLogDirGB;

include/libs/catalog/catalog.h

@@ -32,7 +32,7 @@ extern "C" {
 struct SCatalog;
 
 typedef struct SCatalogReq {
-  char    clusterId[TSDB_CLUSTER_ID_LEN];  //????
+  char    dbName[TSDB_DB_NAME_LEN];
   SArray *pTableName;     // table full name
   SArray *pUdf;           // udf name
   bool    qNodeRequired;  // valid qnode
@@ -45,42 +45,10 @@ typedef struct SMetaData {
   SEpSet    *pEpSet;      // qnode epset list
 } SMetaData;
 
-typedef struct STableComInfo {
-  uint8_t numOfTags;      // the number of tags in schema
-  uint8_t precision;      // the number of precision
-  int16_t numOfColumns;   // the number of columns
-  int32_t rowSize;        // row size of the schema
-} STableComInfo;
-
-/*
- * ASSERT(sizeof(SCTableMeta) == 24)
- * ASSERT(tableType == TSDB_CHILD_TABLE)
- * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
- */
-typedef struct SCTableMeta {
-  int32_t  vgId:24;
-  int8_t   tableType;
-  uint64_t uid;
-  uint64_t suid;
-} SCTableMeta;
-
-/*
- * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
- */
-typedef struct STableMeta {
-  int32_t        vgId:24;
-  int8_t         tableType;
-  uint64_t       uid;
-  uint64_t       suid;
-  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
-  int16_t        sversion;
-  int16_t        tversion;
-  STableComInfo  tableInfo;
-  SSchema        schema[];
-} STableMeta;
-
 typedef struct SCatalogCfg {
-
+  bool     enableVgroupCache;
+  uint32_t maxTblCacheNum;
+  uint32_t maxDBCacheNum;
 } SCatalogCfg;
 
 int32_t catalogInit(SCatalogCfg *cfg);
@@ -93,22 +61,14 @@ int32_t catalogInit(SCatalogCfg *cfg);
  */
 int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle);
 
-
-
-int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version);
-int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList);
-int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup);
-
-
-
 int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
-int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo);
+int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo);
-int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
+int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
 
 
-int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta);
+int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
-int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta);
+int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName);
-int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta);
+int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
 
 
 /**
@@ -117,7 +77,7 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const
  * @pVgroupList  - array of SVgroupInfo
  * @return
  */
-int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList);
+int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList);
 
 
 /**
@@ -129,7 +89,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe
  * @param pMetaData
  * @return
  */
-int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
+int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
 
 int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet);

include/libs/parser/parser.h

@@ -132,13 +132,15 @@ struct SInsertStmtInfo;
 bool qIsInsertSql(const char* pStr, size_t length);
 
 typedef struct SParseContext {
+  const char* pAcctId;
+  const char* pDbname;
+  void *pRpc;
+  const char* pClusterId;
+  const SEpSet* pEpSet;
+  int64_t id;          // query id, generated by uuid generator
+  int8_t schemaAttached; // denote if submit block is built with table schema or not
   const char* pSql;    // sql string
   size_t sqlLen;       // length of the sql string
-  int64_t id;          // operator id, generated by uuid generator
-  const char* pDbname;
-  const SEpSet* pEpSet; 
-  int8_t schemaAttached; // denote if submit block is built with table schema or not
-
   char* pMsg;           // extended error message if exists to help avoid the problem in sql statement.
   int32_t msgLen;      // max length of the msg
 } SParseContext;

include/libs/planner/planner.h

@@ -20,52 +20,92 @@
 extern "C" {
 #endif
 
+#include "taosmsg.h"
+
 #define QUERY_TYPE_MERGE       1
 #define QUERY_TYPE_PARTIAL     2
 #define QUERY_TYPE_SCAN        3
 
 enum OPERATOR_TYPE_E {
-  OP_TableScan         = 1,
+  OP_Unknown,
-  OP_DataBlocksOptScan = 2,
+#define INCLUDE_AS_ENUM
-  OP_TableSeqScan      = 3,
+#include "plannerOp.h"
-  OP_TagScan           = 4,
+#undef INCLUDE_AS_ENUM
-  OP_TableBlockInfoScan= 5,
+  OP_TotalNum
-  OP_Aggregate         = 6,
-  OP_Project           = 7,
-  OP_Groupby           = 8,
-  OP_Limit             = 9,
-  OP_SLimit            = 10,
-  OP_TimeWindow        = 11,
-  OP_SessionWindow     = 12,
-  OP_StateWindow       = 22,
-  OP_Fill              = 13,
-  OP_MultiTableAggregate     = 14,
-  OP_MultiTableTimeInterval  = 15,
-//  OP_DummyInput        = 16,   //TODO remove it after fully refactor.
-//  OP_MultiwayMergeSort = 17,   // multi-way data merge into one input stream.
-//  OP_GlobalAggregate   = 18,   // global merge for the multi-way data sources.
-  OP_Filter            = 19,
-  OP_Distinct          = 20,
-  OP_Join              = 21,
-  OP_AllTimeWindow     = 23,
-  OP_AllMultiTableTimeInterval = 24,
-  OP_Order             = 25,
-  OP_Exchange          = 26,
 };
 
 struct SEpSet;
-struct SQueryPlanNode;
-struct SPhyNode;
 struct SQueryStmtInfo;
 
+typedef SSchema SSlotSchema;
+
+typedef struct SDataBlockSchema {
+  SSlotSchema        *pSchema;
+  int32_t             numOfCols;    // number of columns
+} SDataBlockSchema;
+
+typedef struct SQueryNodeBasicInfo {
+  int32_t     type;          // operator type
+  const char *name;          // operator name
+} SQueryNodeBasicInfo;
+
+typedef struct SPhyNode {
+  SQueryNodeBasicInfo info;
+  SArray             *pTargets;      // target list to be computed or scanned at this node
+  SArray             *pConditions;   // implicitly-ANDed qual conditions
+  SDataBlockSchema    targetSchema;
+  // children plan to generated result for current node to process
+  // in case of join, multiple plan nodes exist.
+  SArray             *pChildren;
+  struct SPhyNode    *pParent;
+} SPhyNode;
+
+typedef struct SScanPhyNode {
+  SPhyNode    node;
+  uint64_t    uid;  // unique id of the table
+  int8_t      tableType;
+} SScanPhyNode;
+
+typedef SScanPhyNode SSystemTableScanPhyNode;
+typedef SScanPhyNode STagScanPhyNode;
+
+typedef struct STableScanPhyNode {
+  SScanPhyNode scan;
+  uint8_t      scanFlag;         // denotes reversed scan of data or not
+  STimeWindow  window;
+  SArray      *pTagsConditions; // implicitly-ANDed tag qual conditions
+} STableScanPhyNode;
+
+typedef STableScanPhyNode STableSeqScanPhyNode;
+
+typedef struct SProjectPhyNode {
+  SPhyNode node;
+} SProjectPhyNode;
+
+typedef struct SExchangePhyNode {
+  SPhyNode    node;
+  uint64_t    templateId;
+  SArray     *pSourceEpSet;  // SEpSet
+} SExchangePhyNode;
+
+typedef struct SSubplanId {
+  uint64_t queryId;
+  uint64_t templateId;
+  uint64_t subplanId;
+} SSubplanId;
+
 typedef struct SSubplan {
-  int32_t   type;               // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN
+  SSubplanId id;          // unique id of the subplan
-  SArray   *pDatasource;          // the datasource subplan,from which to fetch the result
+  int32_t   type;         // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN
-  struct SPhyNode *pNode;  // physical plan of current subplan
+  int32_t   level;        // the execution level of current subplan, starting from 0.
+  SEpSet    execEpSet;    // for the scan sub plan, the optional execution node
+  SArray   *pChildern;    // the datasource subplan,from which to fetch the result
+  SArray   *pParents;     // the data destination subplan, get data from current subplan
+  SPhyNode *pNode;        // physical plan of current subplan
 } SSubplan;
 
 typedef struct SQueryDag {
-  SArray  **pSubplans;
+  SArray  *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
 } SQueryDag;
 
 /**
@@ -75,6 +115,7 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet*
 
 int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
 
+
 /**
  * Convert to subplan to string for the scheduler to send to the executor
  */

include/libs/planner/plannerOp.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if defined(INCLUDE_AS_ENUM)  // enum define mode
+    #undef OP_ENUM_MACRO
+    #define OP_ENUM_MACRO(op)    OP_##op,
+#elif defined(INCLUDE_AS_NAME)  // comment define mode
+    #undef OP_ENUM_MACRO
+    #define OP_ENUM_MACRO(op)    #op,
+#else
+    #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME
+#endif
+
+OP_ENUM_MACRO(TableScan)
+OP_ENUM_MACRO(DataBlocksOptScan)
+OP_ENUM_MACRO(TableSeqScan)
+OP_ENUM_MACRO(TagScan)
+OP_ENUM_MACRO(TableBlockInfoScan)
+OP_ENUM_MACRO(Aggregate)
+OP_ENUM_MACRO(Project)
+OP_ENUM_MACRO(Groupby)
+OP_ENUM_MACRO(Limit)
+OP_ENUM_MACRO(SLimit)
+OP_ENUM_MACRO(TimeWindow)
+OP_ENUM_MACRO(SessionWindow)
+OP_ENUM_MACRO(StateWindow)
+OP_ENUM_MACRO(Fill)
+OP_ENUM_MACRO(MultiTableAggregate)
+OP_ENUM_MACRO(MultiTableTimeInterval)
+OP_ENUM_MACRO(Filter)
+OP_ENUM_MACRO(Distinct)
+OP_ENUM_MACRO(Join)
+OP_ENUM_MACRO(AllTimeWindow)
+OP_ENUM_MACRO(AllMultiTableTimeInterval)
+OP_ENUM_MACRO(Order)
+OP_ENUM_MACRO(Exchange)

include/libs/query/query.h

@@ -21,21 +21,66 @@ extern "C" {
 #endif
 
 #include "tarray.h"
+#include "thash.h"
 
 typedef SVgroupListRspMsg SVgroupListInfo;
 
+typedef struct STableComInfo {
+  uint8_t numOfTags;      // the number of tags in schema
+  uint8_t precision;      // the number of precision
+  int16_t numOfColumns;   // the number of columns
+  int32_t rowSize;        // row size of the schema
+} STableComInfo;
+
+/*
+ * ASSERT(sizeof(SCTableMeta) == 24)
+ * ASSERT(tableType == TSDB_CHILD_TABLE)
+ * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
+ */
+typedef struct SCTableMeta {
+  int32_t  vgId:24;
+  int8_t   tableType;
+  uint64_t uid;
+  uint64_t suid;
+} SCTableMeta;
+
+/*
+ * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
+ */
+typedef struct STableMeta {
+  //BEGIN: KEEP THIS PART SAME WITH SCTableMeta
+  int32_t        vgId:24;
+  int8_t         tableType;
+  uint64_t       uid;
+  uint64_t       suid;
+  //END: KEEP THIS PART SAME WITH SCTableMeta
+  
+  // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
+  int16_t        sversion;
+  int16_t        tversion;
+  STableComInfo  tableInfo;
+  SSchema        schema[];
+} STableMeta;
+
+
 typedef struct SDBVgroupInfo {
-  int32_t vgroupVersion;
+  int32_t   vgVersion;  
-  SArray *vgId;
+  int8_t    hashMethod;
-  int32_t hashRange;
+  SHashObj *vgInfo;  //key:vgId, value:SVgroupInfo
 } SDBVgroupInfo;
 
 typedef struct SUseDbOutput {
-  SVgroupListInfo *vgroupList;
+  char db[TSDB_FULL_DB_NAME_LEN];
-  char db[TSDB_TABLE_FNAME_LEN];
+  SDBVgroupInfo dbVgroup;
-  SDBVgroupInfo *dbVgroup;
 } SUseDbOutput;
 
+typedef struct STableMetaOutput {
+  int32_t     metaNum;
+  char        ctbFname[TSDB_TABLE_FNAME_LEN];
+  char        tbFname[TSDB_TABLE_FNAME_LEN];  
+  SCTableMeta ctbMeta;
+  STableMeta *tbMeta;
+} STableMetaOutput;
 
 extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
 extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize);

include/util/tarray.h

@@ -41,7 +41,7 @@ typedef struct SArray {
  * @param elemSize
  * @return
  */
-void* taosArrayInit(size_t size, size_t elemSize);
+SArray* taosArrayInit(size_t size, size_t elemSize);
 
 /**
  *

include/util/tlog.h

@@ -32,7 +32,6 @@ extern int32_t mDebugFlag;
 extern int32_t cDebugFlag;
 extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
-extern int32_t sdbDebugFlag;
 extern int32_t httpDebugFlag;
 extern int32_t mqttDebugFlag;
 extern int32_t monDebugFlag;

source/common/inc/commonInt.h

@@ -20,6 +20,7 @@
 extern "C" {
 #endif
 
+extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
 
 

source/common/src/tglobal.c

@@ -30,19 +30,12 @@
 // cluster
 char     tsFirst[TSDB_EP_LEN] = {0};
 char     tsSecond[TSDB_EP_LEN] = {0};
-char     tsArbitrator[TSDB_EP_LEN] = {0};
 char     tsLocalFqdn[TSDB_FQDN_LEN] = {0};
 char     tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 int32_t  tsStatusInterval = 1;  // second
-int32_t  tsNumOfMnodes = 1;
-int8_t   tsEnableVnodeBak = 1;
 int8_t   tsEnableTelemetryReporting = 0;
-int8_t   tsArbOnline = 0;
-int64_t  tsArbOnlineTimestamp = TSDB_ARB_DUMMY_TIME;
 char     tsEmail[TSDB_FQDN_LEN] = {0};
-int32_t  tsDnodeId = 0;
-int64_t  tsDnodeStartTime = 0;
 
 // common
 int32_t tsRpcTimer       = 300;
@@ -129,59 +122,19 @@ int32_t tsRetrieveBlockingModel = 0;
 // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
 int8_t  tsKeepOriginalColumnName = 0;
 
-// db parameters
-int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
-int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS;
-int16_t tsDaysPerFile    = TSDB_DEFAULT_DAYS_PER_FILE;
-int32_t tsDaysToKeep     = TSDB_DEFAULT_KEEP;
-int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
-int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
-int16_t tsCommitTime    = TSDB_DEFAULT_COMMIT_TIME;  // seconds
-int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
-int8_t  tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
-int8_t  tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
-int32_t tsFsyncPeriod   = TSDB_DEFAULT_FSYNC_PERIOD;
-int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
-int32_t tsQuorum        = TSDB_DEFAULT_DB_QUORUM_OPTION;
-int8_t  tsUpdate        = TSDB_DEFAULT_DB_UPDATE_OPTION;
-int8_t  tsCacheLastRow  = TSDB_DEFAULT_CACHE_LAST_ROW;
-int32_t tsMaxVgroupsPerDb  = 0;
-int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO;
-
 // tsdb config 
 // For backward compatibility
 bool tsdbForceKeepFile = false;
 
 // balance
-int8_t  tsEnableBalance = 1;
-int8_t  tsAlternativeRole = 0;
-int32_t tsBalanceInterval = 300;           // seconds
-int32_t tsOfflineThreshold = 86400 * 10;  // seconds of 10 days
 int8_t  tsEnableFlowCtrl = 1;
 int8_t  tsEnableSlaveQuery = 1;
 int8_t  tsEnableAdjustMaster = 1;
 
-// restful
-int32_t  tsRestRowLimit = 10240;
-int8_t   tsTelegrafUseFieldNum = 0;
-
-// mqtt
-int8_t tsEnableMqttModule = 0;  // not finished yet, not started it by default
-char    tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org";
-char    tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883";
-char    tsMqttUser[TSDB_MQTT_USER_LEN] = {0};
-char    tsMqttPass[TSDB_MQTT_PASS_LEN] = {0};
-char    tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber";
-char    tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // #
 
 // monitor
-int8_t  tsEnableMonitorModule = 1;
 char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
-int32_t tsMonitorInterval = 30;  // seconds
-
-// stream
-int8_t  tsEnableStream = 1;
 
 // internal
 int8_t tsCompactMnodeWal = 0;
@@ -191,7 +144,6 @@ char   tsDnodeDir[PATH_MAX] = {0};
 char   tsMnodeDir[PATH_MAX] = {0};
 
 int32_t  tsDiskCfgNum = 0;
-int32_t tsTopicBianryLen = 16000;
 
 #ifndef _STORAGE
 SDiskCfg tsDiskCfg[1];
@@ -245,14 +197,10 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
 void taosSetAllDebugFlag() {
   if (debugFlag != 0) { 
     mDebugFlag = debugFlag;
-    sdbDebugFlag = debugFlag;
     dDebugFlag = debugFlag;
     vDebugFlag = debugFlag;
     jniDebugFlag = debugFlag;
     odbcDebugFlag = debugFlag;
-    httpDebugFlag = debugFlag;
-    mqttDebugFlag = debugFlag;
-    monDebugFlag = debugFlag;
     qDebugFlag = debugFlag;    
     rpcDebugFlag = debugFlag;
     uDebugFlag = debugFlag;
@@ -459,16 +407,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "arbitrator";
-  cfg.ptr = tsArbitrator;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_EP_LEN;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   // dnode configs
   cfg.option = "numOfThreadsPerCore";
   cfg.ptr = &tsNumOfThreadsPerCore;
@@ -509,26 +447,6 @@ static void doInitGlobalConfig(void) {
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
-  
-  cfg.option = "numOfMnodes";
-  cfg.ptr = &tsNumOfMnodes;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 1;
-  cfg.maxValue = 3;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "vnodeBak";
-  cfg.ptr = &tsEnableVnodeBak;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
 
   cfg.option = "telemetryReporting";
   cfg.ptr = &tsEnableTelemetryReporting;
@@ -540,37 +458,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "balance";
-  cfg.ptr = &tsEnableBalance;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "balanceInterval";
-  cfg.ptr = &tsBalanceInterval;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 1;
-  cfg.maxValue = 30000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  // 0-any; 1-mnode; 2-vnode
-  cfg.option = "role";
-  cfg.ptr = &tsAlternativeRole;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 0;
-  cfg.maxValue = 2;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   // timer
   cfg.option = "maxTmrCtrl";
   cfg.ptr = &tsMaxTmrCtrl;
@@ -582,26 +469,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "monitorInterval";
-  cfg.ptr = &tsMonitorInterval;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 1;
-  cfg.maxValue = 600;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_SECOND;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "offlineThreshold";
-  cfg.ptr = &tsOfflineThreshold;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 3;
-  cfg.maxValue = 86400 * 365;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_SECOND;
-  taosAddConfigOption(cfg);
-
   cfg.option = "rpcTimer";
   cfg.ptr = &tsRpcTimer;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -712,186 +579,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "maxVgroupsPerDb";
-  cfg.ptr = &tsMaxVgroupsPerDb;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 8192;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "cache";
-  cfg.ptr = &tsCacheBlockSize;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE;
-  cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_MB;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "blocks";
-  cfg.ptr = &tsBlocksPerVnode;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_TOTAL_BLOCKS;
-  cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "days";
-  cfg.ptr = &tsDaysPerFile;
-  cfg.valType = TAOS_CFG_VTYPE_INT16;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DAYS_PER_FILE;
-  cfg.maxValue = TSDB_MAX_DAYS_PER_FILE;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "keep";
-  cfg.ptr = &tsDaysToKeep;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_KEEP;
-  cfg.maxValue = TSDB_MAX_KEEP;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "minRows";
-  cfg.ptr = &tsMinRowsInFileBlock;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK;
-  cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "maxRows";
-  cfg.ptr = &tsMaxRowsInFileBlock;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK;
-  cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "precision";
-  cfg.ptr = &tsTimePrecision;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_PRECISION;
-  cfg.maxValue = TSDB_MAX_PRECISION;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "comp";
-  cfg.ptr = &tsCompression;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_COMP_LEVEL;
-  cfg.maxValue = TSDB_MAX_COMP_LEVEL;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "walLevel";
-  cfg.ptr = &tsWAL;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_WAL_LEVEL;
-  cfg.maxValue = TSDB_MAX_WAL_LEVEL;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "fsync";
-  cfg.ptr = &tsFsyncPeriod;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_FSYNC_PERIOD;
-  cfg.maxValue = TSDB_MAX_FSYNC_PERIOD;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "replica";
-  cfg.ptr = &tsReplications;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DB_REPLICA_OPTION;
-  cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "quorum";
-  cfg.ptr = &tsQuorum;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DB_QUORUM_OPTION;
-  cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "update";
-  cfg.ptr = &tsUpdate;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DB_UPDATE;
-  cfg.maxValue = TSDB_MAX_DB_UPDATE;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "cachelast";
-  cfg.ptr = &tsCacheLastRow;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = TSDB_MIN_DB_CACHE_LAST_ROW;
-  cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "mqttHostName";
-  cfg.ptr = tsMqttHostName;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "mqttPort";
-  cfg.ptr = tsMqttPort;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_MQTT_PORT_LEN;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "mqttTopic";
-  cfg.ptr = tsMqttTopic;
-  cfg.valType = TAOS_CFG_VTYPE_STRING;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_NOT_PRINT;
-  cfg.minValue = 0;
-  cfg.maxValue = 0;
-  cfg.ptrLength = TSDB_MQTT_TOPIC_LEN;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   cfg.option = "compressMsgSize";
   cfg.ptr = &tsCompressMsgSize;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1085,76 +772,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "adjustMaster";
-  cfg.ptr = &tsEnableAdjustMaster;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "mqtt";
-  cfg.ptr = &tsEnableMqttModule;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "monitor";
-  cfg.ptr = &tsEnableMonitorModule;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "stream";
-  cfg.ptr = &tsEnableStream;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "topicBianryLen";
-  cfg.ptr = &tsTopicBianryLen;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 16;
-  cfg.maxValue = 16000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "telegrafUseFieldNum";
-  cfg.ptr = &tsTelegrafUseFieldNum;
-  cfg.valType = TAOS_CFG_VTYPE_INT8;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1;
-  cfg.ptrLength = 1;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "restfulRowLimit";
-  cfg.ptr = &tsRestRowLimit;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 1;
-  cfg.maxValue = 10000000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   // debug flag
   cfg.option = "numOfLogLines";
   cfg.ptr = &tsNumOfLogLines;
@@ -1236,17 +853,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-
-  cfg.option = "sdbDebugFlag";
-  cfg.ptr = &sdbDebugFlag;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
-  cfg.minValue = 0;
-  cfg.maxValue = 255;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   cfg.option = "rpcDebugFlag";
   cfg.ptr = &rpcDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1307,36 +913,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
-  cfg.option = "httpDebugFlag";
-  cfg.ptr = &httpDebugFlag;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
-  cfg.minValue = 0;
-  cfg.maxValue = 255;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "mqttDebugFlag";
-  cfg.ptr = &mqttDebugFlag;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
-  cfg.minValue = 0;
-  cfg.maxValue = 255;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "monDebugFlag";
-  cfg.ptr = &monDebugFlag;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
-  cfg.minValue = 0;
-  cfg.maxValue = 255;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
   cfg.option = "qDebugFlag";
   cfg.ptr = &qDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1456,16 +1032,6 @@ static void doInitGlobalConfig(void) {
   cfg.maxValue = 0;
   cfg.ptrLength = PATH_MAX;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosAddConfigOption(cfg);
-
-  cfg.option = "tsdbMetaCompactRatio";
-  cfg.ptr = &tsTsdbMetaCompactRatio;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
-  cfg.minValue = 0;
-  cfg.maxValue = 100;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
    // enable kill long query
@@ -1630,4 +1196,3 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d
 
   return true;
 }
-

source/dnode/mgmt/impl/test/sut/deploy.cpp

@@ -22,10 +22,6 @@ void initLog(const char* path) {
   cDebugFlag = 0;
   jniDebugFlag = 0;
   tmrDebugFlag = 0;
-  sdbDebugFlag = 0;
-  httpDebugFlag = 0;
-  mqttDebugFlag = 0;
-  monDebugFlag = 0;
   uDebugFlag = 143;
   rpcDebugFlag = 0;
   odbcDebugFlag = 0;

source/dnode/mnode/impl/inc/mndDef.h

@@ -214,6 +214,7 @@ typedef struct {
   int64_t createdTime;
   int64_t updateTime;
   int64_t uid;
+  int32_t version;
   SDbCfg  cfg;
 } SDbObj;
 

source/dnode/mnode/impl/src/mndDb.c

@@ -74,6 +74,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT64(pRaw, dataPos, pDb->createdTime)
   SDB_SET_INT64(pRaw, dataPos, pDb->updateTime)
   SDB_SET_INT64(pRaw, dataPos, pDb->uid)
+  SDB_SET_INT32(pRaw, dataPos, pDb->version)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile)
@@ -117,6 +118,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime)
   SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime)
   SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile)
@@ -571,6 +573,7 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
     return code;
   }
 
+  dbObj.version++;
   code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj);
   mndReleaseDb(pMnode, pDb);
 

source/dnode/mnode/impl/src/mndStb.c

@@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) {
 void mndCleanupStb(SMnode *pMnode) {}
 
 static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
-  int32_t  size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema);
+  int32_t  size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size);
   if (pRaw == NULL) return NULL;
 

source/dnode/mnode/impl/src/mndVgroup.c

@@ -71,7 +71,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
 void mndCleanupVgroup(SMnode *pMnode) {}
 
 static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj));
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
   if (pRaw == NULL) return NULL;
 
   int32_t dataPos = 0;
@@ -141,11 +141,8 @@ static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
 
 static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) {
   mTrace("vgId:%d, perform update action", pOldVgroup->vgId);
-  pOldVgroup->vgId = pNewVgroup->vgId;
-  pOldVgroup->createdTime = pNewVgroup->createdTime;
   pOldVgroup->updateTime = pNewVgroup->updateTime;
   pOldVgroup->version = pNewVgroup->version;
-  memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN);
   pOldVgroup->replica = pNewVgroup->replica;
   memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
   return 0;

source/dnode/vnode/impl/CMakeLists.txt

@@ -19,5 +19,5 @@ target_link_libraries(
 
 # test
 if(${BUILD_TEST})
-   add_subdirectory(test)
+   #add_subdirectory(test)
 endif(${BUILD_TEST})

source/libs/catalog/CMakeLists.txt

@@ -10,3 +10,5 @@ target_link_libraries(
    catalog
    PRIVATE os util common transport query
 )
+
+ADD_SUBDIRECTORY(test)

source/libs/catalog/inc/catalogInt.h

@@ -24,16 +24,16 @@ extern "C" {
 #include "common.h"
 #include "tlog.h"
 
-#define CTG_DEFAULT_CLUSTER_NUMBER 6
+#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
-#define CTG_DEFAULT_VGROUP_NUMBER 100
+#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
-#define CTG_DEFAULT_DB_NUMBER 20
+#define CTG_DEFAULT_CACHE_DB_NUMBER 20
+#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000
 
 #define CTG_DEFAULT_INVALID_VERSION (-1)
 
 typedef struct SVgroupListCache {
   int32_t vgroupVersion;
-  SHashObj *cache;        // key:vgId, value:SVgroupInfo*
+  SHashObj *cache;        // key:vgId, value:SVgroupInfo
-  SArray   *arrayCache;   // SVgroupInfo
 } SVgroupListCache;
 
 typedef struct SDBVgroupCache {
@@ -41,20 +41,23 @@ typedef struct SDBVgroupCache {
 } SDBVgroupCache;
 
 typedef struct STableMetaCache {
-  SHashObj *cache;     //key:fulltablename, value:STableMeta
+  SHashObj *cache;           //key:fulltablename, value:STableMeta
+  SHashObj *stableCache;     //key:suid, value:STableMeta*
 } STableMetaCache;
 
 typedef struct SCatalog {
   SVgroupListCache vgroupCache;
-  SDBVgroupCache dbCache;
+  SDBVgroupCache   dbCache;
-  STableMetaCache tableCache;
+  STableMetaCache  tableCache;
 } SCatalog;
 
 typedef struct SCatalogMgmt {
   void       *pMsgSender;   // used to send messsage to mnode to fetch necessary metadata
   SHashObj   *pCluster;     // items cached for each cluster, the hash key is the cluster-id got from mgmt node
+  SCatalogCfg cfg;
 } SCatalogMgmt;
 
+typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 
 extern int32_t ctgDebugFlag;
 

source/libs/catalog/src/catalog.c

@@ -16,54 +16,11 @@
 #include "catalogInt.h"
 #include "trpc.h"
 #include "query.h"
+#include "tname.h"
 
 SCatalogMgmt ctgMgmt = {0};
 
-int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) {
+int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) {
-  char *msg = NULL;
-  SEpSet *pVnodeEpSet = NULL;
-  int32_t msgLen = 0;
-
-  int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen);
-  if (code) {
-    return code;
-  }
-
-  SRpcMsg rpcMsg = {
-      .msgType = TSDB_MSG_TYPE_VGROUP_LIST,
-      .pCont   = msg,
-      .contLen = msgLen,
-  };
-
-  SRpcMsg rpcRsp = {0};
-
-  rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
-
-  code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen);
-  if (code) {
-    return code;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) {
-  if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) {
-    *exist = 0;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (pVgroupList) {
-    *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache);
-  }
-
-  *exist = 1;
-  
-  return TSDB_CODE_SUCCESS;
-}
-
-
-int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) {
   if (NULL == pCatalog->dbCache.cache) {
     *exist = 0;
     return TSDB_CODE_SUCCESS;
@@ -71,27 +28,13 @@ int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgrou
 
   SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
 
-  if (NULL == info || info->vgroupVersion < pCatalog->vgroupCache.vgroupVersion) {
+  if (NULL == info) {
     *exist = 0;
     return TSDB_CODE_SUCCESS;
   }
 
   if (dbInfo) {
-    *dbInfo = calloc(1, sizeof(**dbInfo));
+    *dbInfo = *info;
-    if (NULL == *dbInfo) {
-      ctgError("calloc size[%d] failed", (int32_t)sizeof(**dbInfo));
-      return TSDB_CODE_CTG_MEM_ERROR;
-    }
-    
-    (*dbInfo)->vgId = taosArrayDup(info->vgId);
-    if (NULL == (*dbInfo)->vgId) {
-      ctgError("taos array duplicate failed");
-      tfree(*dbInfo);
-      return TSDB_CODE_CTG_MEM_ERROR;
-    }
-
-    (*dbInfo)->vgroupVersion = info->vgroupVersion;
-    (*dbInfo)->hashRange = info->hashRange;
   }
 
   *exist = 1;
@@ -130,16 +73,345 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
 }
 
 
-int32_t catalogInit(SCatalogCfg *cfg) {
+int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, const char* pTableName, STableMeta** pTableMeta, int32_t *exist) {
-  ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  if (NULL == pCatalog->tableCache.cache) {
-  if (NULL == ctgMgmt.pCluster) {
+    *exist = 0;
-    CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  char tbFullName[TSDB_TABLE_FNAME_LEN];
+
+  snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName);
+
+  STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName));
+
+  if (NULL == tbMeta) {
+    *exist = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (tbMeta->tableType == TSDB_CHILD_TABLE) {
+    STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid));
+    if (NULL == stbMeta || NULL == *stbMeta) {
+      *exist = 0;
+      return TSDB_CODE_SUCCESS;
+    }
+
+    if ((*stbMeta)->suid != tbMeta->suid) {
+      ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid);
+      return TSDB_CODE_CTG_INTERNAL_ERROR;
+    }
+
+    int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema);
+    *pTableMeta = calloc(1, metaSize);
+    if (NULL == *pTableMeta) {
+      ctgError("calloc size[%d] failed", metaSize);
+      return TSDB_CODE_CTG_MEM_ERROR;
+    }
+
+    memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta));
+    memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta));
+  } else {
+    int32_t metaSize = sizeof(STableMeta) + (tbMeta->tableInfo.numOfTags + tbMeta->tableInfo.numOfColumns) * sizeof(SSchema);
+    *pTableMeta = calloc(1, metaSize);
+    if (NULL == *pTableMeta) {
+      ctgError("calloc size[%d] failed", metaSize);
+      return TSDB_CODE_CTG_MEM_ERROR;
+    }
+
+    memcpy(*pTableMeta, tbMeta, metaSize);
+  }
+
+  *exist = 1;
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
+  epSet->inUse = 0;
+  epSet->numOfEps = vgroupInfo->numOfEps;
+
+  for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) {
+    memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i]));
+    memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i]));
+  }
+}
+
+
+int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
+  if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
+    return TSDB_CODE_CTG_INVALID_INPUT;
+  }
+
+  char tbFullName[TSDB_TABLE_FNAME_LEN];
+
+  snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
+
+  SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName};
+  char *msg = NULL;
+  SEpSet *pVnodeEpSet = NULL;
+  int32_t msgLen = 0;
+
+  int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen);
+  if (code) {
+    return code;
+  }
+
+  SRpcMsg rpcMsg = {
+      .msgType = TSDB_MSG_TYPE_TABLE_META,
+      .pCont   = msg,
+      .contLen = msgLen,
+  };
+
+  SRpcMsg rpcRsp = {0};
+  SEpSet  epSet;
+  
+  ctgGenEpSet(&epSet, vgroupInfo);
+
+  rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp);
+
+  if (TSDB_CODE_SUCCESS != rpcRsp.code) {
+    ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code);
+    return rpcRsp.code;
+  }
+
+  code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen);
+  if (code) {
+    return code;
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
 
+int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
+  switch (hashMethod) {
+    default:
+      *fp = MurmurHash3_32;
+      break;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) {
+  SHashObj *vgroupHash = NULL;
+  SVgroupInfo *vgInfo = NULL;
+
+  void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
+  while (pIter) {
+    vgInfo = pIter;
+
+    if (NULL == taosArrayPush(vgroupList, vgInfo)) {
+      ctgError("taosArrayPush failed");
+      break;
+    }
+    
+    pIter = taosHashIterate(dbInfo->vgInfo, pIter);
+    vgInfo = NULL;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
+  int32_t vgNum = taosHashGetSize(dbInfo->vgInfo);
+  if (vgNum <= 0) {
+    ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum);
+    return TSDB_CODE_TSC_DB_NOT_SELECTED;
+  }
+
+  tableNameHashFp fp = NULL;
+  SVgroupInfo *vgInfo = NULL;
+
+  CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
+
+  char tbFullName[TSDB_TABLE_FNAME_LEN];
+
+  snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
+
+  uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
+
+  void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
+  while (pIter) {
+    vgInfo = pIter;
+    if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
+      break;
+    }
+    
+    pIter = taosHashIterate(dbInfo->vgInfo, pIter);
+    vgInfo = NULL;
+  }
+
+  if (NULL == vgInfo) {
+    ctgError("no hash range found for hashvalue[%u]", hashValue);
+    return TSDB_CODE_CTG_INTERNAL_ERROR;
+  }
+
+  *pVgroup = *vgInfo;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
+  SDBVgroupInfo dbInfo = {0};
+  int32_t code = 0;
+  int32_t vgId = 0;
+
+  CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo));
+
+  if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) {
+    ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo);
+    return TSDB_CODE_TSC_DB_NOT_SELECTED;
+  }
+
+  CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup));
+
+  return code;
+}
+
+
+
+STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) {
+  assert(pChild != NULL);
+  int32_t total = pChild->numOfColumns + pChild->numOfTags;
+
+  STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
+  pTableMeta->tableType = TSDB_SUPER_TABLE;
+  pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
+  pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
+  pTableMeta->tableInfo.precision = pChild->precision;
+
+  pTableMeta->uid = pChild->suid;
+  pTableMeta->tversion = pChild->tversion;
+  pTableMeta->sversion = pChild->sversion;
+
+  memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total);
+
+  int32_t num = pTableMeta->tableInfo.numOfColumns;
+  for(int32_t i = 0; i < num; ++i) {
+    pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
+  }
+
+  return pTableMeta;
+}
+
+int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) {
+  if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
+    return TSDB_CODE_CTG_INVALID_INPUT;
+  }
+
+  int32_t exist = 0;
+
+  if (!forceUpdate) {  
+    CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist));
+
+    if (exist) {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName));
+
+  CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pDBName, pTableName, pTableMeta, &exist));
+
+  if (0 == exist) {
+    ctgError("get table meta from cache failed, but fetch succeed");
+    return TSDB_CODE_CTG_INTERNAL_ERROR;
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) {
+  if (output->metaNum != 1 && output->metaNum != 2) {
+    ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum);
+    return TSDB_CODE_CTG_INTERNAL_ERROR;
+  }
+
+  if (NULL == output->tbMeta) {
+    ctgError("no valid table meta got from meta rsp");
+    return TSDB_CODE_CTG_INTERNAL_ERROR;
+  }
+
+  if (NULL == pCatalog->tableCache.cache) {
+    pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    if (NULL == pCatalog->tableCache.cache) {
+      ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
+      return TSDB_CODE_CTG_MEM_ERROR;
+    }
+  }
+
+  if (NULL == pCatalog->tableCache.cache) {
+    pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    if (NULL == pCatalog->tableCache.cache) {
+      ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
+      return TSDB_CODE_CTG_MEM_ERROR;
+    }
+
+    pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
+    if (NULL == pCatalog->tableCache.stableCache) {
+      ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
+      return TSDB_CODE_CTG_MEM_ERROR;
+    }
+  }
+
+  if (output->metaNum == 2) {
+    if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) {
+      ctgError("push ctable[%s] to table cache failed", output->ctbFname);
+      goto error_exit;
+    }
+
+    if (TSDB_SUPER_TABLE != output->tbMeta->tableType) {
+      ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE);
+      goto error_exit;
+    }    
+  }
+
+  if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, sizeof(*output->tbMeta)) != 0) {
+    ctgError("push table[%s] to table cache failed", output->tbFname);
+    goto error_exit;
+  }
+
+  if (TSDB_SUPER_TABLE == output->tbMeta->tableType) {
+    if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) {
+      ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid);
+      goto error_exit;
+    }
+  }
+  
+  return TSDB_CODE_SUCCESS;
+
+error_exit:
+  if (pCatalog->vgroupCache.cache) {
+    taosHashCleanup(pCatalog->vgroupCache.cache);
+    pCatalog->vgroupCache.cache = NULL;
+  }
+
+  pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
+
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t catalogInit(SCatalogCfg *cfg) {
+  ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  if (NULL == ctgMgmt.pCluster) {
+    CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
+  }
+
+  if (cfg) {
+    memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg));
+  } else {
+    ctgMgmt.cfg.enableVgroupCache = true;
+    ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
+    ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) {
   if (NULL == clusterId || NULL == catalogHandle) {
     return TSDB_CODE_CTG_INVALID_INPUT;
@@ -177,117 +449,6 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle)
   return TSDB_CODE_SUCCESS;
 }
 
-
-int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) {
-  if (NULL == pCatalog || NULL == version) {
-    return TSDB_CODE_CTG_INVALID_INPUT;
-  }
-  
-  *version = pCatalog->vgroupCache.vgroupVersion;
-
-  return TSDB_CODE_SUCCESS;
-}
-
-
-
-int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) {
-  if (NULL == pVgroup) {
-    ctgError("no valid vgroup list info to update");
-    return TSDB_CODE_CTG_INTERNAL_ERROR;
-  }
-
-  if (pVgroup->vgroupVersion < 0) {
-    ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion);
-    return TSDB_CODE_CTG_INVALID_INPUT;
-  }
-  
-
-  if (NULL == pCatalog->vgroupCache.arrayCache) {
-    pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0]));
-    if (NULL == pCatalog->vgroupCache.arrayCache) {
-      ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum);
-      return TSDB_CODE_CTG_MEM_ERROR;
-    }
-  } else {
-    taosArrayClear(pCatalog->vgroupCache.arrayCache);
-  }
-
-  if (NULL == pCatalog->vgroupCache.cache) {
-    pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-    if (NULL == pCatalog->vgroupCache.cache) {
-      ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER);
-      return TSDB_CODE_CTG_MEM_ERROR;
-    }
-  } else {
-    taosHashClear(pCatalog->vgroupCache.cache);
-  }
-
-  SVgroupInfo *vInfo = NULL;
-  for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) {
-    vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]);
-    if (NULL == vInfo) {
-      ctgError("push to vgroup array cache failed");
-      goto error_exit;
-    }
-    
-    if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) {
-      ctgError("push to vgroup hash cache failed");
-      goto error_exit;
-    }
-  }
-
-  pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion;
-
-  return TSDB_CODE_SUCCESS;
-
-error_exit:
-  if (pCatalog->vgroupCache.arrayCache) {
-    taosArrayDestroy(pCatalog->vgroupCache.arrayCache);
-    pCatalog->vgroupCache.arrayCache = NULL;
-  }
-
-  if (pCatalog->vgroupCache.cache) {
-    taosHashCleanup(pCatalog->vgroupCache.cache);
-    pCatalog->vgroupCache.cache = NULL;
-  }
-
-  pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
-
-  return TSDB_CODE_CTG_INTERNAL_ERROR;
-}
-
-
-int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) {
-  if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) {
-    return TSDB_CODE_CTG_INVALID_INPUT;
-  }
-
-  int32_t exist = 0;
-  
-  CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist));
-
-  if (exist) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SVgroupListInfo *pVgroup = NULL;
-  
-  CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup));
-
-  CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup));
-
-  if (pVgroupList) {
-    CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist));
-  }
-
-  if (0 == exist) {
-    ctgError("catalog fetched but get from cache failed");
-    return TSDB_CODE_CTG_INTERNAL_ERROR;
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
 int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) {
   if (NULL == pCatalog || NULL == dbName || NULL == version) {
     return TSDB_CODE_CTG_INVALID_INPUT;
@@ -304,17 +465,17 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName,
     return TSDB_CODE_SUCCESS;
   }
 
-  *version = dbInfo->vgroupVersion;
+  *version = dbInfo->vgVersion;
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
+int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
   if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) {
     return TSDB_CODE_CTG_INVALID_INPUT;
   }
 
-  if (dbInfo->vgroupVersion < 0) {
+  if (dbInfo->vgVersion < 0) {
     if (pCatalog->dbCache.cache) {
       taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName));
     }
@@ -324,11 +485,17 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB
   }
 
   if (NULL == pCatalog->dbCache.cache) {
-    pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
     if (NULL == pCatalog->dbCache.cache) {
-      ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_DB_NUMBER);
+      ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
       return TSDB_CODE_CTG_MEM_ERROR;
     }
+  } else {
+    SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
+    if (oldInfo && oldInfo->vgInfo) {
+      taosHashCleanup(oldInfo->vgInfo);
+      oldInfo->vgInfo = NULL;
+    }
   }
 
   if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) {
@@ -342,7 +509,7 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB
 
 
 
-int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) {
+int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) {
   if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) {
     return TSDB_CODE_CTG_INVALID_INPUT;
   }
@@ -363,72 +530,132 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
 
   strncpy(input.db, dbName, sizeof(input.db));
   input.db[sizeof(input.db) - 1] = 0;
-  input.vgroupVersion = pCatalog->vgroupCache.vgroupVersion;
+  input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
-  input.dbGroupVersion = CTG_DEFAULT_INVALID_VERSION;
   
   CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
 
-  if (DbOut.vgroupList) {
+  CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup));
-    CTG_ERR_JRET(catalogUpdateVgroup(pCatalog, DbOut.vgroupList));
-  }
-
-  if (DbOut.dbVgroup) {
-    CTG_ERR_JRET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup));
-  }
 
   if (dbInfo) {
     *dbInfo = DbOut.dbVgroup;
-    DbOut.dbVgroup = NULL;
   }
 
-_return:
-  tfree(DbOut.dbVgroup);
-  tfree(DbOut.vgroupList);
-
   return code;
 }
 
+int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
+  return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta);
+}
 
-
+int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) {
-int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) {
+  if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) {
-  if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
     return TSDB_CODE_CTG_INVALID_INPUT;
   }
 
-  SBuildTableMetaInput bInput = {0};
+  SVgroupInfo vgroupInfo = {0};
-  char *msg = NULL;
+  
-  SEpSet *pVnodeEpSet = NULL;
+  CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo));
-  int32_t msgLen = 0;
 
-  int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen);
+  STableMetaOutput output = {0};
-  if (code) {
+  
-    return code;
+  CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
-  }
 
-  SRpcMsg rpcMsg = {
+  CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output));
-      .msgType = TSDB_MSG_TYPE_TABLE_META,
-      .pCont   = msg,
-      .contLen = msgLen,
-  };
-
-  SRpcMsg rpcRsp = {0};
-
-  rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
 
+  tfree(output.tbMeta);
+  
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) {
+int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
-
+  return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta);
 }
 
-
+int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) {
-int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) {
+  if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) {
-  if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) {
     return TSDB_CODE_CTG_INVALID_INPUT;
   }
   
-  return 0;
+  STableMeta *tbMeta = NULL;
+  int32_t code = 0;
+  SVgroupInfo vgroupInfo = {0};
+  SDBVgroupInfo dbVgroup = {0};
+  
+  CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta));
+
+  CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup));
+
+  if (tbMeta->tableType == TSDB_SUPER_TABLE) {
+    CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList));
+  } else {
+    int32_t vgId = tbMeta->vgId;
+    if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) {
+      ctgError("vgId[%d] not found in vgroup list", vgId);
+      return TSDB_CODE_CTG_INTERNAL_ERROR;    
+    }
+
+    if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) {
+      ctgError("push vgroupInfo to array failed");
+      CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
+    }
+  }
+
+_return:
+  tfree(tbMeta);
+  
+  return code;
+}
+
+
+int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) {
+  if (NULL == pCatalog || NULL == pRpc  || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) {
+    return TSDB_CODE_CTG_INVALID_INPUT;
+  }
+
+  int32_t code = 0;
+
+  if (pReq->pTableName) {
+    char dbName[TSDB_FULL_DB_NAME_LEN];
+    int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName);
+    if (tbNum > 0) {
+      pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);
+      if (NULL == pRsp->pTableMeta) {
+        ctgError("taosArrayInit num[%d] failed", tbNum);
+        return TSDB_CODE_CTG_MEM_ERROR;
+      }
+    }
+    
+    for (int32_t i = 0; i < tbNum; ++i) {
+      SName *name = taosArrayGet(pReq->pTableName, i);
+      STableMeta *pTableMeta = NULL;
+      
+      snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname);
+
+      CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta));
+
+      if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) {
+        ctgError("taosArrayPush failed, idx:%d", i);
+        tfree(pTableMeta);
+        CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:  
+
+  if (pRsp->pTableMeta) {
+    int32_t aSize = taosArrayGetSize(pRsp->pTableMeta);
+    for (int32_t i = 0; i < aSize; ++i) {
+      STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i);
+      tfree(pMeta);
+    }
+    
+    taosArrayDestroy(pRsp->pTableMeta);
+  }
+  
+  return code;
 }
 
 void catalogDestroy(void) {

source/libs/catalog/test/CMakeLists.txt

@@ -0,0 +1,18 @@
+
+MESSAGE(STATUS "build catalog unit test")
+
+# GoogleTest requires at least C++11
+SET(CMAKE_CXX_STANDARD 11)
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
+TARGET_LINK_LIBRARIES(
+        catalogTest
+        PUBLIC os util common catalog transport gtest query
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        catalogTest
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/"
+        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc"
+)

source/libs/catalog/test/catalogTests.cpp

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+#include <tglobal.h>
+#include <iostream>
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#include "os.h"
+
+#include "taos.h"
+#include "tdef.h"
+#include "tvariant.h"
+#include "catalog.h"
+
+namespace {
+
+
+}
+
+TEST(testCase, normalCase) {
+  char *clusterId = "cluster1";
+  struct SCatalog* pCtg = NULL;
+  
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(clusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  
+}
+
+/*
+TEST(testCase, normalCase) {
+  SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)");
+  ASSERT_EQ(info1.valid, true);
+
+  char    msg[128] = {0};
+  SMsgBuf buf;
+  buf.len = 128;
+  buf.buf = msg;
+
+  SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0);
+  int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
+  ASSERT_EQ(code, 0);
+
+  SCatalogReq req = {0};
+  int32_t  ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
+
+  SQueryStmtInfo* pQueryInfo = createQueryInfo();
+  setTableMetaInfo(pQueryInfo, &req);
+
+  SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0);
+  ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
+  ASSERT_EQ(ret, 0);
+
+  SArray* pExprList = pQueryInfo->exprList[0];
+
+  int32_t num = tsCompatibleModel? 2:1;
+  ASSERT_EQ(taosArrayGetSize(pExprList), num);
+
+  SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1);
+  ASSERT_EQ(p1->base.pColumns->uid, 110);
+  ASSERT_EQ(p1->base.numOfParams, 1);
+  ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE);
+  ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)");
+  ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP);
+  ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)");
+  ASSERT_EQ(p1->base.interBytes, 16);
+
+  ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE);
+  ASSERT_STREQ(p1->pExpr->_function.functionName, "top");
+
+  tExprNode* pParam = p1->pExpr->_function.pChild[0];
+
+  ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE);
+  ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3);
+  ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2);
+
+  struct SQueryPlanNode* n = nullptr;
+  code = createQueryPlan(pQueryInfo, &n);
+
+  char* str = NULL;
+  queryPlanToString(n, &str);
+  printf("%s\n", str);
+
+  destroyQueryInfo(pQueryInfo);
+  qParserClearupMetaRequestInfo(&req);
+  destroySqlInfo(&info1);
+}
+
+TEST(testCase, displayPlan) {
+  generateLogicplan("select count(*) from `t.1abc`");
+  generateLogicplan("select count(*)+ 22 from `t.1abc`");
+  generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30");
+  generateLogicplan("select count(*) from `t.1abc` group by a");
+  generateLogicplan("select count(A+B) from `t.1abc` group by a");
+  generateLogicplan("select count(length(a)+b) from `t.1abc` group by a");
+  generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)");
+  generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s) order by 1 desc ");
+  generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`");
+  generateLogicplan("select count(*), min(a) + 99 from `t.1abc`");
+  generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`");
+  generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20");
+  generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)");
+  generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)");
+
+  // order by + group by column + limit offset
+  generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1");
+
+  // fill
+  generateLogicplan("select min(a) from `t.1abc` where ts>now and ts<now+2h interval(1s) fill(linear)");
+
+  // union + union all
+
+
+
+  // join
+
+  // Aggregate(count(*) [count(*) #5056], sum(a) [sum(a) #5057], avg(b) [avg(b) #5058], min(a+b) [min(a+b) #5060])
+  // Projection(cols: [a+b #5059]) filters:(nil)
+  //  Projection(cols: [ts #0], [a #1], [b #2]) filters:(nil)
+  //   TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807
+
+}
+*/
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+
+

source/libs/executor/src/executorimpl.c

@@ -4831,8 +4831,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
   STableBlockDist tableBlockDist = {0};
   tableBlockDist.numOfTables     = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
 
-  int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS;
+  int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS;
-  if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
+  if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
     ++numRowSteps;
   }
   tableBlockDist.dataBlockInfos  = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));

source/libs/index/inc/index_fst.h

@@ -30,6 +30,7 @@ extern "C" {
 
 typedef struct Fst     Fst;
 typedef struct FstNode FstNode;
+typedef struct StreamWithState StreamWithState;
 
 typedef enum { Included, Excluded, Unbounded} FstBound; 
 
@@ -283,6 +284,9 @@ Output       fstEmptyFinalOutput(Fst *fst, bool *null);
 FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
 FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
 
+// into stream to expand later 
+StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb);
+
 bool         fstVerify(Fst *fst);
 
 

source/libs/index/src/index_fst.c

@@ -1094,6 +1094,10 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
 FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) {
   return fstStreamBuilderCreate(fst, ctx);
 }
+StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) {
+  if (sb == NULL) { return NULL; } 
+  return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
+}
 FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) {
   return fstStreamBuilderCreate(fst, ctx);
 }
@@ -1119,7 +1123,7 @@ CompiledAddr fstGetRootAddr(Fst *fst) {
 
 Output fstEmptyFinalOutput(Fst *fst, bool *null) {
   Output res = 0;
-  FstNode *node = fst->root;
+  FstNode *node = fstGetRoot(fst);
   if (FST_NODE_IS_FINAL(node)) {
     *null = false;
     res = FST_NODE_FINAL_OUTPUT(node); 
@@ -1176,7 +1180,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
 
 
 bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) {
-  return bound->type == Included ? true : false;
+  return bound->type == Excluded? false : true;
 }
 
 void fstBoundDestroy(FstBoundWithData *bound) {

source/libs/index/src/index_fst_automation.c

@@ -68,7 +68,7 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
 // prefix query, impl later
 
 static void* prefixStart(AutomationCtx *ctx) {  
-  StartWithStateValue *data = (StartWithStateValue *)(ctx->data);
+  StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata);
   return startWithStateValueDump(data);     
 };
 static bool prefixIsMatch(AutomationCtx *ctx, void *sv) {
@@ -145,7 +145,8 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) {
 
   StartWithStateValue *sv = NULL;
   if (atype == AUTOMATION_PREFIX) {
-    sv = startWithStateValueCreate(Running, FST_INT, 0);
+    int val = 0;
+    sv = startWithStateValueCreate(Running, FST_INT, &val);
     ctx->stdata = (void *)sv;
   } else if (atype == AUTMMATION_MATCH) {
       

source/libs/index/test/indexTests.cpp

@@ -59,9 +59,22 @@ class FstReadMemory {
      return ok; 
    }
    // add later
-   bool Search(const std::string &key, std::vector<uint64_t> &result) {
+   bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) { 
+      FstStreamBuilder *sb = fstSearch(_fst, ctx);
+      StreamWithState *st = streamBuilderIntoStream(sb);  
+      StreamWithStateResult *rt = NULL;  
+      
+      while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+        result.push_back((uint64_t)(rt->out.out));
+      }
       return true;
    }
+   bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
+     int64_t s = taosGetTimestampUs();
+     bool ok = this->Search(ctx, result); 
+     int64_t e = taosGetTimestampUs();
+     return ok;
+   }
     
    ~FstReadMemory() {
     fstCountingWriterDestroy(_w);
@@ -186,11 +199,43 @@ void checkFstPerf() {
     printf("success to init fst read");  
   }  
   Performance_fstReadRecords(m); 
-   
   delete m;
 } 
 
+void checkFstPrefixSearch() {
+  FstWriter *fw = new FstWriter;
+  int64_t s = taosGetTimestampUs();
+  int count = 2;
+  std::string key("ab");
+  
+  for (int i = 0; i < count; i++) {
+    key[1] = key[1] + i;   
+    fw->Put(key, i); 
+  }
+  int64_t e = taosGetTimestampUs();
+  
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+  delete fw;
 
+  FstReadMemory *m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl; 
+    delete m;
+    return;
+  }
+  
+  // prefix search 
+  std::vector<uint64_t> result;
+  AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX); 
+  m->Search(ctx, result);   
+  assert(result.size() == count); 
+  for (int i = 0; i < result.size(); i++) {
+    assert(result[i] == i); // check result
+  }
+
+  free(ctx);
+  delete m;
+}  
 void validateFst() {
   int val = 100;
   int count = 100;
@@ -209,6 +254,8 @@ void validateFst() {
   FstReadMemory *m = new FstReadMemory(1024 * 64);
   if (m->init() == false) { 
     std::cout << "init readMemory failed" << std::endl; 
+    delete m;
+    return;
   }
 
   {
@@ -230,10 +277,12 @@ void validateFst() {
    }
   } 
   delete m;
-
 } 
+
+
 int main(int argc, char** argv) {
   checkFstPerf(); 
+  //checkFstPrefixSearch();
   return 1;
 }
 

source/libs/parser/src/astValidate.c

@@ -4088,7 +4088,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
   }
 
   // load the meta data from catalog
-  code = catalogGetAllMeta(pCatalog, NULL, &req, &data);
+  code = catalogGetAllMeta(pCatalog, NULL, NULL, &req, &data);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }

source/libs/parser/src/insertParser.c

@@ -71,8 +71,7 @@ typedef struct SInsertParseContext {
   const char* pSql;
   SMsgBuf msg;
   struct SCatalog* pCatalog;
-  SMetaData meta;               // need release
+  STableMeta* pTableMeta;
-  const STableMeta* pTableMeta;
   SHashObj* pTableBlockHashObj; // data block for each table. need release
   int32_t totalNum;
   SInsertStmtInfo* pOutput;
@@ -165,29 +164,29 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) {
+static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) {
   if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) {
     return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z);
   }
 
-  SName name = {0};
+  char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false);
-  strcpy(name.dbname, pCxt->pComCxt->pDbname);
+  if (NULL != p) { // db.table
-  strncpy(name.tname, pStname->z, pStname->n);
+    strcpy(fullDbName, pCxt->pComCxt->pAcctId);
-  taosArrayPush(tableNameList, &name);
+    fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0];
-
+    strncpy(fullDbName, pStname->z, p - pStname->z);
+    strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1);
+  } else {
+    snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname);
+    strncpy(tableName, pStname->z, pStname->n);
+  }
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) {
-  pMetaReq->pTableName = taosArrayInit(4, sizeof(SName));
-  return buildTableName(pCxt, pStname, pMetaReq->pTableName);
-}
-
 static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
-  SCatalogReq req;
+  char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0};
-  CHECK_CODE(buildMetaReq(pCxt, pTname, &req));
+  char tableName[TSDB_TABLE_NAME_LEN] = {0};
-  CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO
+  CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName));
-  pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0);
+  CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta));
   return TSDB_CODE_SUCCESS;
 }
 
@@ -868,12 +867,11 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
     .pOutput = *pInfo
   };
 
-  CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO
-
   if (NULL == context.pTableBlockHashObj) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
+  CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog));
   CHECK_CODE(skipInsertInto(&context));
   CHECK_CODE(parseInsertBody(&context));
 

source/libs/parser/src/parserUtil.c

@@ -1464,29 +1464,6 @@ int32_t copyTagData(STagData* dst, const STagData* src) {
   return 0;
 }
 
-STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
-  assert(pChild != NULL);
-  int32_t total = pChild->numOfColumns + pChild->numOfTags;
-
-  STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
-  pTableMeta->tableType = TSDB_SUPER_TABLE;
-  pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
-  pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
-  pTableMeta->tableInfo.precision = pChild->precision;
-
-  pTableMeta->uid = pChild->suid;
-  pTableMeta->tversion = pChild->tversion;
-  pTableMeta->sversion = pChild->sversion;
-
-  memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total);
-
-  int32_t num = pTableMeta->tableInfo.numOfColumns;
-  for(int32_t i = 0; i < num; ++i) {
-    pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
-  }
-
-  return pTableMeta;
-}
 
 uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);

source/libs/parser/test/CMakeLists.txt

@@ -6,13 +6,16 @@ SET(CMAKE_CXX_STANDARD 11)
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
 ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
-TARGET_LINK_LIBRARIES(
-        parserTest
-        PUBLIC os util common parser catalog transport gtest function planner query
-)
 
 TARGET_INCLUDE_DIRECTORIES(
         parserTest
         PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/"
         PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc"
 )
+
+TARGET_LINK_LIBRARIES(
+        parserTest
+        PUBLIC os util common parser catalog transport gtest function planner query
+)
+
+TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc)

source/libs/parser/test/insertTest.cpp

@@ -27,6 +27,27 @@ namespace {
   }
 }
 
+extern "C" {
+
+#include <execinfo.h>
+
+void *__real_malloc(size_t);
+
+void *__wrap_malloc(size_t c) {  
+  // printf("My MALLOC called: %d\n", c);
+  // void *array[32]; 
+  // int size = backtrace(array, 32); 
+  // char **symbols = backtrace_symbols(array, size); 
+  // for (int i = 0; i < size; ++i) { 
+  //   cout << symbols[i] << endl;
+  // } 
+  // free(symbols); 
+
+  return __real_malloc(c);
+}
+
+}
+
 // syntax:
 // INSERT INTO
 //   tb_name

source/libs/planner/inc/plannerInt.h

@@ -23,25 +23,23 @@ extern "C" {
 #include "common.h"
 #include "tarray.h"
 #include "planner.h"
+#include "parser.h"
 #include "taosmsg.h"
 
-enum LOGIC_PLAN_E {
+#define QNODE_TAGSCAN       1
-  LP_SCAN     = 1,
+#define QNODE_TABLESCAN     2
-  LP_SESSION  = 2,
+#define QNODE_PROJECT       3
-  LP_STATE    = 3,
+#define QNODE_AGGREGATE     4
-  LP_INTERVAL = 4,
+#define QNODE_GROUPBY       5
-  LP_FILL     = 5,
+#define QNODE_LIMIT         6
-  LP_AGG      = 6,
+#define QNODE_JOIN          7
-  LP_JOIN     = 7,
+#define QNODE_DISTINCT      8
-  LP_PROJECT  = 8,
+#define QNODE_SORT          9
-  LP_DISTINCT = 9,
+#define QNODE_UNION         10
-  LP_ORDER    = 10
+#define QNODE_TIMEWINDOW    11
-};
+#define QNODE_SESSIONWINDOW 12
-
+#define QNODE_STATEWINDOW   13
-typedef struct SQueryNodeBasicInfo {
+#define QNODE_FILL          14
-  int32_t   type;          // operator type
-  char     *name;          // operator name
-} SQueryNodeBasicInfo;
 
 typedef struct SQueryDistPlanNodeInfo {
   bool      stableQuery;   // super table query or not
@@ -52,8 +50,9 @@ typedef struct SQueryDistPlanNodeInfo {
 } SQueryDistPlanNodeInfo;
 
 typedef struct SQueryTableInfo {
-  char       *tableName;
+  char           *tableName; // to be deleted
-  uint64_t    uid;
+  uint64_t        uid;       // to be deleted
+  STableMetaInfo* pMeta;
   STimeWindow window;
 } SQueryTableInfo;
 
@@ -64,50 +63,12 @@ typedef struct SQueryPlanNode {
   SArray             *pExpr;        // the query functions or sql aggregations
   int32_t             numOfExpr;  // number of result columns, which is also the number of pExprs
   void               *pExtInfo;     // additional information
-  // previous operator to generated result for current node to process
+  // children operator to generated result for current node to process
   // in case of join, multiple prev nodes exist.
-  SArray             *pPrevNodes;   // upstream nodes
+  SArray             *pChildren;   // upstream nodes
-  struct SQueryPlanNode  *nextNode;
+  struct SQueryPlanNode  *pParent;
 } SQueryPlanNode;
 
-typedef SSchema SSlotSchema;
-
-typedef struct SDataBlockSchema {
-  int32_t             index;
-  SSlotSchema        *pSchema;
-  int32_t             numOfCols;    // number of columns
-} SDataBlockSchema;
-
-typedef struct SPhyNode {
-  SQueryNodeBasicInfo info;
-  SArray             *pTargets;      // target list to be computed or scanned at this node
-  SArray             *pConditions;         // implicitly-ANDed qual conditions
-  SDataBlockSchema    targetSchema;
-  // children plan to generated result for current node to process
-  // in case of join, multiple plan nodes exist.
-  SArray             *pChildren;
-} SPhyNode;
-
-typedef struct SScanPhyNode {
-  SPhyNode node;
-  uint64_t     uid;  // unique id of the table
-} SScanPhyNode;
-
-typedef SScanPhyNode STagScanPhyNode;
-
-typedef SScanPhyNode SSystemTableScanPhyNode;
-
-typedef struct SMultiTableScanPhyNode {
-  SScanPhyNode scan;
-  SArray      *pTagsConditions; // implicitly-ANDed tag qual conditions
-} SMultiTableScanPhyNode;
-
-typedef SMultiTableScanPhyNode SMultiTableSeqScanPhyNode;
-
-typedef struct SProjectPhyNode {
-  SPhyNode node;
-} SProjectPhyNode;
-
 /**
  * Optimize the query execution plan, currently not implement yet.
  * @param pQueryNode

source/libs/planner/src/physicalPlan.c

@@ -14,23 +14,147 @@
  */
 
 #include "plannerInt.h"
+#include "parser.h"
 
-SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) {
+static const char* gOpName[] = {
-  return NULL;
+  "Unknown",
+#define INCLUDE_AS_NAME
+#include "plannerOp.h"
+#undef INCLUDE_AS_NAME
+};
+
+typedef struct SPlanContext {
+  struct SCatalog* pCatalog;
+  struct SQueryDag* pDag;
+  SSubplan* pCurrentSubplan;
+  SSubplanId nextId;
+} SPlanContext;
+
+static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) {
+  SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*);
+  dataBlockSchema->numOfCols = pPlanNode->numOfCols;
 }
 
-SPhyNode* createPhyNode(SQueryPlanNode* node) {
+static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) {
-  switch (node->info.type) {
+  SPhyNode* node = (SPhyNode*)calloc(1, size);
-    case LP_SCAN:
+  node->info.type = type;
-      return createScanNode(node);
+  node->info.name = gOpName[type];
+  SWAP(node->pTargets, pPlanNode->pExpr, SArray*);
+  toDataBlockSchema(pPlanNode, &(node->targetSchema));
+}
+
+static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) {
+  return initPhyNode(pPlanNode, OP_TagScan, sizeof(STagScanPhyNode));
+}
+
+static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
+  SSubplan* subplan = calloc(1, sizeof(SSubplan));
+  subplan->id = pCxt->nextId;
+  ++(pCxt->nextId.subplanId);
+  subplan->type = type;
+  subplan->level = 0;
+  if (NULL != pCxt->pCurrentSubplan) {
+    subplan->level = pCxt->pCurrentSubplan->level + 1;
+    if (NULL == pCxt->pCurrentSubplan->pChildern) {
+      pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+    }
+    taosArrayPush(pCxt->pCurrentSubplan->pChildern, subplan);
+    subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+    taosArrayPush(subplan->pParents, pCxt->pCurrentSubplan);
   }
-  return NULL;
+  pCxt->pCurrentSubplan = subplan;
+  return subplan;
 }
 
-SPhyNode* createSubplan(SQueryPlanNode* pSubquery) {
+static uint8_t getScanFlag(SQueryPlanNode* pPlanNode) {
-  return NULL;
+  // todo
+  return MASTER_SCAN;
 }
 
-int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) {
+static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
-  return 0;
+  STableScanPhyNode* node = (STableScanPhyNode*)initPhyNode(pPlanNode, OP_TableScan, sizeof(STableScanPhyNode));
+  node->scan.uid = pTable->pMeta->pTableMeta->uid;
+  node->scan.tableType = pTable->pMeta->pTableMeta->tableType;
+  node->scanFlag = getScanFlag(pPlanNode);
+  node->window = pTable->window;
+  // todo tag cond
+}
+
+static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) {
+  // todo
+}
+
+static void splitSubplanBySTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
+  SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList;
+  for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) {
+    SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
+    vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet);
+    subplan->pNode = createTableScanNode(pCxt, pPlanNode, pTable);
+    // todo reset pCxt->pCurrentSubplan
+  }
+}
+
+static SPhyNode* createExchangeNode() {
+
+}
+
+static SPhyNode* createScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
+  SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
+  if (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType) {
+    splitSubplanBySTable(pCxt, pPlanNode, pTable);
+    return createExchangeNode(pCxt, pTable);
+  }
+  return createTableScanNode(pCxt, pPlanNode, pTable);
+}
+
+static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
+  SPhyNode* node = NULL;
+  switch (pPlanNode->info.type) {
+    case QNODE_TAGSCAN:
+      node = createTagScanNode(pPlanNode);
+      break;
+    case QNODE_TABLESCAN:
+      node = createScanNode(pCxt, pPlanNode);
+      break;
+    default:
+      assert(false);
+  }
+  if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) {
+    node->pChildren = taosArrayInit(4, POINTER_BYTES);
+    size_t size = taosArrayGetSize(pPlanNode->pChildren);
+    for(int32_t i = 0; i < size; ++i) {
+      SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i));
+      child->pParent = node;
+      taosArrayPush(node->pChildren, &child);
+    }
+  }
+  return node;
+}
+
+static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) {
+  SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE);
+  subplan->pNode = createPhyNode(pCxt, pRoot);
+  SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+  taosArrayPush(l0, &subplan);
+  taosArrayPush(pCxt->pDag->pSubplans, &l0);
+  // todo deal subquery
+}
+
+int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) {
+  SPlanContext context = {
+    .pCatalog = pCatalog,
+    .pDag = calloc(1, sizeof(SQueryDag)),
+    .pCurrentSubplan = NULL
+  };
+  if (NULL == context.pDag) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+  createSubplanByLevel(&context, pQueryNode);
+  *pDag = context.pDag;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) {
+  return TSDB_CODE_SUCCESS;
 }

source/libs/planner/src/planner.c

@@ -18,21 +18,6 @@
 #include "parser.h"
 #include "plannerInt.h"
 
-#define QNODE_TAGSCAN       1
-#define QNODE_TABLESCAN     2
-#define QNODE_PROJECT       3
-#define QNODE_AGGREGATE     4
-#define QNODE_GROUPBY       5
-#define QNODE_LIMIT         6
-#define QNODE_JOIN          7
-#define QNODE_DISTINCT      8
-#define QNODE_SORT          9
-#define QNODE_UNION         10
-#define QNODE_TIMEWINDOW    11
-#define QNODE_SESSIONWINDOW 12
-#define QNODE_STATEWINDOW   13
-#define QNODE_FILL          14
-
 typedef struct SFillEssInfo {
   int32_t  fillType;  // fill type
   int64_t *val;       // fill value
@@ -104,12 +89,13 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla
     taosArrayPush(pNode->pExpr, &pExpr[i]);
   }
 
-  pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES);
+  pNode->pChildren = taosArrayInit(4, POINTER_BYTES);
   for(int32_t i = 0; i < numOfPrev; ++i) {
-    taosArrayPush(pNode->pPrevNodes, &prev[i]);
+    taosArrayPush(pNode->pChildren, &prev[i]);
   }
 
   switch(type) {
+    case QNODE_TAGSCAN:
     case QNODE_TABLESCAN: {
       SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo));
       memcpy(info, pExtInfo, sizeof(SQueryTableInfo));
@@ -177,7 +163,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe
                                         SArray* pExprs, SArray* tableCols) {
   if (pQueryInfo->info.onlyTagQuery) {
     int32_t num = (int32_t) taosArrayGetSize(pExprs);
-    SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, NULL);
+    SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info);
 
     if (pQueryInfo->info.distinct) {
       pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL);
@@ -386,14 +372,14 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
   tfree(pQueryNode->info.name);
 //  dropAllExprInfo(pQueryNode->pExpr);
 
-  if (pQueryNode->pPrevNodes != NULL) {
+  if (pQueryNode->pChildren != NULL) {
-    int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes);
+    int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren);
     for(int32_t i = 0; i < size; ++i) {
-      SQueryPlanNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i);
+      SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i);
       doDestroyQueryNode(p);
     }
 
-    taosArrayDestroy(pQueryNode->pPrevNodes);
+    taosArrayDestroy(pQueryNode->pChildren);
   }
 
   tfree(pQueryNode);
@@ -607,8 +593,8 @@ int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t
 int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
   int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
 
-  for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) {
+  for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) {
-    SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i);
+    SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i);
     int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
     len = len1;
   }

source/libs/query/src/querymsg.c

@@ -21,17 +21,6 @@ int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msg
 
 int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0};
 
-
-int32_t queryBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) {
-  if (NULL == msg || NULL == msgLen) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
-  }
-
-  *msgLen = 0;
-
-  return TSDB_CODE_SUCCESS;
-}
-
 int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) {
   if (NULL == input || NULL == msg || NULL == msgLen) {
     return TSDB_CODE_TSC_INVALID_INPUT;
@@ -81,8 +70,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
   strncpy(bMsg->db, bInput->db, sizeof(bMsg->db));
   bMsg->db[sizeof(bMsg->db) - 1] = 0;
 
-  bMsg->vgroupVersion = bInput->vgroupVersion;
+  bMsg->vgVersion = bInput->vgVersion;
-  bMsg->dbGroupVersion = bInput->dbGroupVersion;
 
   *msgLen = (int32_t)sizeof(*bMsg);
 
@@ -90,58 +78,12 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
 }
 
 
-
-int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) {
-  if (NULL == output || NULL == msg || msgSize <= 0) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
-  }
-
-  SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg;
-
-  pRsp->vgroupNum = htonl(pRsp->vgroupNum);
-  pRsp->vgroupVersion = htonl(pRsp->vgroupVersion);
-
-  if (pRsp->vgroupNum < 0) {
-    qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum);
-    return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
-  }
-
-  if (pRsp->vgroupVersion < 0) {
-    qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion);
-    return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
-  }
-
-  if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) {
-    qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum);
-    return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
-  }
-
-  // keep SVgroupListInfo/SVgroupListRspMsg the same
-  *(SVgroupListInfo **)output = (SVgroupListInfo *)msg;
-
-  if (pRsp->vgroupNum == 0) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  for (int32_t i = 0; i < pRsp->vgroupNum; ++i) {
-    pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
-    for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
-      pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port);
-    }
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-
-
-
 int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
   if (NULL == output || NULL == msg || msgSize <= 0) {
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
 
-  SUseDbRspMsg *pRsp = (SUseDbRspMsg *)msg;
+  SUseDbRsp *pRsp = (SUseDbRsp *)msg;
   SUseDbOutput *pOut = (SUseDbOutput *)output;
   int32_t code = 0;
 
@@ -150,108 +92,182 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
     return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
   }
   
-  pRsp->vgroupVersion = htonl(pRsp->vgroupVersion);
+  pRsp->vgVersion = htonl(pRsp->vgVersion);
-  pRsp->dbVgroupVersion = htonl(pRsp->dbVgroupVersion);
+  pRsp->vgNum = htonl(pRsp->vgNum);
 
-  pRsp->vgroupNum = htonl(pRsp->vgroupNum);
+  if (pRsp->vgNum < 0) {
-  pRsp->dbVgroupNum = htonl(pRsp->dbVgroupNum);
+    qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum);
-
-  if (pRsp->vgroupNum < 0) {
-    qError("invalid vgroup number[%d]", pRsp->vgroupNum);
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  if (pRsp->dbVgroupNum < 0) {
+  int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp);
-    qError("invalid db vgroup number[%d]", pRsp->dbVgroupNum);
-    return TSDB_CODE_TSC_INVALID_VALUE;
-  }
-
-  int32_t expectSize = pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + pRsp->dbVgroupNum * sizeof(int32_t) + sizeof(*pRsp);
   if (msgSize != expectSize) {
-    qError("vgroup list msg size mis-match, msgSize:%d, expected:%d, vgroup number:%d, db vgroup number:%d", msgSize, expectSize, pRsp->vgroupNum, pRsp->dbVgroupNum);
+    qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum);
     return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
   }
 
-  if (pRsp->vgroupVersion < 0) {
+  pOut->dbVgroup.vgVersion = pRsp->vgVersion;
-    qInfo("no new vgroup list info");
+  pOut->dbVgroup.hashMethod = pRsp->hashMethod;
-    if (pRsp->vgroupNum != 0) {
+  pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-      qError("invalid vgroup number[%d] for no new vgroup list case", pRsp->vgroupNum);
+  if (NULL == pOut->dbVgroup.vgInfo) {
-      return TSDB_CODE_TSC_INVALID_VALUE;
+    qError("hash init[%d] failed", pRsp->vgNum);
-    }
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
-  } else {
-    int32_t s = sizeof(*pOut->vgroupList) + sizeof(pOut->vgroupList->vgroupInfo[0]) * pRsp->vgroupNum;
-    pOut->vgroupList = calloc(1, s);
-    if (NULL == pOut->vgroupList) {
-      qError("calloc size[%d] failed", s);
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
-
-    pOut->vgroupList->vgroupNum = pRsp->vgroupNum;
-    pOut->vgroupList->vgroupVersion = pRsp->vgroupVersion;
-
-    for (int32_t i = 0; i < pRsp->vgroupNum; ++i) {
-      pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
-      for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
-        pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port);
-      }
-
-      memcpy(&pOut->vgroupList->vgroupInfo[i], &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]));
-    }
   }
 
-  int32_t *vgIdList = (int32_t *)((char *)pRsp->vgroupInfo + sizeof(pRsp->vgroupInfo[0]) * pRsp->vgroupNum);
+  for (int32_t i = 0; i < pRsp->vgNum; ++i) {
+    pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
+    pRsp->vgroupInfo[i].hashBegin = htonl(pRsp->vgroupInfo[i].hashBegin);
+    pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd);
+
+    for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
+      pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port);
+    }
+
+    if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) {
+      qError("hash push failed");
+      goto _return;
+    }
+  }
 
   memcpy(pOut->db, pRsp->db, sizeof(pOut->db));
-  
-  if (pRsp->dbVgroupVersion < 0) {
-    qInfo("no new vgroup info for db[%s]", pRsp->db);
-  } else {
-    pOut->dbVgroup = calloc(1, sizeof(*pOut->dbVgroup));
-    if (NULL == pOut->dbVgroup) {
-      qError("calloc size[%d] failed", (int32_t)sizeof(*pOut->dbVgroup));
-      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _exit;
-    }
-
-    pOut->dbVgroup->vgId = taosArrayInit(pRsp->dbVgroupNum, sizeof(int32_t));
-    if (NULL == pOut->dbVgroup->vgId) {
-      qError("taosArrayInit size[%d] failed", pRsp->dbVgroupNum);
-      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _exit;
-    }
-    
-    pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion;
-    pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange);
-
-    for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) {
-      *(vgIdList + i) = htonl(*(vgIdList + i));
-      
-      taosArrayPush(pOut->dbVgroup->vgId, vgIdList + i) ;
-    }
-  }
 
   return code;
 
-_exit:
+_return:
-  if (pOut->dbVgroup && pOut->dbVgroup->vgId) {
+  if (pOut) {
-    taosArrayDestroy(pOut->dbVgroup->vgId);
+    tfree(pOut->dbVgroup.vgInfo);
-    pOut->dbVgroup->vgId = NULL;
   }
   
-  tfree(pOut->dbVgroup);
+  return code;
-  tfree(pOut->vgroupList);
+}
 
+static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) {
+  pMetaMsg->numOfTags = htonl(pMetaMsg->numOfTags);
+  pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns);
+  pMetaMsg->sversion = htonl(pMetaMsg->sversion);
+  pMetaMsg->tversion = htonl(pMetaMsg->tversion);
+  pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
+  pMetaMsg->suid = htobe64(pMetaMsg->suid);
+  pMetaMsg->vgId = htonl(pMetaMsg->vgId);
+
+  if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) {
+    qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) {
+    qError("invalid numOfColumns[%d] in table meta rsp msg", pMetaMsg->numOfColumns);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && pMetaMsg->tableType != TSDB_NORMAL_TABLE) {
+    qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  if (pMetaMsg->sversion < 0) {
+    qError("invalid sversion[%d] in table meta rsp msg", pMetaMsg->sversion);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  if (pMetaMsg->tversion < 0) {
+    qError("invalid tversion[%d] in table meta rsp msg", pMetaMsg->tversion);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+  
+  SSchema* pSchema = pMetaMsg->pSchema;
+
+  int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags;
+  for (int i = 0; i < numOfTotalCols; ++i) {
+    pSchema->bytes = htonl(pSchema->bytes);
+    pSchema->colId = htonl(pSchema->colId);
+
+    pSchema++;
+  }
+
+  if (pMetaMsg->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
+    qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchema[0].colId);
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STableMeta **pMeta) {
+  int32_t total = msg->numOfColumns + msg->numOfTags;
+  int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total;
+  
+  STableMeta* pTableMeta = calloc(1, metaSize);
+  if (NULL == pTableMeta) {
+    qError("calloc size[%d] failed", metaSize);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  
+  pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType;
+  pTableMeta->uid = msg->suid;
+  pTableMeta->suid = msg->suid;
+  pTableMeta->sversion = msg->sversion;
+  pTableMeta->tversion = msg->tversion;
+
+  pTableMeta->tableInfo.numOfTags = msg->numOfTags;
+  pTableMeta->tableInfo.precision = msg->precision;
+  pTableMeta->tableInfo.numOfColumns = msg->numOfColumns;
+
+  for(int32_t i = 0; i < msg->numOfColumns; ++i) {
+    pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
+  }
+
+  memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total);
+
+  *pMeta = pTableMeta;
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
+  STableMetaMsg *pMetaMsg = (STableMetaMsg *)msg;
+  int32_t code = queryConvertTableMetaMsg(pMetaMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  STableMetaOutput *pOut = (STableMetaOutput *)output;
+  
+  if (!tIsValidSchema(pMetaMsg->pSchema, pMetaMsg->numOfColumns, pMetaMsg->numOfTags)) {
+    qError("validate table meta schema in rsp msg failed");
+    return TSDB_CODE_TSC_INVALID_VALUE;
+  }
+
+  if (pMetaMsg->tableType == TSDB_CHILD_TABLE) {
+    pOut->metaNum = 2;
+    
+    memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname));
+    memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname));
+    
+    pOut->ctbMeta.vgId = pMetaMsg->vgId;
+    pOut->ctbMeta.tableType = pMetaMsg->tableType;
+    pOut->ctbMeta.uid = pMetaMsg->tuid;
+    pOut->ctbMeta.suid = pMetaMsg->suid;
+
+    code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta);
+  } else {
+    pOut->metaNum = 1;
+    
+    memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname));
+    
+    code = queryCreateTableMetaFromMsg(pMetaMsg, false, &pOut->tbMeta);
+  }
+  
   return code;
 }
 
 
 void msgInit() {
   queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg;
-  queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg;
   queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg;
 
-  //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp;
+  queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp;
-  queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp;
   queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp;
 
 /*

source/libs/scheduler/CMakeLists.txt

@@ -9,5 +9,5 @@ target_include_directories(
 
 target_link_libraries(
         scheduler
-        PRIVATE os util planner
+        PRIVATE os util planner common
 )

source/util/src/tarray.c

@@ -17,7 +17,7 @@
 #include "tarray.h"
 #include "talgo.h"
 
-void* taosArrayInit(size_t size, size_t elemSize) {
+SArray* taosArrayInit(size_t size, size_t elemSize) {
   assert(elemSize > 0);
 
   if (size < TARRAY_MIN_SIZE) {

source/util/src/tlog.c

@@ -81,15 +81,11 @@ int32_t writeInterval = DEFAULT_LOG_INTERVAL;
 // log
 int32_t tsNumOfLogLines = 10000000;
 int32_t mDebugFlag = 131;
-int32_t sdbDebugFlag = 131;
 int32_t dDebugFlag = 135;
 int32_t vDebugFlag = 135;
 int32_t cDebugFlag = 131;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
-int32_t httpDebugFlag = 131;
-int32_t mqttDebugFlag = 131;
-int32_t monDebugFlag = 131;
 int32_t qDebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;

src/balance/CMakeLists.txt

@@ -1,11 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
-INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
-INCLUDE_DIRECTORIES(inc)
-AUX_SOURCE_DIRECTORY(src SRC)
-
-ADD_LIBRARY(balance ${SRC})

src/balance/inc/bnInt.h

@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_BALANCE_INT_H
-#define TDENGINE_BALANCE_INT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "mnodeInt.h"
-#include "mnodeDef.h"
-#include "mnodeDnode.h"
-
-typedef struct {
-  int32_t     size;
-  int32_t     maxSize;
-  SDnodeObj **list;
-} SBnDnodes;
-
-typedef struct {
-  void *          timer;
-  bool            stop;
-  pthread_mutex_t mutex;
-  pthread_cond_t  cond;
-  pthread_t       thread;
-} SBnThread;
-
-typedef struct {
-  pthread_mutex_t mutex;
-} SBnMgmt;
-
-int32_t bnInit();
-void    bnCleanUp();
-bool    bnStart();
-void    bnCheckStatus();
-void    bnCheckModules();
-
-extern SBnDnodes tsBnDnodes;
-extern void *tsMnodeTmr;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/balance/inc/bnScore.h

@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_BALANCE_SCORE_H
-#define TDENGINE_BALANCE_SCORE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "bnInt.h"
-
-void  bnInitDnodes();
-void  bnCleanupDnodes();
-void  bnAccquireDnodes();
-void  bnReleaseDnodes();
-float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/balance/inc/bnThread.h

@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_BALANCE_THREAD_H
-#define TDENGINE_BALANCE_THREAD_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "bnInt.h"
-
-int32_t bnInitThread();
-void    bnCleanupThread();
-void    bnNotify();
-void    bnStartTimer(int32_t mseconds);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/balance/src/bnMain.c

@@ -1,728 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tref.h"
-#include "tsync.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "bnInt.h"
-#include "bnScore.h"
-#include "bnThread.h"
-#include "mnodeDb.h"
-#include "mnodeMnode.h"
-#include "mnodeSdb.h"
-#include "mnodeShow.h"
-#include "mnodeUser.h"
-#include "mnodeVgroup.h"
-
-extern int64_t tsDnodeRid;
-extern int32_t tsSdbRid;
-static SBnMgmt tsBnMgmt;
-static void  bnMonitorDnodeModule();
-
-static void bnLock() {
-  pthread_mutex_lock(&tsBnMgmt.mutex);
-}
-
-static void bnUnLock() {
-  pthread_mutex_unlock(&tsBnMgmt.mutex);
-}
-
-static bool bnCheckFree(SDnodeObj *pDnode) {
-  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-    mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]);
-    return false;
-  }
-  
-  if (pDnode->openVnodes >= TSDB_MAX_VNODES) {
-    mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES);
-    return false;
-  }
-
-  if (pDnode->diskAvailable <= tsMinimalDataDirGB) {
-    mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable);
-    return false;
-  }
-
-  if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) {
-    mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId);
-    return false;
-  }
-
-  return true;
-}
-
-static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
-  SVnodeGid tmp = *pVnodeGid1;
-  *pVnodeGid1 = *pVnodeGid2;
-  *pVnodeGid2 = tmp;
-}
-
-static void bnAdjustVnodeIndex(SVgObj *pInVg) {
-  int32_t d0Id = pInVg->vnodeGid[0].dnodeId;
-  int32_t d1Id = pInVg->vnodeGid[1].dnodeId;
-  int32_t d2Id = pInVg->vnodeGid[2].dnodeId;
-
-  int32_t vgId = pInVg->vgId;
-  int32_t d0Num = 0;
-  int32_t d1Num = 0;
-  int32_t d2Num = 0;
-
-  void *pIter = NULL;
-  while (1) {
-    SVgObj *pVgroup = NULL;
-    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-    if (pVgroup == NULL) break;
-
-    if (pVgroup->vgId != vgId) {
-      if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++;
-      if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++;
-      if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++;
-    }
-
-    mnodeDecVgroupRef(pVgroup);
-  }
-
-  if (pInVg->numOfVnodes == 1) {
-  }
-
-  if (pInVg->numOfVnodes == 2) {
-    mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num);
-    if (d0Num > d1Num) {
-      mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
-      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
-    }
-  }
-
-  if (pInVg->numOfVnodes >= 3) {
-    mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num);
-    if (d0Num <= d1Num && d0Num <= d2Num) {
-      if (d1Num > d2Num) {
-        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
-        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
-      }
-    } else if (d1Num <= d2Num && d1Num <= d0Num) {
-      mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId);
-      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]);
-      if (d0Num > d2Num) {
-        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
-        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
-      }
-    } else {
-      mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId);
-      bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]);
-      if (d1Num > d0Num) {
-        mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId);
-        bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]);
-      }
-    }
-  }
-
-  for (int i = 0; i < pInVg->numOfVnodes; ++i) {
-    mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId);
-  }
-}
-
-static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
-  mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
-
-  SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
-  if (pDnode != NULL) {
-    atomic_sub_fetch_32(&pDnode->openVnodes, 1);
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */
-  int32_t   numOfVnodes = 0;
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i;
-    if (pTmpVodeGid == pVnodeGid) {
-      continue;
-    }
-    vnodeGid[numOfVnodes] = *pTmpVodeGid;
-    ++numOfVnodes;
-  }
-  memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
-  pVgroup->numOfVnodes = numOfVnodes;
-
-  bnAdjustVnodeIndex(pVgroup);
-  mnodeUpdateVgroup(pVgroup);
-}
-
-int32_t bnAllocVnodes(SVgObj *pVgroup) {
-  int32_t dnode = 0;
-  int32_t vnodes = 0;
-
-  bnLock();
-  bnAccquireDnodes();
-
-  mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes,
-         mnodeGetDnodesNum(), tsBnDnodes.size);
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    for (; dnode < tsBnDnodes.size; ++dnode) {
-      SDnodeObj *pDnode = tsBnDnodes.list[dnode];
-      if (bnCheckFree(pDnode)) {
-        SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
-        pVnodeGid->dnodeId = pDnode->dnodeId;
-        pVnodeGid->pDnode = pDnode;
-        dnode++;
-        vnodes++;
-        mDebug("dnode:%d, is selected, vnodeIndex:%d", pDnode->dnodeId, i);
-        break;
-      } else {
-        mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId,
-               dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
-      }
-    }
-  }
-
-  if (vnodes != pVgroup->numOfVnodes) {
-    bnReleaseDnodes();
-    bnUnLock();
-
-    mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes);
-
-    void *     pIter = NULL;
-    SDnodeObj *pDnode = NULL;
-    while (1) {
-      pIter = mnodeGetNextDnode(pIter, &pDnode);
-      if (pDnode == NULL) break;
-      mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status],
-             pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
-      mnodeDecDnodeRef(pDnode);
-    }
-
-    if (mnodeGetOnlineDnodesNum() == 0) {
-      return TSDB_CODE_MND_NOT_READY;
-    } else {
-      return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-    }
-  }
-
-  bnAdjustVnodeIndex(pVgroup);
-  bnReleaseDnodes();
-  bnUnLock();
-  return TSDB_CODE_SUCCESS;
-}
-
-static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
-  int32_t rmVnodeVer = 0;
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
-    if (pVnode == pRmVnode) {
-      rmVnodeVer = mnodeGetVgidVer(pVnode->vver);
-      mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i,
-             pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer);
-    }
-  }
-
-  bool isReady = false;
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
-    SDnodeObj *pDnode = pVnode->pDnode;
-    if (pVnode == pRmVnode) continue;
-    int32_t vver = mnodeGetVgidVer(pVnode->vver);
-
-    mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i,
-           pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
-    if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
-    if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue;
-
-    if (rmVnodeVer == 0 || vver >= rmVnodeVer) {
-      mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d",
-            pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
-      isReady = true;
-    }
-  }
-
-  return isReady;
-}
-
-/**
- * desc: remove one vnode from vgroup
- * all vnodes in vgroup should in ready state, except the balancing one
- **/
-static int32_t bnRemoveVnode(SVgObj *pVgroup) {
-  if (pVgroup->numOfVnodes <= 1) return -1;
-
-  SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1];
-  mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId);
-
-  if (!bnCheckVgroupReady(pVgroup, pSelVnode)) {
-    mDebug("vgId:%d, is not ready", pVgroup->vgId);
-    return -1;
-  } else {
-    mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
-    bnDiscardVnode(pVgroup, pSelVnode);
-    return TSDB_CODE_SUCCESS;
-  }
-}
-
-static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
- for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SVnodeGid *pGid = &pVgroup->vnodeGid[i];
-    if (pGid->dnodeId == 0) break;
-    if (pGid->dnodeId == pDnode->dnodeId) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) {
-  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-    SDnodeObj *pDnode = tsBnDnodes.list[i];
-    if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue;
-    if (!bnCheckFree(pDnode)) continue;
-
-    mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
-    return pDnode;
-  }
-
-  return NULL;
-}
-
-static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
-  if (pDestDnode == NULL || pSrcDnode == pDestDnode) {
-    return TSDB_CODE_MND_DNODE_NOT_EXIST;
-  }
-
-  SVnodeGid vnodeGids[TSDB_MAX_REPLICA];
-  memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
-
-  int32_t numOfVnodes = pVgroup->numOfVnodes;
-  vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId;
-  vnodeGids[numOfVnodes].pDnode = pDestDnode;
-  numOfVnodes++;
-
-  // move the src vnode to the end
-  for (int32_t v = 0; v < numOfVnodes; ++v) {
-    if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) {
-      bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]);
-      pVgroup->lbDnodeId = pSrcDnode->dnodeId;
-      break;
-    }
-  }
-
-  // adjust the vgroup postion
-  if (pSrcDnode == NULL) {
-    bnAdjustVnodeIndex(pVgroup);
-  }
-
-  memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA);
-  pVgroup->numOfVnodes = numOfVnodes;
-  atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
-
-  mnodeUpdateVgroup(pVgroup);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static bool bnMonitorBalance() {
-  if (tsBnDnodes.size < 2) return false;
-
-  mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size);
-  for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) {
-    SDnodeObj *pDnode = tsBnDnodes.list[src];
-    mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId,
-           dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes);
-  }
-
-  float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score;
-  if (scoresDiff < 0.01) {
-    mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff);
-    return false;
-  }
-
-  for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) {
-    SDnodeObj *pSrcDnode = tsBnDnodes.list[src];
-    float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1);
-    if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
-      continue;
-    }
-
-    void *pIter = NULL;
-    while (1) {
-      SVgObj *pVgroup;
-      pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-      if (pVgroup == NULL) break;
-
-      if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
-        for (int32_t dest = 0; dest < src; dest++) {
-          SDnodeObj *pDestDnode = tsBnDnodes.list[dest];
-          if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
-          if (taosGetTimestampMs() - pDestDnode->createdTime < 2000) continue;
-
-          float destScore = bnTryCalcDnodeScore(pDestDnode, 1);
-          if (srcScore + 0.0001 < destScore) continue;
-          if (!bnCheckFree(pDestDnode)) continue;
-          
-          mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
-                 pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
-                 srcScore, pDestDnode->score, destScore);
-          bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
-          mnodeDecVgroupRef(pVgroup);
-          mnodeCancelGetNextVgroup(pIter);
-          return true;
-        }
-      }
-
-      mnodeDecVgroupRef(pVgroup);
-    }
-  }
-
-  return false;
-}
-
-// if mgmt changed to master
-// 1. reset balanceAccessSquence to zero
-// 2. reset state of dnodes to offline
-// 3. reset lastAccess of dnodes to zero
-void bnReset() {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    // while master change, should reset dnode to offline
-    mInfo("dnode:%d set access:%" PRId64 " to 0", pDnode->dnodeId, pDnode->lastAccess);
-    pDnode->lastAccess = 0;
-    if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
-      pDnode->status = TAOS_DN_STATUS_OFFLINE;
-      pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED;
-    }
-
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  tsAccessSquence = 0;
-}
-
-static bool bnMonitorVgroups() {
-  void *  pIter = NULL;
-  SVgObj *pVgroup = NULL;
-  bool    hasUpdatingVgroup = false;
-
-  while (1) {
-    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-    if (pVgroup == NULL || pVgroup->pDb == NULL) break;
-
-    int32_t dbReplica = pVgroup->pDb->cfg.replications;
-    int32_t vgReplica = pVgroup->numOfVnodes;
-    int32_t code = -1;
-    
-    if (vgReplica > dbReplica) {
-      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
-      hasUpdatingVgroup = true;
-      code = bnRemoveVnode(pVgroup);
-    } else if (vgReplica < dbReplica) {
-      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
-      hasUpdatingVgroup = true;
-
-      SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup);
-      if (pAvailDnode == NULL) {
-        code = TSDB_CODE_MND_DNODE_NOT_EXIST;
-      } else {
-        code = bnAddVnode(pVgroup, NULL, pAvailDnode);
-      }
-    }
-
-    mnodeDecVgroupRef(pVgroup);
-    if (code == TSDB_CODE_SUCCESS) {
-      mnodeCancelGetNextVgroup(pIter);
-      break;
-    }
-  }
-
-  return hasUpdatingVgroup;
-}
-
-static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) {
-  mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
-
-  void *  pIter = NULL;
-  bool    hasThisDnode = false;
-  while (1) {
-    SVgObj *pVgroup = NULL;
-    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-    if (pVgroup == NULL) break;
-
-    hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup);
-    mnodeDecVgroupRef(pVgroup);
-
-    if (hasThisDnode) {
-      mnodeCancelGetNextVgroup(pIter);
-      break;
-    }
-  }
-
-  if (!hasThisDnode) {
-    mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
-    mnodeDropDnode(pDnode, NULL);
-    return true;
-  }
-
-  return false;
-}
-
-static bool bnMontiorDropping() {
-  void *pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-
-  while (1) {
-    mnodeDecDnodeRef(pDnode);
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
-      if (dnodeIsMasterEp(pDnode->dnodeEp)) continue; 
-      if (mnodeGetDnodesNum() <= 1) continue;
-
-      mLInfo("dnode:%d, set to removing state for it offline:%" PRId64 " seconds", pDnode->dnodeId,
-              tsAccessSquence - pDnode->lastAccess);
-
-      pDnode->status = TAOS_DN_STATUS_DROPPING;
-      mnodeUpdateDnode(pDnode);
-      mnodeDecDnodeRef(pDnode);
-      mnodeCancelGetNextDnode(pIter);
-      return true;
-    }
-
-    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
-      bool ret = bnMonitorDnodeDropping(pDnode);
-      mnodeDecDnodeRef(pDnode);
-      mnodeCancelGetNextDnode(pIter);
-      return ret;
-    }
-  }
-
-  return false;
-}
-
-bool bnStart() {
-  if (!sdbIsMaster()) return false;
-
-  bnLock();
-  bnAccquireDnodes();
-
-  bnMonitorDnodeModule();
-
-  bool updateSoon = bnMontiorDropping();
-
-  if (!updateSoon) {
-    updateSoon = bnMonitorVgroups();
-  }
-
-  if (!updateSoon) {
-    updateSoon = bnMonitorBalance();
-  }
- 
-  bnReleaseDnodes();
-  bnUnLock();
-
-  return updateSoon;
-}
-
-static void bnSetVgroupOffline(SDnodeObj* pDnode) {
-  void *pIter = NULL;
-  while (1) {
-    SVgObj *pVgroup;
-    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
-    if (pVgroup == NULL) break;
-
-    for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-      if (pVgroup->vnodeGid[i].pDnode == pDnode) {
-        pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE;
-      }
-    }
-    mnodeDecVgroupRef(pVgroup);
-  }
-}
-
-void bnCheckStatus() {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-
-  void *dnodeSdb = taosAcquireRef(tsSdbRid, tsDnodeRid);
-  if (dnodeSdb == NULL) return;
-
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-    if (tsAccessSquence - pDnode->lastAccess > 3) {
-      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
-        pDnode->status = TAOS_DN_STATUS_OFFLINE;
-        pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT;
-        mInfo("dnode:%d, set to offline state, access seq:%" PRId64 " last seq:%" PRId64 " laststat:%d", pDnode->dnodeId,
-              tsAccessSquence, pDnode->lastAccess, pDnode->status);
-        bnSetVgroupOffline(pDnode);
-        bnStartTimer(3000);
-      }
-    }
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  taosReleaseRef(tsSdbRid, tsDnodeRid);
-}
-
-void bnCheckModules() {
-  if (sdbIsMaster()) {
-    bnLock();
-    bnAccquireDnodes();
-    bnMonitorDnodeModule();
-    bnReleaseDnodes();
-    bnUnLock();
-  }
-}
-
-int32_t bnInit() {
-  pthread_mutex_init(&tsBnMgmt.mutex, NULL);
-  bnInitDnodes();
-  bnInitThread();
-  bnReset();
-  
-  return 0;
-}
-
-void bnCleanUp() {
-  bnCleanupThread();
-  bnCleanupDnodes();
-  pthread_mutex_destroy(&tsBnMgmt.mutex);
-}
-
-int32_t bnDropDnode(SDnodeObj *pDnode) {
-  int32_t    totalFreeVnodes = 0;
-  void *     pIter = NULL;
-  SDnodeObj *pTempDnode = NULL;
-
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pTempDnode);
-    if (pTempDnode == NULL) break;
-
-    if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) {
-      totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
-    }
-
-    mnodeDecDnodeRef(pTempDnode);
-  }
-
-  if (pDnode->openVnodes > totalFreeVnodes) {
-    mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  }
-
-  pDnode->status = TAOS_DN_STATUS_DROPPING;
-  mnodeUpdateDnode(pDnode);
-  
-  bnStartTimer(1100);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode) {
-  if (pDnode == NULL)
-    return 0;
-
-  if (pDnode->isMgmt || pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE
-      || pDnode->status == TAOS_DN_STATUS_DROPPING
-      || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-    return 0;
-  } else {
-    return 1;
-  }
-}
-
-static void bnMonitorDnodeModule() {
-  int32_t numOfMnodes = mnodeGetMnodesNum();
-  if (numOfMnodes >= tsNumOfMnodes) return;
-
-  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-    SDnodeObj *pDnode = tsBnDnodes.list[i];
-    if (pDnode == NULL) break;
-
-    if (!bnDnodeCanCreateMnode(pDnode)) continue;
-
-    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, create mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
-    mnodeCreateMnode(pDnode->dnodeId, pDnode->dnodeEp, true);
-
-#if 0
-    // Only create one mnode each time
-    return;
-#else
-    numOfMnodes = mnodeGetMnodesNum();
-    if (numOfMnodes >= tsNumOfMnodes) return;
-#endif
-  }
-}
-
-int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
-  if (!sdbIsMaster()) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
-    return TSDB_CODE_MND_DNODE_NOT_EXIST;
-  }
-
-  if (tsEnableBalance != 0) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId);
-    return TSDB_CODE_MND_BALANCE_ENABLED;
-  }
-
-  SVgObj *pVgroup = mnodeGetVgroup(vnodeId);
-  if (pVgroup == NULL) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
-    return TSDB_CODE_MND_VGROUP_NOT_EXIST;
-  }
-
-  SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId);
-  if (pDestDnode == NULL) {
-    mnodeDecVgroupRef(pVgroup);
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
-    return TSDB_CODE_MND_DNODE_NOT_EXIST;
-  }
-
-  bnLock();
-  bnAccquireDnodes();
-
-  int32_t code = TSDB_CODE_SUCCESS;
-  if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
-           dnodeId, pSrcDnode->dnodeId);
-    code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
-  } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
-           dnodeId, dnodeId);
-    code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
-  } else if (!bnCheckFree(pDestDnode)) {
-    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
-           dnodeId);
-    code = TSDB_CODE_MND_DNODE_NOT_FREE;
-  } else {
-    code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode);
-    mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
-  }
-
-  bnReleaseDnodes();
-  bnUnLock();
-
-  mnodeDecVgroupRef(pVgroup);
-  mnodeDecDnodeRef(pDestDnode);
-
-  return code;
-}

src/balance/src/bnScore.c

@@ -1,321 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "tglobal.h"
-#include "mnodeShow.h"
-#include "mnodeUser.h"
-#include "bnScore.h"
-
-SBnDnodes tsBnDnodes;
-
-static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-
-static int32_t bnCalcCpuScore(SDnodeObj *pDnode) {
-  if (pDnode->cpuAvgUsage < 80)
-    return 0;
-  else if (pDnode->cpuAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) {
-  if (pDnode->memoryAvgUsage < 80)
-    return 0;
-  else if (pDnode->memoryAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t bnCalcDiskScore(SDnodeObj *pDnode) {
-  if (pDnode->diskAvgUsage < 80)
-    return 0;
-  else if (pDnode->diskAvgUsage < 90)
-    return 10;
-  else
-    return 50;
-}
-
-static int32_t bnCalcBandScore(SDnodeObj *pDnode) {
-  if (pDnode->bandwidthUsage < 30)
-    return 0;
-  else if (pDnode->bandwidthUsage < 80)
-    return 10;
-  else
-    return 50;
-}
-
-static float bnCalcModuleScore(SDnodeObj *pDnode) {
-  if (pDnode->numOfCores <= 0) return 0;
-  if (pDnode->isMgmt) {
-    return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
-  }
-  return 0;
-}
-
-static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
-  if (pDnode->numOfCores <= 0) return 0;
-  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
-}
-
-/**
- * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
- * 1. get the score config
- * 2. if the value is out of range, use border data
- * 3. otherwise use interpolation method
- **/
-static void bnCalcDnodeScore(SDnodeObj *pDnode) {
-  pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
-                  bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) +
-                  pDnode->customScore;
-}
-
-float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
-  int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) +
-                        bnCalcBandScore(pDnode);
-  float moduleScore = bnCalcModuleScore(pDnode);
-  float vnodeScore = bnCalcVnodeScore(pDnode, extra);
-
-  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
-  return score;
-}
-
-void bnInitDnodes() {
-  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta);
-  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores);
-  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
-
-  memset(&tsBnDnodes, 0, sizeof(SBnDnodes));
-  tsBnDnodes.maxSize = 16;
-  tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *));
-}
-
-void bnCleanupDnodes() {
-  if (tsBnDnodes.list != NULL) {
-    free(tsBnDnodes.list);
-    tsBnDnodes.list = NULL;
-  }
-}
-
-static void bnCheckDnodesSize(int32_t dnodesNum) {
-  if (tsBnDnodes.maxSize <= dnodesNum) {
-    int32_t maxSize = dnodesNum * 2;
-    SDnodeObj** list1 = NULL;
-    int32_t retry = 0;
-    
-    while(list1 == NULL && retry++ < 3) {
-      list1 = realloc(tsBnDnodes.list, maxSize * sizeof(SDnodeObj *));
-    }    
-    if(list1) {
-      tsBnDnodes.list = list1;
-      tsBnDnodes.maxSize = maxSize;
-    }
-  }
-}
-
-void bnAccquireDnodes() {
-  int32_t dnodesNum = mnodeGetDnodesNum();
-  bnCheckDnodesSize(dnodesNum);
-
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  int32_t    dnodeIndex = 0;
-
-  while (1) {
-    if (dnodeIndex >= dnodesNum) {
-      mnodeCancelGetNextDnode(pIter);
-      break;
-    }
-
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
-    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
-      mnodeDecDnodeRef(pDnode);
-      continue;
-    }
-
-    bnCalcDnodeScore(pDnode);
-    
-    int32_t orderIndex = dnodeIndex;
-    for (; orderIndex > 0; --orderIndex) {
-      if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) {
-        break;
-      }
-      tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1];
-    }
-    tsBnDnodes.list[orderIndex] = pDnode;
-    dnodeIndex++;
-  }
-
-  tsBnDnodes.size = dnodeIndex;
-}
-
-void bnReleaseDnodes() {
-  for (int32_t i = 0; i < tsBnDnodes.size; ++i) {
-    SDnodeObj *pDnode = tsBnDnodes.list[i];
-    if (pDnode != NULL) {
-      mnodeDecDnodeRef(pDnode);
-    }
-  }
-}
-
-static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mnodeGetUserFromConn(pConn);
-  if (pUser == NULL) return 0;
-
-  if (strcmp(pUser->pAcct->user, "root") != 0) {
-    mnodeDecUserRef(pUser);
-    return TSDB_CODE_MND_NO_RIGHTS;
-  }
-
-  int32_t  cols = 0;
-  SSchema *pSchema = pMeta->schema;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "id");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "system scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "custom scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "module scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "vnode scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
-  strcpy(pSchema[cols].name, "total scores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "open vnodes");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "cpu cores");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "balance state");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htons(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->numOfRows = mnodeGetDnodesNum();
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pIter = NULL;
-
-  mnodeDecUserRef(pUser);
-
-  return 0;
-}
-
-static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
-  int32_t    numOfRows = 0;
-  SDnodeObj *pDnode = NULL;
-  char *     pWrite;
-  int32_t    cols = 0;
-
-  while (numOfRows < rows) {
-    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
-    if (pDnode == NULL) break;
-
-    int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode);
-    float moduleScore = bnCalcModuleScore(pDnode);
-    float vnodeScore = bnCalcVnodeScore(pDnode, 0);
-
-    cols = 0;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->dnodeId;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (float)systemScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (float)pDnode->customScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (float)moduleScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (float)vnodeScore;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->openVnodes;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = pDnode->numOfCores;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]);
-    cols++;
-
-    numOfRows++;
-    mnodeDecDnodeRef(pDnode);
-  }
-
-  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  pShow->numOfReads += numOfRows;
-  return numOfRows;
-}

src/balance/src/bnThread.c

@@ -1,137 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "ttimer.h"
-#include "tglobal.h"
-#include "mnodeSdb.h"
-#include "bnThread.h"
-
-static SBnThread tsBnThread;
-
-static void *bnThreadFunc(void *arg) {
-  setThreadName("balance");
-
-  while (1) {
-    pthread_mutex_lock(&tsBnThread.mutex);
-    if (tsBnThread.stop) {
-      pthread_mutex_unlock(&tsBnThread.mutex);
-      break;
-    }
-
-    pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
-    mDebug("balance thread wakes up to work");
-    bool updateSoon = bnStart();
-    mDebug("balance thread finished this poll, updateSoon:%d", updateSoon);
-    
-    bnStartTimer(updateSoon ? 1000 : -1);
-    pthread_mutex_unlock(&(tsBnThread.mutex));
-  }
-
-  mDebug("balance thread is stopped");
-  return NULL;
-}
-
-int32_t bnInitThread() {
-  memset(&tsBnThread, 0, sizeof(SBnThread));
-  tsBnThread.stop = false;
-  pthread_mutex_init(&tsBnThread.mutex, NULL);
-  pthread_cond_init(&tsBnThread.cond, NULL);
-
-  pthread_attr_t thattr;
-  pthread_attr_init(&thattr);
-  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
-  int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL);
-  pthread_attr_destroy(&thattr);
-
-  if (ret != 0) {
-    mError("failed to create balance thread since %s", strerror(ret));
-    return -1;
-  }
-
-  bnStartTimer(2000);
-  mDebug("balance thread is created");
-  return 0;
-}
-
-void bnCleanupThread() {
-  mDebug("balance thread will be cleanup");
-
-  if (tsBnThread.timer != NULL) {
-    taosTmrStopA(&tsBnThread.timer);
-    tsBnThread.timer = NULL;
-    mDebug("stop balance timer");
-  }
-
-  pthread_mutex_lock(&tsBnThread.mutex);
-  tsBnThread.stop = true;
-  pthread_cond_signal(&tsBnThread.cond);
-  pthread_mutex_unlock(&(tsBnThread.mutex));
-  pthread_join(tsBnThread.thread, NULL);
-
-  pthread_cond_destroy(&tsBnThread.cond);
-  pthread_mutex_destroy(&tsBnThread.mutex);
-}
-
-static void bnPostSignal() {
-  if (tsBnThread.stop) return;
-
-  pthread_mutex_lock(&tsBnThread.mutex);
-  pthread_cond_signal(&tsBnThread.cond);
-  pthread_mutex_unlock(&(tsBnThread.mutex));
-}
-
-/*
- * once sdb work as mater, then tsAccessSquence reset to zero
- * increase tsAccessSquence every balance interval
- */
-
-static void bnProcessTimer(void *handle, void *tmrId) {
-  if (!sdbIsMaster()) return;
-  if (tsBnThread.stop) return;
-
-  tsBnThread.timer = NULL;
-  bnStartTimer(-1);
-  bnCheckStatus();
-
-  if (handle == NULL) {
-    ++tsAccessSquence;
-
-    if (tsAccessSquence % tsBalanceInterval == 0) {
-      mDebug("balance function is scheduled by timer");
-      bnPostSignal();
-    }
-  } else {
-    int64_t mseconds = (int64_t)handle;
-    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
-    bnPostSignal();
-  }
-}
-
-void bnStartTimer(int32_t mseconds) {
-  if (tsBnThread.stop) return;
-
-  if (mseconds != -1) {
-    mTrace("balance function will be called after %d ms", mseconds);
-    taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer);
-  } else {
-    taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
-  }
-}
-
-void bnNotify() {
-  bnStartTimer(500);
-}

src/inc/http.h

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_HTTP_H
-#define TDENGINE_HTTP_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-int32_t httpGetReqCount();
-int32_t httpInitSystem();
-int32_t httpStartSystem();
-void httpStopSystem();
-void httpCleanUpSystem();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/monitor.h

@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_MONITOR_H
-#define TDENGINE_MONITOR_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-typedef struct {
-  char *  acctId;
-  int64_t currentPointsPerSecond;
-  int64_t maxPointsPerSecond;
-  int64_t totalTimeSeries;
-  int64_t maxTimeSeries;
-  int64_t totalStorage;
-  int64_t maxStorage;
-  int64_t totalQueryTime;
-  int64_t maxQueryTime;
-  int64_t totalInbound;
-  int64_t maxInbound;
-  int64_t totalOutbound;
-  int64_t maxOutbound;
-  int64_t totalDbs;
-  int64_t maxDbs;
-  int64_t totalUsers;
-  int64_t maxUsers;
-  int64_t totalStreams;
-  int64_t maxStreams;
-  int64_t totalConns;
-  int64_t maxConns;
-  int8_t  accessState;
-} SAcctMonitorObj;
-
-int32_t monInitSystem();
-int32_t monStartSystem();
-void    monStopSystem();
-void    monCleanupSystem();
-void    monSaveAcctLog(SAcctMonitorObj *pMonObj);
-void    monSaveLog(int32_t level, const char *const format, ...);
-void    monExecuteSQL(char *sql);
-typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code);
-void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param);
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tacct.h

@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_ACCT_H
-#define TDENGINE_ACCT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum {
-  ACCT_GRANT_USER,
-  ACCT_GRANT_DB,
-  ACCT_GRANT_TABLE
-} EAcctGrantType;
-
-int32_t acctInit();
-void    acctCleanUp();
-int32_t acctCheck(void *pAcct, EAcctGrantType type);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tadmin.h

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_ADMIN_H
-#define TDENGINE_ADMIN_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct HttpServer;
-
-void adminInitHandle(struct HttpServer* pServer);
-void opInitHandle(struct HttpServer* pServer);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tbn.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_BALANCE_H
-#define TDENGINE_BALANCE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct SVgObj;
-struct SDnodeObj;
-
-int32_t bnInit();
-void    bnCleanUp();
-void    bnNotify();
-void    bnCheckModules();
-void    bnReset();
-int32_t bnAllocVnodes(struct SVgObj *pVgroup);
-int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId);
-int32_t bnDropDnode(struct SDnodeObj *pDnode);
-int32_t bnDnodeCanCreateMnode(struct SDnodeObj *pDnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tgrant.h

@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_GTANT_H
-#define TDENGINE_GTANT_H
-
-#ifdef __cplusplus
-"C" {
-#endif
-
-typedef enum {
-  TSDB_GRANT_ALL,
-  TSDB_GRANT_TIME,
-  TSDB_GRANT_USER,
-  TSDB_GRANT_DB,
-  TSDB_GRANT_TIMESERIES,
-  TSDB_GRANT_DNODE,
-  TSDB_GRANT_ACCT,
-  TSDB_GRANT_STORAGE,
-  TSDB_GRANT_SPEED,
-  TSDB_GRANT_QUERY_TIME,
-  TSDB_GRANT_CONNS,
-  TSDB_GRANT_STREAMS,
-  TSDB_GRANT_CPU_CORES,
-} EGrantType;
-
-int32_t grantInit();
-void    grantCleanUp();
-void    grantParseParameter();
-int32_t grantCheck(EGrantType grant);
-void    grantReset(EGrantType grant, uint64_t value);
-void    grantAdd(EGrantType grant, uint64_t value);
-void    grantRestore(EGrantType grant, uint64_t value);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tmqtt.h

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_MQTT_H
-#define TDENGINE_MQTT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t mqttInitSystem();
-int32_t mqttStartSystem();
-void    mqttStopSystem();
-void    mqttCleanUpSystem();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tp.h

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TP
-#define TDENGINE_TP
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t tpInit();
-void    tpCleanUp();
-void    tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tsync.h

@@ -1,127 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_SYNC_H
-#define TDENGINE_SYNC_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TAOS_SYNC_MAX_REPLICA 5
-#define TAOS_SYNC_MAX_INDEX   0x7FFFFFFF
-
-typedef enum {
-  TAOS_SYNC_ROLE_OFFLINE  = 0,
-  TAOS_SYNC_ROLE_UNSYNCED = 1,
-  TAOS_SYNC_ROLE_SYNCING  = 2,
-  TAOS_SYNC_ROLE_SLAVE    = 3,
-  TAOS_SYNC_ROLE_MASTER   = 4
-} ESyncRole;
-
-typedef enum {
-  TAOS_SYNC_STATUS_INIT  = 0,
-  TAOS_SYNC_STATUS_START = 1,
-  TAOS_SYNC_STATUS_FILE  = 2,
-  TAOS_SYNC_STATUS_CACHE = 3
-} ESyncStatus;
-
-typedef struct {
-  uint32_t  nodeId;    // node ID assigned by TDengine
-  uint16_t  nodePort;  // node sync Port
-  char      nodeFqdn[TSDB_FQDN_LEN]; // node FQDN  
-} SNodeInfo;
-
-typedef struct {
-  int8_t     quorum;    // number of confirms required, >=1 
-  int8_t     replica;   // number of replications, >=1
-  SNodeInfo  nodeInfo[TAOS_SYNC_MAX_REPLICA];
-} SSyncCfg;
-
-typedef struct {
-  int32_t  selfIndex;
-  uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
-  int32_t  role[TAOS_SYNC_MAX_REPLICA];
-} SNodesRole;
-
-// get the wal file from index or after
-// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
- 
-// when a forward pkt is received, call this to handle data
-typedef int32_t  (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
-
-// when forward is confirmed by peer, master call this API to notify app
-typedef void     (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code);
-
-// when role is changed, call this to notify app
-typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
-
-// if a number of retrieving data failed, call this to start flow control 
-typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
-
-// when data file is synced successfully, notity app
-typedef void     (*FStartSyncFile)(int32_t vgId);
-typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
-
-// get file version
-typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
-
-typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
-typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
-
-typedef struct {
-  int32_t  vgId;       // vgroup ID
-  uint64_t version;    // initial version
-  SSyncCfg syncCfg;    // configuration from mgmt
-  char     path[TSDB_FILENAME_LEN];  // path to the file
-  void *   pTsdb;
-  FGetWalInfo       getWalInfoFp;
-  FWriteToCache     writeToCacheFp;
-  FConfirmForward   confirmForward;
-  FNotifyRole       notifyRoleFp;
-  FNotifyFlowCtrl   notifyFlowCtrlFp;
-  FStartSyncFile    startSyncFileFp;
-  FStopSyncFile     stopSyncFileFp;
-  FGetVersion       getVersionFp;
-  FSendFile         sendFileFp;
-  FRecvFile         recvFileFp;
-} SSyncInfo;
-
-typedef void *tsync_h;
-
-int32_t syncInit();
-void    syncCleanUp();
-
-int64_t syncStart(const SSyncInfo *);
-void    syncStop(int64_t rid);
-int32_t syncReconfig(int64_t rid, const SSyncCfg *);
-int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force);
-void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force);
-void    syncRecover(int64_t rid);  // recover from other nodes:
-int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
-
-extern char *syncRole[];
-
-//global configurable parameters
-extern int32_t  sDebugFlag;
-extern char     tsArbitrator[];
-extern uint16_t tsSyncPort;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_SYNC_H

src/plugins/monitor/CMakeLists.txt

@@ -1,15 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-INCLUDE_DIRECTORIES(inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
-AUX_SOURCE_DIRECTORY(./src SRC)
-
-ADD_LIBRARY(monitor ${SRC})
-
-IF (TD_SOMODE_STATIC)
-  TARGET_LINK_LIBRARIES(monitor taos_static)
-ELSE ()
-  TARGET_LINK_LIBRARIES(monitor taos)
-ENDIF ()

src/plugins/monitor/src/monMain.c

@@ -1,436 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taosdef.h"
-#include "taoserror.h"
-#include "tlog.h"
-#include "ttimer.h"
-#include "tutil.h"
-#include "tscUtil.h"
-#include "tsclient.h"
-#include "dnode.h"
-#include "monitor.h"
-#include "taoserror.h"
-
-#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define monWarn(...)  { if (monDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define monInfo(...)  { if (monDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
-#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
-
-#define SQL_LENGTH     1030
-#define LOG_LEN_STR    512
-#define IP_LEN_STR     TSDB_EP_LEN
-#define CHECK_INTERVAL 1000
-
-typedef enum {
-  MON_CMD_CREATE_DB,
-  MON_CMD_CREATE_TB_LOG,
-  MON_CMD_CREATE_MT_DN,
-  MON_CMD_CREATE_MT_ACCT,
-  MON_CMD_CREATE_TB_DN,
-  MON_CMD_CREATE_TB_ACCT_ROOT,
-  MON_CMD_CREATE_TB_SLOWQUERY,
-  MON_CMD_MAX
-} EMonCmd;
-
-typedef enum {
-  MON_STATE_NOT_INIT,
-  MON_STATE_INITED
-} EMonState;
-
-typedef struct {
-  pthread_t thread;
-  void *    conn;
-  char      ep[TSDB_EP_LEN];
-  int8_t    cmdIndex;
-  int8_t    state;
-  int8_t    start;   // enable/disable by mnode
-  int8_t    quiting; // taosd is quiting 
-  char      sql[SQL_LENGTH + 1];
-} SMonConn;
-
-static SMonConn tsMonitor = {0};
-static void  monSaveSystemInfo();
-static void *monThreadFunc(void *param);
-static void  monBuildMonitorSql(char *sql, int32_t cmd);
-extern int32_t (*monStartSystemFp)();
-extern void    (*monStopSystemFp)();
-extern void    (*monExecuteSQLFp)(char *sql);
-
-int32_t monInitSystem() {
-  if (tsMonitor.ep[0] == 0) {
-    strcpy(tsMonitor.ep, tsLocalEp);
-  }
-
-  int32_t len = (int32_t)strlen(tsMonitor.ep);
-  for (int32_t i = 0; i < len; ++i) {
-    if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') {
-      tsMonitor.ep[i] = '_';
-    }
-  }
-
-  pthread_attr_t thAttr;
-  pthread_attr_init(&thAttr);
-  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-
-  if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) {
-    monError("failed to create thread to for monitor module, reason:%s", strerror(errno));
-    return -1;
-  }
-
-  pthread_attr_destroy(&thAttr);
-  monDebug("monitor thread is launched");
-
-  monStartSystemFp = monStartSystem;
-  monStopSystemFp = monStopSystem;
-  return 0;
-}
-
-int32_t monStartSystem() {
-  if (taos_init()) {
-    return -1;
-  }
-  tsMonitor.start = 1;
-  monExecuteSQLFp = monExecuteSQL;
-  monInfo("monitor module start");
-  return 0;
-}
-
-static void *monThreadFunc(void *param) {
-  monDebug("starting to initialize monitor module ...");
-  setThreadName("monitor");
-
-  while (1) {
-    static int32_t accessTimes = 0;
-    accessTimes++;
-    taosMsleep(1000);
-
-    if (tsMonitor.quiting) {
-      tsMonitor.state = MON_STATE_NOT_INIT;
-      monInfo("monitor thread will quit, for taosd is quiting");
-      break;
-    } else {
-      taosGetDisk();
-    }
-
-    if (tsMonitor.start == 0) {
-      continue;
-    }
-    
-    if (dnodeGetDnodeId() <= 0) {
-      monDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
-      continue;
-    }
-
-    if (tsMonitor.conn == NULL) {
-      tsMonitor.state = MON_STATE_NOT_INIT;
-      tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
-      if (tsMonitor.conn == NULL) {
-        monError("failed to connect to database, reason:%s", tstrerror(terrno));
-        continue;
-      } else {
-        monDebug("connect to database success");
-      }
-    }
-
-    if (tsMonitor.state == MON_STATE_NOT_INIT) {
-      int32_t code = 0;
-
-      for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) {
-        monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex);
-        void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
-        code = taos_errno(res);
-        taos_free_result(res);
-
-        if (code != 0) {
-          monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
-          break;
-        } else {
-          monDebug("successfully to exec sql:%s", tsMonitor.sql);
-        }
-      }
-
-      if (tsMonitor.start && code == 0) {
-        tsMonitor.state = MON_STATE_INITED;
-      }
-    }
-
-    if (tsMonitor.state == MON_STATE_INITED) {
-      if (accessTimes % tsMonitorInterval == 0) {
-        monSaveSystemInfo();
-      }
-    }
-  }
-
-  monInfo("monitor thread is stopped");
-  return NULL;
-}
-
-static void monBuildMonitorSql(char *sql, int32_t cmd) {
-  memset(sql, 0, SQL_LENGTH);
-
-#ifdef _STORAGE
-  char *keepValue = "30,30,30";
-#else
-  char *keepValue = "30";
-#endif
-
-  if (cmd == MON_CMD_CREATE_DB) {
-    snprintf(sql, SQL_LENGTH,
-             "create database if not exists %s replica 1 days 10 keep %s cache %d "
-             "blocks %d precision 'us'",
-             tsMonitorDbName, keepValue, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MIN_TOTAL_BLOCKS);
-  } else if (cmd == MON_CMD_CREATE_MT_DN) {
-    snprintf(sql, SQL_LENGTH,
-             "create table if not exists %s.dn(ts timestamp"
-             ", cpu_taosd float, cpu_system float, cpu_cores int"
-             ", mem_taosd float, mem_system float, mem_total int"
-             ", disk_used float, disk_total int"
-             ", band_speed float"
-             ", io_read float, io_write float"
-             ", req_http int, req_select int, req_insert int"
-             ") tags (dnodeid int, fqdn binary(%d))",
-             tsMonitorDbName, TSDB_FQDN_LEN);
-  } else if (cmd == MON_CMD_CREATE_TB_DN) {
-    snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn%d using %s.dn tags(%d, '%s')", tsMonitorDbName,
-             dnodeGetDnodeId(), tsMonitorDbName, dnodeGetDnodeId(), tsLocalEp);
-  } else if (cmd == MON_CMD_CREATE_MT_ACCT) {
-    snprintf(sql, SQL_LENGTH,
-             "create table if not exists %s.acct(ts timestamp "
-             ", currentPointsPerSecond bigint, maxPointsPerSecond bigint"
-             ", totalTimeSeries bigint, maxTimeSeries bigint"
-             ", totalStorage bigint, maxStorage bigint"
-             ", totalQueryTime bigint, maxQueryTime bigint"
-             ", totalInbound bigint, maxInbound bigint"
-             ", totalOutbound bigint, maxOutbound bigint"
-             ", totalDbs smallint, maxDbs smallint"
-             ", totalUsers smallint, maxUsers smallint"
-             ", totalStreams smallint, maxStreams smallint"
-             ", totalConns smallint, maxConns smallint"
-             ", accessState smallint"
-             ") tags (acctId binary(%d))",
-             tsMonitorDbName, TSDB_USER_LEN);
-  } else if (cmd == MON_CMD_CREATE_TB_ACCT_ROOT) {
-    snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct_%s using %s.acct tags('%s')", tsMonitorDbName, TSDB_DEFAULT_USER,
-             tsMonitorDbName, TSDB_DEFAULT_USER);
-  } else if (cmd == MON_CMD_CREATE_TB_SLOWQUERY) {
-    snprintf(sql, SQL_LENGTH,
-             "create table if not exists %s.slowquery(ts timestamp, username "
-             "binary(%d), created_time timestamp, time bigint, sql binary(%d))",
-             tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN);
-  } else if (cmd == MON_CMD_CREATE_TB_LOG) {
-    snprintf(sql, SQL_LENGTH,
-             "create table if not exists %s.log(ts timestamp, level tinyint, "
-             "content binary(%d), ipaddr binary(%d))",
-             tsMonitorDbName, LOG_LEN_STR, IP_LEN_STR);
-  }
-
-  sql[SQL_LENGTH] = 0;
-}
-
-void monStopSystem() {
-  tsMonitor.start = 0;
-  tsMonitor.state = MON_STATE_NOT_INIT;
-  monExecuteSQLFp = NULL;
-  monInfo("monitor module stopped");
-}
-
-void monCleanupSystem() {
-  tsMonitor.quiting = 1;
-  monStopSystem();
-  if (taosCheckPthreadValid(tsMonitor.thread)) {
-    pthread_join(tsMonitor.thread, NULL);
-  }
-
-  if (tsMonitor.conn != NULL) {
-    taos_close(tsMonitor.conn);
-    tsMonitor.conn = NULL;
-  }
-  monInfo("monitor module is cleaned up");
-}
-
-// unit is MB
-static int32_t monBuildMemorySql(char *sql) {
-  float sysMemoryUsedMB = 0;
-  bool  suc = taosGetSysMemory(&sysMemoryUsedMB);
-  if (!suc) {
-    monDebug("failed to get sys memory info");
-  }
-
-  float procMemoryUsedMB = 0;
-  suc = taosGetProcMemory(&procMemoryUsedMB);
-  if (!suc) {
-    monDebug("failed to get proc memory info");
-  }
-
-  return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
-}
-
-// unit is %
-static int32_t monBuildCpuSql(char *sql) {
-  float sysCpuUsage = 0, procCpuUsage = 0;
-  bool  suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
-  if (!suc) {
-    monDebug("failed to get cpu usage");
-  }
-
-  if (sysCpuUsage <= procCpuUsage) {
-    sysCpuUsage = procCpuUsage + 0.1f;
-  }
-
-  return sprintf(sql, ", %f, %f, %d", procCpuUsage, sysCpuUsage, tsNumOfCores);
-}
-
-// unit is GB
-static int32_t monBuildDiskSql(char *sql) {
-  return sprintf(sql, ", %f, %d", tsUsedDataDirGB, (int32_t)tsTotalDataDirGB);
-}
-
-// unit is Kb
-static int32_t monBuildBandSql(char *sql) {
-  float bandSpeedKb = 0;
-  bool  suc = taosGetBandSpeed(&bandSpeedKb);
-  if (!suc) {
-    monDebug("failed to get bandwidth speed");
-  }
-
-  return sprintf(sql, ", %f", bandSpeedKb);
-}
-
-static int32_t monBuildReqSql(char *sql) {
-  SStatisInfo info = dnodeGetStatisInfo();
-  return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
-}
-
-static int32_t monBuildIoSql(char *sql) {
-  float readKB = 0, writeKB = 0;
-  bool  suc = taosGetProcIO(&readKB, &writeKB);
-  if (!suc) {
-    monDebug("failed to get io info");
-  }
-
-  return sprintf(sql, ", %f, %f", readKB, writeKB);
-}
-
-static void monSaveSystemInfo() {
-  int64_t ts = taosGetTimestampUs();
-  char *  sql = tsMonitor.sql;
-  int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts);
-
-  pos += monBuildCpuSql(sql + pos);
-  pos += monBuildMemorySql(sql + pos);
-  pos += monBuildDiskSql(sql + pos);
-  pos += monBuildBandSql(sql + pos);
-  pos += monBuildIoSql(sql + pos);
-  pos += monBuildReqSql(sql + pos);
-
-  void *res = taos_query(tsMonitor.conn, tsMonitor.sql);
-  int32_t code = taos_errno(res);
-  taos_free_result(res);
-
-  if (code != 0) {
-    monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
-  } else {
-    monDebug("successfully to save system info, sql:%s", tsMonitor.sql);
-  }
-}
-
-static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
-  int32_t c = taos_errno(result);
-  if (c != TSDB_CODE_SUCCESS) {
-    monError("save %s failed, reason:%s", (char *)param, tstrerror(c));
-  } else {
-    int32_t rows = taos_affected_rows(result);
-    monDebug("save %s succ, rows:%d", (char *)param, rows);
-  }
-
-  taos_free_result(result);
-}
-
-void monSaveAcctLog(SAcctMonitorObj *pMon) {
-  if (tsMonitor.state != MON_STATE_INITED) return;
-
-  char sql[1024] = {0};
-  sprintf(sql,
-          "insert into %s.acct_%s using %s.acct tags('%s') values(now"
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %" PRId64 ", %" PRId64
-          ", %d)",
-          tsMonitorDbName, pMon->acctId, tsMonitorDbName, pMon->acctId,
-          pMon->currentPointsPerSecond, pMon->maxPointsPerSecond,
-          pMon->totalTimeSeries, pMon->maxTimeSeries,
-          pMon->totalStorage, pMon->maxStorage,
-          pMon->totalQueryTime, pMon->maxQueryTime,
-          pMon->totalInbound, pMon->maxInbound,
-          pMon->totalOutbound, pMon->maxOutbound,
-          pMon->totalDbs, pMon->maxDbs,
-          pMon->totalUsers, pMon->maxUsers,
-          pMon->totalStreams, pMon->maxStreams,
-          pMon->totalConns, pMon->maxConns,
-          pMon->accessState);
-
-  monDebug("save account info, sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info");
-}
-
-void monSaveLog(int32_t level, const char *const format, ...) {
-  if (tsMonitor.state != MON_STATE_INITED) return;
-
-  va_list argpointer;
-  char    sql[SQL_LENGTH] = {0};
-  int32_t max_length = SQL_LENGTH - 30;
-  int32_t len = snprintf(sql, (size_t)max_length, "insert into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName,
-                         taosGetTimestampUs(), level);
-
-  va_start(argpointer, format);
-  len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer);
-  va_end(argpointer);
-  if (len > max_length) len = max_length;
-
-  len += sprintf(sql + len, "', '%s')", tsLocalEp);
-  sql[len++] = 0;
-
-  monDebug("save log, sql: %s", sql);
-  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log");
-}
-
-void monExecuteSQL(char *sql) {
-  if (tsMonitor.state != MON_STATE_INITED) return;
-
-  monDebug("execute sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql");
-}
-
-void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) {
-  if (tsMonitor.conn == NULL) {
-    callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID);
-    return;
-  }
-
-  monDebug("execute sql:%s", sql);
-  taos_query_a(tsMonitor.conn, sql, callback, param);
-}

src/wal/CMakeLists.txt

@@ -1,11 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE)
-
-INCLUDE_DIRECTORIES(inc)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC)
-
-ADD_LIBRARY(twal ${SRC})
-TARGET_LINK_LIBRARIES(twal tutil common)
-ADD_SUBDIRECTORY(test)

src/wal/test/CMakeLists.txt

@@ -1,21 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-IF (TD_LINUX)
-  INCLUDE_DIRECTORIES(../inc)
-
-  LIST(APPEND WALTEST_SRC ./waltest.c)
-  ADD_EXECUTABLE(waltest ${WALTEST_SRC})
-  TARGET_LINK_LIBRARIES(waltest twal os tutil)
-
-ENDIF ()
-
-IF (TD_DARWIN)
-  INCLUDE_DIRECTORIES(../inc)
-
-  LIST(APPEND WALTEST_SRC ./waltest.c)
-  ADD_EXECUTABLE(waltest ${WALTEST_SRC})
-  TARGET_LINK_LIBRARIES(waltest twal os tutil)
-
-ENDIF ()
-