Merge branch 'feature/mpeer' into develop

cmake/define.inc

@@ -1,25 +1,18 @@
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF (TD_CLUSTER)
+IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_CLUSTER)
+  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
-IF (TD_MPEER)
-  ADD_DEFINITIONS(-D_MPEER)
-ENDIF ()
-
-IF (TD_VPEER)
-  ADD_DEFINITIONS(-D_VPEER)
-  #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=3)
-ELSE ()
-  #ADD_DEFINITIONS(-DTSDB_REPLICA_MAX_NUM=1)  
 ENDIF ()
 
 IF (TD_ACCOUNT)
   ADD_DEFINITIONS(-D_ACCOUNT)
 ENDIF ()
 
+IF (TD_ADMIN)
+  ADD_DEFINITIONS(-D_ADMIN)
+ENDIF ()
+
 IF (TD_GRANT)
   ADD_DEFINITIONS(-D_GRANT)
 ENDIF ()

src/client/inc/tsclient.h

@@ -28,12 +28,12 @@ extern "C" {
 #include "taos.h"
 #include "taosdef.h"
 #include "taosmsg.h"
+#include "tarray.h"
 #include "tglobalcfg.h"
 #include "tlog.h"
 #include "trpc.h"
 #include "tsqlfunction.h"
 #include "tutil.h"
-#include "tarray.h"
 
 #define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows)
 
@@ -61,10 +61,8 @@ typedef struct STableMeta {
   //super table if it is created according to super table, otherwise, tableInfo is used
   union { struct STableMeta* pSTable; STableComInfo tableInfo; };
   uint8_t    tableType;
-  int8_t     numOfVpeers;
   int16_t    sversion;
-  SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
+  SCMVgroupInfo vgroupInfo;
-  int32_t    vgId;     // virtual group id, which current table belongs to
   int32_t    sid;      // the index of one table in a virtual node
   uint64_t   uid;      // unique id of a table
   SSchema    schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
@@ -77,7 +75,7 @@ typedef struct STableMetaInfo {
    * 1. keep the vnode index during the multi-vnode super table projection query
    * 2. keep the vnode index for multi-vnode insertion
    */
-  int32_t dnodeIndex;
+  int32_t vgroupIndex;
   char    name[TSDB_TABLE_ID_LEN];        // (super) table name
   int16_t numOfTags;                      // total required tags in query, including groupby tags
   int16_t tagColumnIndex[TSDB_MAX_TAGS];  // clause + tag projection

src/client/src/tscAsync.c

@@ -45,9 +45,10 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   SSqlRes *pRes = &pSql->res;
   
   pSql->signature = pSql;
-  pSql->pTscObj = pObj;
-  pSql->fp = fp;
   pSql->param = param;
+  pSql->pTscObj = pObj;
+  pSql->maxRetry = TSDB_REPLICA_MAX_NUM;
+  pSql->fp = fp;
   
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     tscError("failed to malloc payload");
@@ -406,31 +407,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
-  if (pSql->fp == (void *)1) {
-    pSql->fp = NULL;
-
-    if (code != 0) {
-      pRes->code = code;
-      tscTrace("%p failed to renew tableMeta", pSql);
-//      tsem_post(&pSql->rspSem);
-    } else {
-      tscTrace("%p renew tableMeta successfully, command:%d, code:%d, retry:%d",
-          pSql, pSql->cmd.command, pSql->res.code, pSql->retry);
-  
-      STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-      assert(pTableMetaInfo->pTableMeta == NULL);
-      
-      tscGetTableMeta(pSql, pTableMetaInfo);
-      code = tscSendMsgToServer(pSql);
-      if (code != 0) {
-        pRes->code = code;
-//        tsem_post(&pSql->rspSem);
-      }
-    }
-
-    return;
-  }
-
   if (code != TSDB_CODE_SUCCESS) {
     pRes->code = code;
     tscQueueAsyncRes(pSql);
@@ -443,12 +419,12 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   
     if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->dnodeIndex >= 0 && pSql->param != NULL);
+      assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL);
 
       SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
       SSqlObj *         pParObj = trs->pParentSqlObj;
       
-      assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->dnodeIndex &&
+      assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0);
 
       tscTrace("%p get metricMeta during super table query successfully", pSql);

src/client/src/tscParseInsert.c

@@ -699,7 +699,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st
   SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData);
   tsSetBlockInfo(pBlocks, pTableMeta, numOfRows);
 
-  dataBuf->vgId = pTableMeta->vgId;
+  dataBuf->vgId = pTableMeta->vgroupInfo.vgId;
   dataBuf->numOfTables = 1;
 
   /*

src/client/src/tscPrepare.c

@@ -408,7 +408,7 @@ static int insertStmtReset(STscStmt* pStmt) {
   pCmd->batchSize = 0;
   
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-  pTableMetaInfo->dnodeIndex = 0;
+  pTableMetaInfo->vgroupIndex = 0;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -438,7 +438,7 @@ static int insertStmtExecute(STscStmt* stmt) {
     }
 
     // set the next sent data vnode index in data block arraylist
-    pTableMetaInfo->dnodeIndex = 1;
+    pTableMetaInfo->vgroupIndex = 1;
   } else {
     pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
   }

src/client/src/tscSchemaUtil.c

@@ -168,10 +168,8 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   
   pTableMeta->sid = pTableMetaMsg->sid;
   pTableMeta->uid = pTableMetaMsg->uid;
-  pTableMeta->vgId = pTableMetaMsg->vgId;
+  pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
   
-  pTableMeta->numOfVpeers = pTableMetaMsg->numOfVpeers;
-  memcpy(pTableMeta->vpeerDesc, pTableMetaMsg->vpeerDesc, sizeof(SVnodeDesc) * pTableMeta->numOfVpeers);
   memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
   
   int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;

src/client/src/tscSecondaryMerge.c

@@ -636,7 +636,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
 
-  size_t numOfSubs = pTableMetaInfo->vgroupList->numOfDnodes;
+  size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
   for (int32_t i = 0; i < numOfSubs; ++i) {
     (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
     (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;

src/client/src/tscServer.c

@@ -47,12 +47,12 @@ static int32_t minMsgSize() { return tsRpcHeadSize + 100; }
 static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) {
   SRpcIpSet* pIpList = &pSql->ipList;
   
-  pIpList->numOfIps = pTableMeta->numOfVpeers;
+  pIpList->numOfIps = pTableMeta->vgroupInfo.numOfIps;
   pIpList->port     = tsDnodeShellPort;
   pIpList->inUse    = 0;
   
-  for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) {
+  for(int32_t i = 0; i < pTableMeta->vgroupInfo.numOfIps; ++i) {
-    pIpList->ip[i] = pTableMeta->vpeerDesc[i].ip;
+    pIpList->ip[i] = pTableMeta->vgroupInfo.ipAddr[i].ip;
   }
 }
 
@@ -270,7 +270,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
       } else {
         tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
         
-        pSql->maxRetry = TSDB_VNODES_SUPPORT * 2;  // todo move away
         pSql->res.code = rpcMsg->code;  // keep the previous error code
         if (pSql->retry > pSql->maxRetry) {
           tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
@@ -327,7 +326,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
      * There is not response callback function for submit response.
      * The actual inserted number of points is the first number.
      */
-    if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP) {
+    if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) {
       SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp;
       pMsg->code = htonl(pMsg->code);
       pMsg->numOfRows = htonl(pMsg->numOfRows);
@@ -512,12 +511,12 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   // todo valid the vgroupId at the client side
   if (UTIL_TABLE_IS_SUPERTABLE(pQueryInfo->pTableMetaInfo[0])) {
     SVgroupsInfo* pVgroupInfo = pQueryInfo->pTableMetaInfo[0]->vgroupList;
-    assert(pVgroupInfo->dnodeVgroups->numOfVgroups == 1); // todo fix me
+    assert(pVgroupInfo->numOfVgroups == 1); // todo fix me
     
-    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->dnodeVgroups[0].vgId[0]);
+    pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[0].vgId);
   } else {
     STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta;
-    pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId);
+    pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId);
   }
   
   pMsg += sizeof(SRetrieveTableMsg);
@@ -543,8 +542,9 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pMsg += sizeof(SMsgDesc);
   
   SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
+  int32_t vgId = pTableMeta->vgroupInfo.vgId;
   
-  pShellMsg->header.vgId = htonl(pTableMeta->vgId);
+  pShellMsg->header.vgId = htonl(vgId);
   pShellMsg->header.contLen = htonl(size);
   pShellMsg->length = pShellMsg->header.contLen;
   
@@ -554,7 +554,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
   tscSetDnodeIpList(pSql, pTableMeta);
   
-  tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes));
+  tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes));
   return TSDB_CODE_SUCCESS;
 }
 
@@ -580,7 +580,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
   
 #if 0
   SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta;
-  SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->dnodeIndex);
+  SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex);
 
   int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableIdInfo)) * pVnodeSidList->numOfSids;
   int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg);
@@ -648,12 +648,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) {
     numOfTables = 1;
     tscSetDnodeIpList(pSql, pTableMeta);
-    pQueryMsg->head.vgId = htonl(pTableMeta->vgId);
+    pQueryMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId);
     tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name);
   } else {  // query super table
     
-    if (pTableMetaInfo->dnodeIndex < 0) {
+    if (pTableMetaInfo->vgroupIndex < 0) {
-      tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->dnodeIndex);
+      tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vgroupIndex);
       return -1;
     }
     
@@ -662,11 +662,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pSql->ipList.inUse    = 0;
   
     // todo extract method
-    STableDnodeVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->dnodeVgroups[pTableMetaInfo->dnodeIndex];
+    SCMVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[pTableMetaInfo->vgroupIndex];
-    pSql->ipList.ip[0] = pVgroupInfo->ipAddr.ip;
+    pSql->ipList.ip[0] = pVgroupInfo->ipAddr[0].ip;
     
 #if 0
-    SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->dnodeIndex);
+    SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex);
     uint32_t       vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode;
 
     numOfTables = pVnodeSidList->numOfSids;
@@ -676,10 +676,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     }
 #endif
     
-    tscTrace("%p query on super table, numOfVgroup:%d, dnodeIndex:%d", pSql, pVgroupInfo->numOfVgroups,
+    tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups,
-        pTableMetaInfo->dnodeIndex);
+        pTableMetaInfo->vgroupIndex);
     
-    pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId[0]);
+    pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId);
     numOfTables = 1;
   }
 
@@ -857,7 +857,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   int32_t numOfBlocks = 0;
 
   if (pQueryInfo->tsBuf != NULL) {
-    STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->dnodeIndex);
+    STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->vgroupIndex);
     assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL);  // this query should not be sent
 
     // todo refactor
@@ -1828,13 +1828,15 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
 
   pMetaMsg->sid = htonl(pMetaMsg->sid);
   pMetaMsg->sversion = htons(pMetaMsg->sversion);
-  pMetaMsg->vgId = htonl(pMetaMsg->vgId);
+  
+  pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId);
+  
   pMetaMsg->uid = htobe64(pMetaMsg->uid);
   pMetaMsg->contLen = htons(pMetaMsg->contLen);
   pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
 
-  if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) {
+  if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfIps < 0) {
-    tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid);
+    tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfIps, pMetaMsg->sid);
     return TSDB_CODE_INVALID_VALUE;
   }
 
@@ -1848,9 +1850,11 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_INVALID_VALUE;
   }
 
-  for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) {
+  for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) {
-    pMetaMsg->vpeerDesc[i].ip = htonl(pMetaMsg->vpeerDesc[i].ip);
+    pMetaMsg->vgroup.ipAddr[i].ip = htonl(pMetaMsg->vgroup.ipAddr[i].ip);
-    pMetaMsg->vpeerDesc[i].dnodeId = htonl(pMetaMsg->vpeerDesc[i].dnodeId);
+    pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port);
+    
+    assert(pMetaMsg->vgroup.ipAddr[i].ip != 0);
   }
 
   SSchema* pSchema = pMetaMsg->schema;
@@ -1859,6 +1863,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   for (int i = 0; i < numOfTotalCols; ++i) {
     pSchema->bytes = htons(pSchema->bytes);
     pSchema->colId = htons(pSchema->colId);
+    
+    assert(pSchema->type >= TSDB_DATA_TYPE_BOOL && pSchema->type <= TSDB_DATA_TYPE_NCHAR);
     pSchema++;
   }
 
@@ -1899,9 +1905,6 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
  *  |...... 1B        1B            4B
  **/
 int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
-//  uint8_t  ieType;
-//  int32_t  totalNum;
-//  int32_t  i;
 #if 0
   char *rsp = pSql->res.pRsp;
 
@@ -1958,7 +1961,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
     //      return TSDB_CODE_OTHERS;
     //    }
     //
-    //    for (int j = 0; j < TSDB_VNODES_SUPPORT; ++j) {
+    //    for (int j = 0; j < TSDB_REPLICA_MAX_NUM; ++j) {
     //      pMeta->vpeerDesc[j].vnode = htonl(pMeta->vpeerDesc[j].vnode);
     //    }
     //
@@ -2116,7 +2119,7 @@ _error_clean:
   SSqlRes* pRes = &pSql->res;
   
   SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp;
-  pStableVgroup->numOfDnodes = htonl(pStableVgroup->numOfDnodes);
+  pStableVgroup->numOfVgroups = htonl(pStableVgroup->numOfVgroups);
   
   // master sqlObj locates in param
   SSqlObj* parent = pSql->param;
@@ -2128,14 +2131,14 @@ _error_clean:
   pInfo->vgroupList = malloc(pRes->rspLen);
   memcpy(pInfo->vgroupList, pStableVgroup, pRes->rspLen);
   
-  for(int32_t i = 0; i < pInfo->vgroupList->numOfDnodes; ++i) {
+  for(int32_t i = 0; i < pInfo->vgroupList->numOfVgroups; ++i) {
-    STableDnodeVgroupInfo* pVgroups = &pInfo->vgroupList->dnodeVgroups[i];
+    SCMVgroupInfo* pVgroups = &pInfo->vgroupList->vgroups[i];
-    pVgroups->numOfVgroups = htonl(pVgroups->numOfVgroups);
+    pVgroups->numOfIps = htonl(pVgroups->numOfIps);
-    pVgroups->ipAddr.ip = htonl(pVgroups->ipAddr.ip);
+    pVgroups->vgId = htonl(pVgroups->vgId);
-    pVgroups->ipAddr.port = htons(pVgroups->ipAddr.port);
     
-    for(int32_t j = 0; j < pVgroups->numOfVgroups; ++j) {
+    for(int32_t j = 0; j < tListLen(pVgroups->ipAddr); ++j) {
-      pVgroups->vgId[j] = htonl(pVgroups->vgId[j]);
+      pVgroups->ipAddr[j].ip = htonl(pVgroups->ipAddr[j].ip);
+      pVgroups->ipAddr[j].port = htons(pVgroups->ipAddr[j].port);
     }
   }
   

src/client/src/tscSql.c

@@ -129,6 +129,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   pSql->pTscObj = pObj;
   pSql->signature = pSql;
+  pSql->maxRetry = TSDB_REPLICA_MAX_NUM;
+  
   tsem_init(&pSql->rspSem, 0, 0);
   
   pObj->pSql = pSql;

src/client/src/tscSub.c

@@ -382,7 +382,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
       pSql->cmd.command = TSDB_SQL_SELECT;
       pQueryInfo->type = type;
 
-      tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->dnodeIndex = 0;
+      tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
     }
 
     tscDoQuery(pSql);

src/client/src/tscSubquery.c

@@ -341,8 +341,8 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
   
     tscPrintSelectClause(pNew, 0);
   
-    tscTrace("%p subquery:%p tableIndex:%d, dnodeIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
+    tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
-             pSql, pNew, 0, pTableMetaInfo->dnodeIndex, pNewQueryInfo->type,
+             pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type,
              pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols,
              pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name);
   }
@@ -457,7 +457,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         assert(pQueryInfo->numOfTables == 1);  // for subquery, only one metermetaInfo
         STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-        tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->dnodeIndex);
+        tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex);
         tsBufDestory(pBuf);
       }
 
@@ -478,9 +478,9 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
         // for projection query, need to try next vnode
 //        int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
         int32_t totalVnode = 0;
-        if ((++pTableMetaInfo->dnodeIndex) < totalVnode) {
+        if ((++pTableMetaInfo->vgroupIndex) < totalVnode) {
           tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
-                   pTableMetaInfo->dnodeIndex - 1, pTableMetaInfo->dnodeIndex, totalVnode, pRes->numOfTotal);
+                   pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotal);
   
           pSql->cmd.command = TSDB_SQL_SELECT;
           pSql->fp = tscJoinQueryCallback;
@@ -542,7 +542,7 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) {
       assert(pQueryInfo->numOfTables == 1);
 
       // for projection query, need to try next vnode if current vnode is exhausted
-//      if ((++pTableMetaInfo->dnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) {
+//      if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) {
 //        pSupporter->pState->numOfCompleted = 0;
 //        pSupporter->pState->numOfTotal = 1;
 //
@@ -609,7 +609,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
 //    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
     if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-//      if (pRes->row >= pRes->numOfRows && pTableMetaInfo->dnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes &&
+//      if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vgroupIndex < pTableMetaInfo->pMetricMeta->numOfVnodes &&
 //          (!tscHasReachLimitation(pQueryInfo, pRes))) {
 //        numOfFetch++;
 //      }
@@ -647,8 +647,8 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) {
     STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
     
     if (pRes1->row >= pRes1->numOfRows) {
-      tscTrace("%p subquery:%p retrieve data from vnode, subquery:%d, dnodeIndex:%d", pSql, pSql1,
+      tscTrace("%p subquery:%p retrieve data from vnode, subquery:%d, vgroupIndex:%d", pSql, pSql1,
-               pSupporter->subqueryIndex, pTableMetaInfo->dnodeIndex);
+               pSupporter->subqueryIndex, pTableMetaInfo->vgroupIndex);
 
       tscResetForNextRetrieve(pRes1);
       pSql1->fp = joinRetrieveCallback;
@@ -785,11 +785,11 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
         STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
         /**
-         * if the query is a continue query (dnodeIndex > 0 for projection query) for next vnode, do the retrieval of
+         * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of
          * data instead of returning to its invoker
          */
-        if (pTableMetaInfo->dnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+        if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
-//          assert(pTableMetaInfo->dnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes);
+//          assert(pTableMetaInfo->vgroupIndex < pTableMetaInfo->pMetricMeta->numOfVnodes);
           pSupporter->pState->numOfCompleted = 0;  // reset the record value
 
           pSql->fp = joinRetrieveCallback;  // continue retrieve data
@@ -897,14 +897,14 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu
     
     tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, "
              "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
-             pSql, pNew, tableIndex, pTableMetaInfo->dnodeIndex, pNewQueryInfo->type,
+             pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type,
              pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols,
              pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name);
     tscPrintSelectClause(pNew, 0);
     
     tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, "
              "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s",
-             pSql, pNew, tableIndex, pTableMetaInfo->dnodeIndex, pNewQueryInfo->type,
+             pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type,
              pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols,
              pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name);
     tscPrintSelectClause(pNew, 0);
@@ -1005,7 +1005,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
-  pSql->numOfSubs = pTableMetaInfo->vgroupList->numOfDnodes;
+  pSql->numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
   assert(pSql->numOfSubs > 0);
   
   int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
@@ -1241,7 +1241,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   // data in from current vnode is stored in cache and disk
   uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
     tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql,
-        pTableMetaInfo->vgroupList->dnodeVgroups[0].ipAddr.ip, pTableMetaInfo->vgroupList->dnodeVgroups[0].vgId[0],
+        pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].ip, pTableMetaInfo->vgroupList->vgroups[0].vgId,
         numOfRowsFromSubquery, idx);
   
   tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
@@ -1401,9 +1401,9 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu
     
     assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1);
     
-    // launch subquery for each vnode, so the subquery index equals to the dnodeIndex.
+    // launch subquery for each vnode, so the subquery index equals to the vgroupIndex.
     STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index);
-    pTableMetaInfo->dnodeIndex = trsupport->subqueryIndex;
+    pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex;
     
     pSql->pSubs[trsupport->subqueryIndex] = pNew;
   }
@@ -1421,7 +1421,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
   assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1);
   
   STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
-  STableDnodeVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->dnodeVgroups[0];
+  SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0];
   
   SSubqueryState* pState = trsupport->pState;
   assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 &&
@@ -1459,7 +1459,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
       SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql);
       if (pNew == NULL) {
         tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
-                 trsupport->pParentSqlObj, pSql, pVgroupInfo->vgId[0], trsupport->subqueryIndex);
+                 trsupport->pParentSqlObj, pSql, pVgroup->vgId, trsupport->subqueryIndex);
         
         pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
         trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
@@ -1475,12 +1475,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
   
   if (pState->code != TSDB_CODE_SUCCESS) {  // at least one peer subquery failed, abort current query
     tscTrace("%p sub:%p query failed,ip:%u,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql,
-               pVgroupInfo->ipAddr.ip, pVgroupInfo->vgId[0], trsupport->subqueryIndex, pState->code);
+               pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex, pState->code);
   
     tscHandleSubqueryError(param, tres, pState->code);
   } else {  // success, proceed to retrieve data from dnode
     tscTrace("%p sub:%p query complete, ip:%u, vgId:%d, orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql,
-               pVgroupInfo->ipAddr.ip, pVgroupInfo->vgId[0], trsupport->subqueryIndex);
+               pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex);
     
     taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
   }

src/client/src/tscSystem.c

@@ -171,17 +171,49 @@ void taos_init_imp() {
   if(0 == tscEmbedded){
     taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr);      
   }
+  
   int64_t refreshTime = tsMetricMetaKeepTimer < tsMeterMetaKeepTimer ? tsMetricMetaKeepTimer : tsMeterMetaKeepTimer;
   refreshTime = refreshTime > 2 ? 2 : refreshTime;
   refreshTime = refreshTime < 1 ? 1 : refreshTime;
 
-  if (tscCacheHandle == NULL) tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
+  if (tscCacheHandle == NULL) {
+    tscCacheHandle = taosCacheInit(tscTmr, refreshTime);
+  }
 
   tscTrace("client is initialized successfully");
 }
 
 void taos_init() { pthread_once(&tscinit, taos_init_imp); }
 
+void taos_cleanup() {
+  if (tscCacheHandle != NULL) {
+    taosCacheCleanup(tscCacheHandle);
+  }
+  
+  if (tscQhandle != NULL) {
+    taosCleanUpScheduler(tscQhandle);
+    tscQhandle = NULL;
+  }
+  
+  taosCloseLogger();
+  
+  if (pVnodeConn != NULL) {
+    rpcClose(pVnodeConn);
+    pVnodeConn = NULL;
+  }
+  
+  if (pTscMgmtConn != NULL) {
+    rpcClose(pTscMgmtConn);
+    pTscMgmtConn = NULL;
+  }
+  
+  if (tsGlobalConfig != NULL) {
+    tfree(tsGlobalConfig);
+  }
+  
+  taosTmrCleanUp(tscTmr);
+}
+
 static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
   SGlobalConfig *cfg = NULL;
 

src/client/src/tscUtil.c

@@ -1937,9 +1937,9 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
   pTableMetaInfo->numOfTags = numOfTags;
   
   if (vgroupList != NULL) {
-    assert(vgroupList->numOfDnodes == 1);  // todo fix me
+    assert(vgroupList->numOfVgroups == 1);  // todo fix me
-    size_t size = sizeof(SVgroupsInfo) + (sizeof(STableDnodeVgroupInfo) +
+    
-        vgroupList->dnodeVgroups[0].numOfVgroups * sizeof(int32_t)) * vgroupList->numOfDnodes;
+    size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups;
     
     pTableMetaInfo->vgroupList = malloc(size);
     memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
@@ -2020,7 +2020,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   pNew->sqlstr = strdup(pSql->sqlstr);
   if (pNew->sqlstr == NULL) {
-    tscError("%p new subquery failed, tableIndex:%d, dnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->dnodeIndex);
+    tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
 
     free(pNew);
     return NULL;
@@ -2064,7 +2064,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {
-    tscError("%p new subquery failed, tableIndex:%d, dnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->dnodeIndex);
+    tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
     tscFreeSqlObj(pNew);
     return NULL;
   }
@@ -2155,13 +2155,13 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscTrace(
         "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d,"
         "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64,
-        pSql, pNew, tableIndex, pTableMetaInfo->dnodeIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs,
+        pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs,
         pNewQueryInfo->colList.numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime,
         pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit);
     
     tscPrintSelectClause(pNew, 0);
   } else {
-    tscTrace("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->dnodeIndex);
+    tscTrace("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex);
   }
 
   return pNew;
@@ -2258,7 +2258,7 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) {
   
 //  int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
 //  return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
-//         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->dnodeIndex < totalVnode - 1);
+//         (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < totalVnode - 1);
 }
 
 void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
@@ -2277,9 +2277,9 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
   int32_t totalVnode = 0;
 //  int32_t         totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
 
-  while (++pTableMetaInfo->dnodeIndex < totalVnode) {
+  while (++pTableMetaInfo->vgroupIndex < totalVnode) {
     tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
-             pTableMetaInfo->dnodeIndex - 1, pTableMetaInfo->dnodeIndex, totalVnode, pRes->numOfTotalInCurrentClause);
+             pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotalInCurrentClause);
 
     /*
      * update the limit and offset value for the query on the next vnode,
@@ -2298,7 +2298,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
 
     assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0));
     tscTrace("%p new query to next vnode, vnode index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, pSql,
-             pTableMetaInfo->dnodeIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
+             pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit);
 
     /*
      * For project query with super table join, the numOfSub is equalled to the number of all subqueries.

src/dnode/CMakeLists.txt

@@ -21,20 +21,13 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)
   ENDIF ()
+  
   IF (TD_GRANT)
     TARGET_LINK_LIBRARIES(taosd grant)
   ENDIF ()
 
-  IF (TD_CLUSTER)
+  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd cluster)
+    TARGET_LINK_LIBRARIES(taosd replica sync)
-  ENDIF ()
-  
-  IF (TD_VPEER)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
-  IF (TD_MPEER)
-    TARGET_LINK_LIBRARIES(taosd mpeer sync)
   ENDIF ()
 
   SET(PREPARE_ENV_CMD "prepare_env_cmd")

src/dnode/inc/dnodeMClient.h

@@ -23,7 +23,6 @@ extern "C" {
 int32_t  dnodeInitMClient();
 void     dnodeCleanupMClient();
 void     dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
-uint32_t dnodeGetMnodeMasteIp();
 void *   dnodeGetMpeerInfos();
 int32_t  dnodeGetDnodeId();
 

src/dnode/inc/dnodeModule.h

@@ -21,8 +21,8 @@ extern "C" {
 #endif
 
 int32_t dnodeInitModules();
-void dnodeCleanUpModules();
 void    dnodeStartModules();
+void    dnodeCleanUpModules();
 void    dnodeProcessModuleStatus(uint32_t moduleStatus);
 
 #ifdef __cplusplus

src/dnode/src/dnodeMClient.c

@@ -23,12 +23,12 @@
 #include "tsync.h"
 #include "ttime.h"
 #include "ttimer.h"
+#include "treplica.h"
 #include "dnode.h"
 #include "dnodeMClient.h"
 #include "dnodeModule.h"
 #include "dnodeMgmt.h"
 #include "vnode.h"
-#include "mpeer.h"
 
 #define MPEER_CONTENT_LEN 2000
 
@@ -181,7 +181,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
              tsMnodeInfos.nodeInfos[i].nodeName);
     }
     dnodeSaveMnodeIpList();
-    mpeerUpdateSync();
+    replicaNotify();
   }
 
   taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);

src/dnode/src/dnodeMain.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "tglobalcfg.h"
 #include "tlog.h"
-#include "tmodule.h"
 #include "trpc.h"
 #include "tutil.h"
 #include "dnode.h"
@@ -229,3 +228,7 @@ static int32_t dnodeInitStorage() {
 }
 
 static void dnodeCleanupStorage() {}
+
+bool  dnodeIsFirstDeploy() {
+  return strcmp(tsMasterIp, tsPrivateIp) == 0;
+}

src/dnode/src/dnodeModule.c

@@ -15,55 +15,74 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "taosdef.h"
 #include "tlog.h"
-#include "tmodule.h"
 #include "tglobalcfg.h"
 #include "mnode.h"
 #include "http.h"
 #include "monitor.h"
 #include "dnodeModule.h"
-#include "dnode.h"
+
+typedef struct {
+  bool      enable;
+  char *    name;
+  int32_t (*initFp)();
+  int32_t (*startFp)();
+  void    (*cleanUpFp)();
+  void    (*stopFp)();
+} SModule;
+
+static SModule  tsModule[TSDB_MOD_MAX] = {0};
+static uint32_t tsModuleStatus = 0;
+
+static void dnodeSetModuleStatus(int32_t module) {
+  tsModuleStatus |= (1 << module);
+}
+
+static void dnodeUnSetModuleStatus(int32_t module) {
+  tsModuleStatus &= ~(1 << module);
+}
 
 static void dnodeAllocModules() {
+  tsModule[TSDB_MOD_MGMT].name         = false;
   tsModule[TSDB_MOD_MGMT].name         = "mgmt";
   tsModule[TSDB_MOD_MGMT].initFp       = mgmtInitSystem;
   tsModule[TSDB_MOD_MGMT].cleanUpFp    = mgmtCleanUpSystem;
   tsModule[TSDB_MOD_MGMT].startFp      = mgmtStartSystem;
   tsModule[TSDB_MOD_MGMT].stopFp       = mgmtStopSystem;
-  tsModule[TSDB_MOD_MGMT].num           = tsNumOfMPeers;
-  tsModule[TSDB_MOD_MGMT].curNum        = 0;
-  tsModule[TSDB_MOD_MGMT].equalVnodeNum = tsMgmtEqualVnodeNum;
  
+  tsModule[TSDB_MOD_HTTP].enable       = (tsEnableHttpModule == 1);
   tsModule[TSDB_MOD_HTTP].name         = "http";
   tsModule[TSDB_MOD_HTTP].initFp       = httpInitSystem;
   tsModule[TSDB_MOD_HTTP].cleanUpFp    = httpCleanUpSystem;
   tsModule[TSDB_MOD_HTTP].startFp      = httpStartSystem;
   tsModule[TSDB_MOD_HTTP].stopFp       = httpStopSystem;
-  tsModule[TSDB_MOD_HTTP].num           = (tsEnableHttpModule == 1) ? -1 : 0;
+  if (tsEnableHttpModule) {
-  tsModule[TSDB_MOD_HTTP].curNum        = 0;
+    dnodeSetModuleStatus(TSDB_MOD_HTTP);
-  tsModule[TSDB_MOD_HTTP].equalVnodeNum = 0;
+  }
   
+  tsModule[TSDB_MOD_MONITOR].enable    = (tsEnableMonitorModule == 1);
   tsModule[TSDB_MOD_MONITOR].name      = "monitor";
   tsModule[TSDB_MOD_MONITOR].initFp    = monitorInitSystem;
   tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem;
   tsModule[TSDB_MOD_MONITOR].startFp   = monitorStartSystem;
   tsModule[TSDB_MOD_MONITOR].stopFp    = monitorStopSystem;
-  tsModule[TSDB_MOD_MONITOR].num           = (tsEnableMonitorModule == 1) ? -1 : 0;
+  if (tsEnableMonitorModule) {
-  tsModule[TSDB_MOD_MONITOR].curNum        = 0;
+    dnodeSetModuleStatus(TSDB_MOD_MONITOR);
-  tsModule[TSDB_MOD_MONITOR].equalVnodeNum = 0;
+  }
 }
 
 void dnodeCleanUpModules() {
-  for (int mod = 1; mod < TSDB_MOD_MAX; ++mod) {
+  for (int32_t module = 1; module < TSDB_MOD_MAX; ++module) {
-    if (tsModule[mod].num != 0 && tsModule[mod].stopFp) {
+    if (tsModule[module].enable && tsModule[module].stopFp) {
-      (*tsModule[mod].stopFp)();
+      (*tsModule[module].stopFp)();
     }
-    if (tsModule[mod].num != 0 && tsModule[mod].cleanUpFp) {
+    if (tsModule[module].cleanUpFp) {
-      (*tsModule[mod].cleanUpFp)();
+      (*tsModule[module].cleanUpFp)();
     }
   }
 
-  if (tsModule[TSDB_MOD_MGMT].num != 0 && tsModule[TSDB_MOD_MGMT].cleanUpFp) {
+  if (tsModule[TSDB_MOD_MGMT].enable && tsModule[TSDB_MOD_MGMT].cleanUpFp) {
     (*tsModule[TSDB_MOD_MGMT].cleanUpFp)();
   }
 }
@@ -71,57 +90,41 @@ void dnodeCleanUpModules() {
 int32_t dnodeInitModules() {
   dnodeAllocModules();
 
-  for (int mod = 0; mod < TSDB_MOD_MAX; ++mod) {
+  for (int32_t module = 0; module < TSDB_MOD_MAX; ++module) {
-    if (tsModule[mod].num != 0 && tsModule[mod].initFp) {
+    if (tsModule[module].initFp) {
-      if ((*tsModule[mod].initFp)() != 0) {
+      if ((*tsModule[module].initFp)() != 0) {
-        dError("failed to init modules");
+        dError("failed to init module:%s", tsModule[module].name);
         return -1;
       }
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  return 0;
 }
 
 void dnodeStartModules() {
-  // for (int mod = 1; mod < TSDB_MOD_MAX; ++mod) {
+  for (int32_t module = 1; module < TSDB_MOD_MAX; ++module) {
-  //   if (tsModule[mod].num != 0 && tsModule[mod].startFp) {
+    if (tsModule[module].enable && tsModule[module].startFp) {
-  //     if ((*tsModule[mod].startFp)() != 0) {
+      if ((*tsModule[module].startFp)() != 0) {
-  //       dError("failed to start module:%d", mod);
+        dError("failed to start module:%s", tsModule[module].name);
-  //     }
+      }
-  //   }
+    }
-  // }
+  }
 }
 
 void dnodeProcessModuleStatus(uint32_t moduleStatus) {
-  if (moduleStatus == tsModuleStatus) return;
+  bool enableMgmtModule = moduleStatus & (1 << TSDB_MOD_MGMT);
-
+  if (!tsModule[TSDB_MOD_MGMT].enable && enableMgmtModule) {
-  dPrint("module status is received, old:%d, new:%d", tsModuleStatus, moduleStatus);
+    dPrint("module status is received, start mgmt module", tsModuleStatus, moduleStatus);
-
+    tsModule[TSDB_MOD_MGMT].enable = true;
-  int news = moduleStatus;
+    dnodeSetModuleStatus(TSDB_MOD_MGMT);
-  int olds = tsModuleStatus;
+    (*tsModule[TSDB_MOD_MGMT].stopFp)();
-
-  for (int moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
-    int newStatus = news & (1 << moduleType);
-    int oldStatus = olds & (1 << moduleType);
-
-    if (oldStatus > 0) {
-      if (newStatus == 0) {
-        if (tsModule[moduleType].stopFp) {
-          dPrint("module:%s is stopped on this node", tsModule[moduleType].name);
-          (*tsModule[moduleType].stopFp)();
-        }
-      }
-    } else if (oldStatus == 0) {
-      if (newStatus > 0) {
-        if (tsModule[moduleType].startFp) {
-          dPrint("module:%s is started on this node", tsModule[moduleType].name);
-          (*tsModule[moduleType].startFp)();
-        }
-      }
-    } else {
-    }
   }
 
-  tsModuleStatus = moduleStatus;
+  if (tsModule[TSDB_MOD_MGMT].enable && !enableMgmtModule) {
+    dPrint("module status is received, stop mgmt module", tsModuleStatus, moduleStatus);
+    tsModule[TSDB_MOD_MGMT].enable = false;
+    dnodeUnSetModuleStatus(TSDB_MOD_MGMT);
+    (*tsModule[TSDB_MOD_MGMT].stopFp)();
+  }
 }

src/inc/dnode.h

@@ -20,9 +20,6 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
-#include <stdbool.h>
-
 typedef struct {
   int32_t queryReqNum;
   int32_t submitReqNum;
@@ -44,6 +41,9 @@ void *dnodeAllocateRqueue(void *pVnode);
 void  dnodeFreeRqueue(void *rqueue);
 void  dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code);
 
+bool  dnodeIsFirstDeploy();
+uint32_t dnodeGetMnodeMasteIp();
+
 #ifdef __cplusplus
 }
 #endif

src/inc/mnode.h

@@ -20,243 +20,12 @@
 extern "C" {
 #endif
 
-#include "os.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "taoserror.h"
-#include "tglobalcfg.h"
-#include "thash.h"
-#include "tidpool.h"
-#include "tlog.h"
-#include "tmempool.h"
-#include "trpc.h"
-#include "taosdef.h"
-#include "tskiplist.h"
-#include "tsocket.h"
-#include "ttime.h"
-#include "ttimer.h"
-#include "tutil.h"
-
-struct _vg_obj;
-struct _db_obj;
-struct _acct_obj;
-struct _user_obj;
-struct _mnode_obj;
-
-typedef struct _mnode_obj {
-  int32_t  mnodeId;
-  int64_t  createdTime;
-  int8_t   reserved[14];
-  int8_t   updateEnd[1];
-  int32_t  refCount;
-  uint32_t privateIp;
-  uint32_t publicIp;  
-  uint16_t port;
-  int8_t   role;
-  char     mnodeName[TSDB_NODE_NAME_LEN + 1];
-} SMnodeObj;
-
-typedef struct _dnode_obj {
-  int32_t    dnodeId;
-  uint32_t   privateIp;
-  uint32_t   publicIp;
-  uint32_t   moduleStatus;
-  int64_t    createdTime;
-  uint32_t   lastAccess;
-  int32_t    openVnodes;
-  int32_t    numOfTotalVnodes; // from dnode status msg, config information
-  uint32_t   rack;
-  uint16_t   idc;
-  uint16_t   slot;
-  uint16_t   numOfCores;       // from dnode status msg
-  int8_t     alternativeRole;  // from dnode status msg, 0-any, 1-mgmt, 2-dnode
-  int8_t     status;           // set in balance function
-  int32_t    customScore;      // config by user
-  char       dnodeName[TSDB_NODE_NAME_LEN + 1];
-  int8_t     reserved[15];
-  int8_t     updateEnd[1];
-  int32_t    refCount;
-  SVnodeLoad vload[TSDB_MAX_VNODES];
-  uint32_t   lastReboot;       // time stamp for last reboot
-  float      score;          // calc in balance function
-  float      diskAvailable;    // from dnode status msg
-  int16_t    diskAvgUsage;     // calc from sys.disk
-  int16_t    cpuAvgUsage;      // calc from sys.cpu
-  int16_t    memoryAvgUsage;   // calc from sys.mem
-  int16_t    bandwidthUsage;   // calc from sys.band
-} SDnodeObj;
-
-typedef struct {
-  int32_t  dnodeId;
-  uint32_t privateIp;
-  uint32_t publicIp;
-} SVnodeGid;
-
-typedef struct {
-  char   tableId[TSDB_TABLE_ID_LEN + 1];
-  int8_t type;
-} STableObj;
-
-typedef struct SSuperTableObj {
-  STableObj  info;
-  uint64_t   uid;
-  int64_t    createdTime;
-  int32_t    sversion;
-  int32_t    numOfColumns;
-  int32_t    numOfTags;
-  int8_t     reserved[15];
-  int8_t     updateEnd[1];
-  int32_t    refCount;
-  int32_t    numOfTables;
-  int16_t    nextColId;
-  SSchema *  schema;
-} SSuperTableObj;
-
-typedef struct {
-  STableObj  info;
-  uint64_t   uid;
-  int64_t    createdTime;
-  int32_t    sversion;     //used by normal table
-  int32_t    numOfColumns; //used by normal table
-  int32_t    sid;
-  int32_t    vgId;
-  char       superTableId[TSDB_TABLE_ID_LEN + 1];
-  int32_t    sqlLen;
-  int8_t     reserved[1]; 
-  int8_t     updateEnd[1];
-  int16_t    nextColId;    //used by normal table
-  int32_t    refCount;
-  char*      sql;          //used by normal table
-  SSchema*   schema;       //used by normal table
-  SSuperTableObj *superTable;
-} SChildTableObj;
-
-typedef struct _vg_obj {
-  uint32_t        vgId;
-  char            dbName[TSDB_DB_NAME_LEN + 1];
-  int64_t         createdTime;
-  SVnodeGid       vnodeGid[TSDB_VNODES_SUPPORT];
-  int32_t         numOfVnodes;
-  int32_t         lbDnodeId;
-  int32_t         lbTime;
-  int8_t          status;
-  int8_t          inUse;
-  int8_t          reserved[13];
-  int8_t          updateEnd[1];
-  int32_t         refCount;
-  struct _vg_obj *prev, *next;
-  struct _db_obj *pDb;
-  int32_t         numOfTables;
-  void *          idPool;
-  SChildTableObj ** tableList;
-} SVgObj;
-
-typedef struct _db_obj {
-  char    name[TSDB_DB_NAME_LEN + 1];
-  int8_t  status;
-  int64_t createdTime;
-  SDbCfg  cfg;
-  int8_t  reserved[15];
-  int8_t  updateEnd[1];
-  int32_t refCount;
-  int32_t numOfVgroups;
-  int32_t numOfTables;
-  int32_t numOfSuperTables;
-  SVgObj *pHead;
-  SVgObj *pTail;
-  struct _acct_obj *pAcct;
-} SDbObj;
-
-typedef struct _user_obj {
-  char              user[TSDB_USER_LEN + 1];
-  char              pass[TSDB_KEY_LEN + 1];
-  char              acct[TSDB_USER_LEN + 1];
-  int64_t           createdTime;
-  int8_t            superAuth;
-  int8_t            writeAuth;
-  int8_t            reserved[13];
-  int8_t            updateEnd[1];
-  int32_t           refCount;
-  struct _acct_obj * pAcct;
-  SQqueryList *     pQList;  // query list
-  SStreamList *     pSList;  // stream list
-} SUserObj;
-
-typedef struct {
-  int32_t numOfUsers;
-  int32_t numOfDbs;
-  int32_t numOfTimeSeries;
-  int32_t numOfPointsPerSecond;
-  int32_t numOfConns;
-  int32_t numOfQueries;
-  int32_t numOfStreams;
-  int64_t totalStorage;  // Total storage wrtten from this account
-  int64_t compStorage;   // Compressed storage on disk
-  int64_t queryTime;
-  int64_t totalPoints;
-  int64_t inblound;
-  int64_t outbound;
-  int64_t sKey;
-  int8_t  accessState;   // Checked by mgmt heartbeat message
-} SAcctInfo;
-
-typedef struct _acct_obj {
-  char      user[TSDB_USER_LEN + 1];
-  char      pass[TSDB_KEY_LEN + 1];
-  SAcctCfg  cfg;
-  int32_t   acctId;
-  int64_t   createdTime;
-  int8_t    dirty;
-  int8_t    reserved[14];
-  int8_t    updateEnd[1];
-  int32_t  refCount;
-  SAcctInfo acctInfo;
-  pthread_mutex_t  mutex;
-} SAcctObj;
-
-typedef struct {
-  int8_t   type;
-  char     db[TSDB_DB_NAME_LEN + 1];
-  void *   pNode;
-  int16_t  numOfColumns;
-  int32_t  rowSize;
-  int32_t  numOfRows;
-  int32_t  numOfReads;
-  int16_t  offset[TSDB_MAX_COLUMNS];
-  int16_t  bytes[TSDB_MAX_COLUMNS];
-  void *   signature;
-  uint16_t payloadLen;
-  char     payload[];
-} SShowObj;
-
-typedef struct {
-  uint8_t  msgType;
-  int8_t   usePublicIp;
-  int8_t   received;
-  int8_t   successed;
-  int8_t   expected;
-  int8_t   retry;
-  int8_t   maxRetry;
-  int32_t  contLen;
-  int32_t  code;
-  void     *ahandle;
-  void     *thandle;
-  void     *pCont;
-  SAcctObj *pAcct;
-  SDnodeObj*pDnode;
-  SUserObj *pUser;
-  SDbObj   *pDb;
-  SVgObj   *pVgroup;
-  STableObj *pTable;
-} SQueuedMsg;
-
 int32_t mgmtInitSystem();
 int32_t mgmtStartSystem();
 void    mgmtCleanUpSystem();
 void    mgmtStopSystem();
 
 extern char  version[];
-extern void *tsMgmtTmr;
 extern char  tsMnodeDir[];
 
 #ifdef __cplusplus

src/inc/mpeer.h

@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_MPEER_H
-#define TDENGINE_MPEER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct _mnode_obj;
-
-enum _TAOS_MN_STATUS {
-  TAOS_MN_STATUS_OFFLINE,
-  TAOS_MN_STATUS_DROPPING,
-  TAOS_MN_STATUS_READY
-};
-
-// general implementation
-int32_t mpeerInit();
-void    mpeerCleanup();
-
-// special implementation
-int32_t mpeerInitMnodes();
-void    mpeerCleanupMnodes();
-int32_t mpeerAddMnode(int32_t dnodeId);
-int32_t mpeerRemoveMnode(int32_t dnodeId);
-
-void *  mpeerGetMnode(int32_t mnodeId);
-int32_t mpeerGetMnodesNum();
-void *  mpeerGetNextMnode(void *pNode, struct _mnode_obj **pMnode);
-void    mpeerReleaseMnode(struct _mnode_obj *pMnode);
-
-bool    mpeerIsMaster();
-
-void    mpeerGetPrivateIpList(SRpcIpSet *ipSet);
-void    mpeerGetPublicIpList(SRpcIpSet *ipSet);
-void    mpeerGetMpeerInfos(void *mpeers);
-
-int32_t mpeerForwardReqToPeer(void *pHead);
-void    mpeerUpdateSync();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc/tacct.h

@@ -13,8 +13,25 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
+#ifndef TDENGINE_ACCT_H
-#include "tmodule.h"
+#define TDENGINE_ACCT_H
 
-SModule  tsModule[TSDB_MOD_MAX] = {0};
+#ifdef __cplusplus
-uint32_t tsModuleStatus         = 0;
+extern "C" {
+#endif
+
+typedef enum {
+  ACCT_GRANT_USER,
+  ACCT_GRANT_DB,
+  ACCT_GRANT_TABLE
+} EAcctGrantType;
+
+int32_t acctInit();
+void    acctCleanUp();
+int32_t acctCheck(void *pAcct, EAcctGrantType type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/taos.h

@@ -52,6 +52,7 @@ typedef struct taosField {
 #endif
 
 DLL_EXPORT void  taos_init();
+DLL_EXPORT void  taos_cleanup();
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);

src/inc/taosdef.h

@@ -319,11 +319,11 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_NORMAL_TABLES 1000
 #define TSDB_MAX_CHILD_TABLES  100000
 
-enum {
+typedef enum {
   TSDB_PRECISION_MILLI,
   TSDB_PRECISION_MICRO,
   TSDB_PRECISION_NANO
-};
+} EPrecisionType;
 
 typedef enum {
   TSDB_SUPER_TABLE        = 0,  // super table
@@ -331,7 +331,14 @@ typedef enum {
   TSDB_NORMAL_TABLE       = 2,  // ordinary table
   TSDB_STREAM_TABLE       = 3,  // table created from stream computing
   TSDB_TABLE_MAX          = 4
-} TSDB_TABLE_TYPE;
+} ETableType;
+
+typedef enum {
+  TSDB_MOD_MGMT,
+  TSDB_MOD_HTTP,
+  TSDB_MOD_MONITOR,
+  TSDB_MOD_MAX
+} EModuleType;
 
 #ifdef __cplusplus
 }

src/inc/taosmsg.h

@@ -634,14 +634,14 @@ typedef struct SCMSTableVgroupMsg {
 } SCMSTableVgroupMsg;
 
 typedef struct {
-  SIpAddr   ipAddr;
+  int32_t   vgId;
-  int32_t   numOfVgroups;
+  int8_t    numOfIps;
-  int32_t   vgId[];
+  SIpAddr   ipAddr[TSDB_REPLICA_MAX_NUM];
-} STableDnodeVgroupInfo;
+} SCMVgroupInfo;
 
 typedef struct {
-  int32_t  numOfDnodes;
+  int32_t  numOfVgroups;
-  STableDnodeVgroupInfo dnodeVgroups[];
+  SCMVgroupInfo vgroups[];
 } SCMSTableVgroupRspMsg;
 
 typedef struct {
@@ -674,7 +674,7 @@ typedef struct {
 } SSuperTableMetaMsg;
 
 typedef struct {
-  SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
+  SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM];
   int16_t    index;  // used locally
   int32_t    numOfSids;
   int32_t    pSidExtInfoList[];  // offset value of STableIdInfo
@@ -689,7 +689,6 @@ typedef struct {
 
 typedef struct STableMetaMsg {
   int32_t       contLen;
-  
   char          tableId[TSDB_TABLE_ID_LEN];   // table id
   char          stableId[TSDB_TABLE_ID_LEN];  // stable name if it is created according to super table
   uint8_t       numOfTags;
@@ -697,11 +696,9 @@ typedef struct STableMetaMsg {
   uint8_t       tableType;
   int16_t       numOfColumns;
   int16_t       sversion;
-  int8_t     numOfVpeers;
-  SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT];
   int32_t       sid;
-  int32_t    vgId;
   uint64_t      uid;
+  SCMVgroupInfo vgroup;
   SSchema       schema[];
 } STableMetaMsg;
 

src/inc/treplica.h

@@ -13,27 +13,23 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_BALANCE_H
+#ifndef TDENGINE_REPLICA_H
-#define TDENGINE_BALANCE_H
+#define TDENGINE_REPLICA_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdint.h>
+struct SVgObj;
-#include <stdbool.h>
+struct SDnodeObj;
-#include <pthread.h>
 
-struct _db_obj;
+int32_t replicaInit();
-struct _vg_obj;
+void    replicaCleanUp();
-struct _dnode_obj;
+void    replicaNotify();
-
+void    replicaReset();
-int32_t balanceInit();
+int32_t replicaAllocVnodes(struct SVgObj *pVgroup);
-void    balanceCleanUp();
+int32_t replicaForwardReqToPeer(void *pHead);
-void    balanceNotify();
+int32_t replicaDropDnode(struct SDnodeObj *pDnode);
-void    balanceReset();
-int32_t balanceAllocVnodes(struct _vg_obj *pVgroup);
-int32_t balanceDropDnode(struct _dnode_obj *pDnode);
 
 #ifdef __cplusplus
 }

src/inc/tsdb.h

@@ -79,7 +79,7 @@ typedef struct {
 
 // --------- TSDB TABLE configuration
 typedef struct {
-  TSDB_TABLE_TYPE type;
+  ETableType      type;
   STableId        tableId;
   int32_t         sversion;
   int64_t         superUid;
@@ -88,7 +88,7 @@ typedef struct {
   SDataRow        tagValues;
 } STableCfg;
 
-int  tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid);
+int  tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid);
 int  tsdbTableSetSuperUid(STableCfg *config, int64_t uid);
 int  tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
 int  tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);

src/kit/shell/inc/shell.h

@@ -71,7 +71,7 @@ extern void* shellLoopQuery(void* arg);
 extern void taos_error(TAOS* con);
 extern int regex_match(const char* s, const char* reg, int cflags);
 void shellReadCommand(TAOS* con, char command[]);
-void shellRunCommand(TAOS* con, char* command);
+int32_t shellRunCommand(TAOS* con, char* command);
 void shellRunCommandOnServer(TAOS* con, char command[]);
 void read_history();
 void write_history();

src/kit/shell/src/shellEngine.c

@@ -166,10 +166,10 @@ void shellReplaceCtrlChar(char *str) {
   *pstr = '\0';
 }
 
-void shellRunCommand(TAOS *con, char *command) {
+int32_t shellRunCommand(TAOS *con, char *command) {
   /* If command is empty just return */
   if (regex_match(command, "^[ \t;]*$", REG_EXTENDED)) {
-    return;
+    return 0;
   }
 
   /* Update the history vector. */
@@ -193,11 +193,11 @@ void shellRunCommand(TAOS *con, char *command) {
   if (regex_match(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
     taos_close(con);
     write_history();
-    exitShell();
+    return -1;
   } else if (regex_match(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
     // If clear the screen.
     system("clear");
-    return;
+    return 0;
   } else if (regex_match(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
     /* If source file. */
     char *c_ptr = strtok(command, " ;");
@@ -209,6 +209,8 @@ void shellRunCommand(TAOS *con, char *command) {
   } else {
     shellRunCommandOnServer(con, command);
   }
+  
+  return 0;
 }
 
 void shellRunCommandOnServer(TAOS *con, char command[]) {

src/kit/shell/src/shellLinux.c

@@ -295,6 +295,7 @@ void *shellLoopQuery(void *arg) {
     tscError("failed to malloc command");
     return NULL;
   }
+  
   while (1) {
     // Read command from shell.
 
@@ -304,8 +305,13 @@ void *shellLoopQuery(void *arg) {
     reset_terminal_mode();
 
     // Run the command
-    shellRunCommand(con, command);
+    if (shellRunCommand(con, command) != 0) {
+      break;
     }
+  }
+  
+  tfree(command);
+  exitShell();
 
   pthread_cleanup_pop(1);
   
@@ -487,6 +493,7 @@ void showOnScreen(Command *cmd) {
 void cleanup_handler(void *arg) { tcsetattr(0, TCSANOW, &oldtio); }
 
 void exitShell() {
-  tcsetattr(0, TCSANOW, &oldtio);
+  /*int32_t ret =*/ tcsetattr(STDIN_FILENO, TCSANOW, &oldtio);
+  taos_cleanup();
   exit(EXIT_SUCCESS);
 }

src/kit/shell/src/shellMain.c

@@ -95,7 +95,7 @@ int main(int argc, char* argv[]) {
     exit(EXIT_FAILURE);
   }
 
-  /* Interupt handler. */
+  /* Interrupt handler. */
   struct sigaction act;
   memset(&act, 0, sizeof(struct sigaction));
   

src/mnode/inc/mgmtAcct.h

@@ -13,42 +13,25 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TMODULE_H
+#ifndef TDENGINE_MGMT_ACCT_H
-#define TDENGINE_TMODULE_H
+#define TDENGINE_MGMT_ACCT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdint.h>
+#include "tacct.h"
-#include <stdbool.h>
-#include <pthread.h>
 
-enum _module {
+int32_t mgmtInitAccts();
-  TSDB_MOD_MGMT,
+void    mgmtCleanUpAccts();
-  TSDB_MOD_HTTP,
+void   *mgmtGetAcct(char *acctName);
-  TSDB_MOD_MONITOR,
+void    mgmtIncAcctRef(SAcctObj *pAcct);
-  TSDB_MOD_MAX
+void    mgmtDecAcctRef(SAcctObj *pAcct);
-};
 
-#define tsetModuleStatus(mod) \
+void    mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);
-  { tsModuleStatus |= (1 << mod); }
+void    mgmtDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb);
-#define tclearModuleStatus(mod) \
+void    mgmtAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser);
-  { tsModuleStatus &= ~(1 << mod); }
+void    mgmtDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser);
-
-typedef struct {
-  char  *name;
-  int  (*initFp)();
-  void (*cleanUpFp)();
-  int  (*startFp)();
-  void (*stopFp)();
-  int    num;
-  int    curNum;
-  int    equalVnodeNum;
-} SModule;
-
-extern uint32_t tsModuleStatus;
-extern SModule tsModule[];
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtDb.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#include "mnode.h"
+#include "mgmtDef.h"
 
 enum _TSDB_DB_STATUS {
   TSDB_DB_STATUS_READY,
@@ -33,7 +33,7 @@ void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
 void    mgmtIncDbRef(SDbObj *pDb);
-void    mgmtReleaseDb(SDbObj *pDb);
+void    mgmtDecDbRef(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);
 void    mgmtDropAllDbs(SAcctObj *pAcct);
 

src/mnode/inc/mgmtDef.h

@@ -0,0 +1,246 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MGMT_DEF_H
+#define TDENGINE_MGMT_DEF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosdef.h"
+#include "taosmsg.h"
+
+struct SVgObj;
+struct SDbObj;
+struct SAcctObj;
+struct SUserObj;
+struct SMnodeObj;
+
+typedef struct SDnodeObj {
+  int32_t    dnodeId;
+  uint32_t   privateIp;
+  uint32_t   publicIp;
+  uint16_t   mnodeShellPort;
+  uint16_t   mnodeDnodePort;
+  uint16_t   dnodeShellPort;
+  uint16_t   dnodeMnodePort;
+  uint16_t   syncPort;
+  int64_t    createdTime;
+  uint32_t   lastAccess;
+  int32_t    openVnodes;
+  int32_t    totalVnodes;      // from dnode status msg, config information
+  int32_t    customScore;      // config by user
+  uint16_t   numOfCores;       // from dnode status msg
+  int8_t     alternativeRole;  // from dnode status msg, 0-any, 1-mgmt, 2-dnode
+  int8_t     status;           // set in balance function
+  int8_t     isMgmt;
+  char       dnodeName[TSDB_NODE_NAME_LEN + 1];
+  int8_t     reserved[15];
+  int8_t     updateEnd[1];
+  int32_t    refCount;
+  SVnodeLoad vload[TSDB_MAX_VNODES];
+  uint32_t   moduleStatus;
+  uint32_t   lastReboot;       // time stamp for last reboot
+  float      score;          // calc in balance function
+  float      diskAvailable;    // from dnode status msg
+  int16_t    diskAvgUsage;     // calc from sys.disk
+  int16_t    cpuAvgUsage;      // calc from sys.cpu
+  int16_t    memoryAvgUsage;   // calc from sys.mem
+  int16_t    bandwidthUsage;   // calc from sys.band
+} SDnodeObj;
+
+typedef struct SMnodeObj {
+  int32_t    mnodeId;
+  int64_t    createdTime;
+  int8_t     reserved[14];
+  int8_t     updateEnd[1];
+  int32_t    refCount;
+  int8_t     role;
+  SDnodeObj *pDnode;
+} SMnodeObj;
+
+
+typedef struct {
+  int32_t  dnodeId;
+  uint32_t privateIp;
+  uint32_t publicIp;
+} SVnodeGid;
+
+typedef struct {
+  char   tableId[TSDB_TABLE_ID_LEN + 1];
+  int8_t type;
+} STableObj;
+
+typedef struct SSuperTableObj {
+  STableObj  info;
+  uint64_t   uid;
+  int64_t    createdTime;
+  int32_t    sversion;
+  int32_t    numOfColumns;
+  int32_t    numOfTags;
+  int8_t     reserved[15];
+  int8_t     updateEnd[1];
+  int32_t    refCount;
+  int32_t    numOfTables;
+  int16_t    nextColId;
+  SSchema *  schema;
+  int32_t    vgLen;
+  int32_t *  vgList;
+} SSuperTableObj;
+
+typedef struct {
+  STableObj  info;
+  uint64_t   uid;
+  int64_t    createdTime;
+  int32_t    sversion;     //used by normal table
+  int32_t    numOfColumns; //used by normal table
+  int32_t    sid;
+  int32_t    vgId;
+  char       superTableId[TSDB_TABLE_ID_LEN + 1];
+  int32_t    sqlLen;
+  int8_t     reserved[1]; 
+  int8_t     updateEnd[1];
+  int16_t    nextColId;    //used by normal table
+  int32_t    refCount;
+  char*      sql;          //used by normal table
+  SSchema*   schema;       //used by normal table
+  SSuperTableObj *superTable;
+} SChildTableObj;
+
+typedef struct SVgObj {
+  uint32_t        vgId;
+  char            dbName[TSDB_DB_NAME_LEN + 1];
+  int64_t         createdTime;
+  SVnodeGid       vnodeGid[TSDB_VNODES_SUPPORT];
+  int32_t         numOfVnodes;
+  int32_t         lbDnodeId;
+  int32_t         lbTime;
+  int8_t          status;
+  int8_t          inUse;
+  int8_t          reserved[13];
+  int8_t          updateEnd[1];
+  int32_t         refCount;
+  struct SVgObj *prev, *next;
+  struct SDbObj *pDb;
+  int32_t         numOfTables;
+  void *          idPool;
+  SChildTableObj ** tableList;
+} SVgObj;
+
+typedef struct SDbObj {
+  char    name[TSDB_DB_NAME_LEN + 1];
+  int8_t  status;
+  int64_t createdTime;
+  SDbCfg  cfg;
+  int8_t  reserved[15];
+  int8_t  updateEnd[1];
+  int32_t refCount;
+  int32_t numOfVgroups;
+  int32_t numOfTables;
+  int32_t numOfSuperTables;
+  SVgObj *pHead;
+  SVgObj *pTail;
+  struct SAcctObj *pAcct;
+} SDbObj;
+
+typedef struct SUserObj {
+  char              user[TSDB_USER_LEN + 1];
+  char              pass[TSDB_KEY_LEN + 1];
+  char              acct[TSDB_USER_LEN + 1];
+  int64_t           createdTime;
+  int8_t            superAuth;
+  int8_t            writeAuth;
+  int8_t            reserved[13];
+  int8_t            updateEnd[1];
+  int32_t           refCount;
+  struct SAcctObj * pAcct;
+  SQqueryList *     pQList;  // query list
+  SStreamList *     pSList;  // stream list
+} SUserObj;
+
+typedef struct {
+  int32_t numOfUsers;
+  int32_t numOfDbs;
+  int32_t numOfTimeSeries;
+  int32_t numOfPointsPerSecond;
+  int32_t numOfConns;
+  int32_t numOfQueries;
+  int32_t numOfStreams;
+  int64_t totalStorage;  // Total storage wrtten from this account
+  int64_t compStorage;   // Compressed storage on disk
+  int64_t queryTime;
+  int64_t totalPoints;
+  int64_t inblound;
+  int64_t outbound;
+  int64_t sKey;
+  int8_t  accessState;   // Checked by mgmt heartbeat message
+} SAcctInfo;
+
+typedef struct SAcctObj {
+  char      user[TSDB_USER_LEN + 1];
+  char      pass[TSDB_KEY_LEN + 1];
+  SAcctCfg  cfg;
+  int32_t   acctId;
+  int64_t   createdTime;
+  int8_t    status;
+  int8_t    reserved[14];
+  int8_t    updateEnd[1];
+  int32_t   refCount;
+  SAcctInfo acctInfo;
+  pthread_mutex_t  mutex;
+} SAcctObj;
+
+typedef struct {
+  int8_t   type;
+  char     db[TSDB_DB_NAME_LEN + 1];
+  void *   pNode;
+  int16_t  numOfColumns;
+  int32_t  rowSize;
+  int32_t  numOfRows;
+  int32_t  numOfReads;
+  int16_t  offset[TSDB_MAX_COLUMNS];
+  int16_t  bytes[TSDB_MAX_COLUMNS];
+  void *   signature;
+  uint16_t payloadLen;
+  char     payload[];
+} SShowObj;
+
+typedef struct {
+  uint8_t  msgType;
+  int8_t   usePublicIp;
+  int8_t   received;
+  int8_t   successed;
+  int8_t   expected;
+  int8_t   retry;
+  int8_t   maxRetry;
+  int32_t  contLen;
+  int32_t  code;
+  void     *ahandle;
+  void     *thandle;
+  void     *pCont;
+  SAcctObj *pAcct;
+  SDnodeObj*pDnode;
+  SUserObj *pUser;
+  SDbObj   *pDb;
+  SVgObj   *pVgroup;
+  STableObj *pTable;
+} SQueuedMsg;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mgmtDnode.h

@@ -20,34 +20,27 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
+typedef enum {
-#include <stdbool.h>
-#include <pthread.h>
-
-struct _dnode_obj;
-
-enum _TAOS_DN_STATUS {
   TAOS_DN_STATUS_OFFLINE,
   TAOS_DN_STATUS_DROPPING,
   TAOS_DN_STATUS_BALANCING,
   TAOS_DN_STATUS_READY
-};
+} EDnodeStatus;
 
-int32_t clusterInit();
+int32_t mgmtInitDnodes();
-void    clusterCleanUp();
+void    mgmtCleanupDnodes();
-char*   clusterGetDnodeStatusStr(int32_t dnodeStatus);
-bool    clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType);
-void    clusterMonitorDnodeModule();
 
-int32_t clusterInitDnodes();
+char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
-void    clusterCleanupDnodes();
+bool    mgmtCheckModuleInDnode(SDnodeObj *pDnode, int moduleType);
-int32_t clusterGetDnodesNum();
+void    mgmtMonitorDnodeModule();
-void *  clusterGetNextDnode(void *pNode, struct _dnode_obj **pDnode);
+
-void    clusterReleaseDnode(struct _dnode_obj *pDnode);
+int32_t mgmtGetDnodesNum();
-void *  clusterGetDnode(int32_t dnodeId);
+void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
-void *  clusterGetDnodeByIp(uint32_t ip);
+void    mgmtReleaseDnode(SDnodeObj *pDnode);
-void    clusterUpdateDnode(struct _dnode_obj *pDnode);
+void *  mgmtGetDnode(int32_t dnodeId);
-int32_t clusterDropDnode(struct _dnode_obj *pDnode);
+void *  mgmtGetDnodeByIp(uint32_t ip);
+void    mgmtUpdateDnode(SDnodeObj *pDnode);
+int32_t mgmtDropDnode(SDnodeObj *pDnode);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtLog.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MGMT_LOG_H
+#define TDENGINE_MGMT_LOG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlog.h"
+
+// mnode log function
+#define mError(...)                          \
+  if (mdebugFlag & DEBUG_ERROR) {            \
+    tprintf("ERROR MND ", 255, __VA_ARGS__); \
+  }
+#define mWarn(...)                                  \
+  if (mdebugFlag & DEBUG_WARN) {                    \
+    tprintf("WARN  MND ", mdebugFlag, __VA_ARGS__); \
+  }
+#define mTrace(...)                           \
+  if (mdebugFlag & DEBUG_TRACE) {             \
+    tprintf("MND ", mdebugFlag, __VA_ARGS__); \
+  }
+#define mPrint(...) \
+  { tprintf("MND ", 255, __VA_ARGS__); }
+
+#define mLError(...) taosLogError(__VA_ARGS__) mError(__VA_ARGS__)
+#define mLWarn(...) taosLogWarn(__VA_ARGS__) mWarn(__VA_ARGS__)
+#define mLPrint(...) taosLogPrint(__VA_ARGS__) mPrint(__VA_ARGS__)
+
+#define sdbError(...)                            \
+  if (sdbDebugFlag & DEBUG_ERROR) {              \
+    tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \
+  }
+#define sdbWarn(...)                                      \
+  if (sdbDebugFlag & DEBUG_WARN) {                        \
+    tprintf("WARN  MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
+  }
+#define sdbTrace(...)                               \
+  if (sdbDebugFlag & DEBUG_TRACE) {                 \
+    tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
+  }
+#define sdbPrint(...) \
+  { tprintf("MND-SDB ", 255, __VA_ARGS__); }
+
+#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__)
+#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__)
+#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mgmtMnode.h

@@ -13,34 +13,36 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_ACCT_H
+#ifndef TDENGINE_MGMT_MNODE_H
-#define TDENGINE_ACCT_H
+#define TDENGINE_MGMT_MNODE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-struct _acct_obj;
+struct SMnodeObj;
-struct _user_obj;
-struct _db_obj;
 
 typedef enum {
-  TSDB_ACCT_USER,
+  TAOS_MN_STATUS_OFFLINE,
-  TSDB_ACCT_DB,
+  TAOS_MN_STATUS_DROPPING,
-  TSDB_ACCT_TABLE
+  TAOS_MN_STATUS_READY
-} EAcctGrantType;
+} EMnodeStatus;
 
-int32_t acctInit();
+int32_t mgmtInitMnodes();
-void    acctCleanUp();
+void    mgmtCleanupMnodes();
-void   *acctGetAcct(char *acctName);
-void    acctIncRef(struct _acct_obj *pAcct);
-void    acctReleaseAcct(struct _acct_obj *pAcct);
-int32_t acctCheck(struct _acct_obj *pAcct, EAcctGrantType type);
 
-void    acctAddDb(struct _acct_obj *pAcct, struct _db_obj *pDb);
+int32_t mgmtAddMnode(int32_t dnodeId);
-void    acctRemoveDb(struct _acct_obj *pAcct, struct _db_obj *pDb);
+int32_t mgmtDropMnode(int32_t dnodeId);
-void    acctAddUser(struct _acct_obj *pAcct, struct _user_obj *pUser);
+
-void    acctRemoveUser(struct _acct_obj *pAcct, struct _user_obj *pUser);
+void *  mgmtGetMnode(int32_t mnodeId);
+int32_t mgmtGetMnodesNum();
+void *  mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
+void    mgmtReleaseMnode(struct SMnodeObj *pMnode);
+
+bool    mgmtIsMaster();
+
+void    mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp);
+void    mgmtGetMnodeList(void *mpeers);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtProfile.h

@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
+#include "mgmtDef.h"
 
 int32_t mgmtInitProfile();
 void    mgmtCleanUpProfile();

src/mnode/inc/mgmtShell.h

@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
+#include "mgmtDef.h"
 
 int32_t mgmtInitShell();
 void    mgmtCleanUpShell();

src/mnode/inc/mgmtTable.h

@@ -20,10 +20,7 @@
 extern "C" {
 #endif
 
-#include <stdio.h>
+#include "mgmtDef.h"
-#include <stdlib.h>
-#include <stdint.h>
-#include "mnode.h"
 
 int32_t     mgmtInitTables();
 void        mgmtCleanUpTables();

src/mnode/inc/mgmtUser.h

@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
+#include "mgmtDef.h"
 
 int32_t   mgmtInitUsers();
 void      mgmtCleanUpUsers();

src/mnode/inc/mgmtVgroup.h

@@ -20,9 +20,7 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
+#include "mgmtDef.h"
-#include <stdbool.h>
-#include "mnode.h"
 
 enum _TSDB_VG_STATUS {
   TSDB_VG_STATUS_READY,

src/mnode/src/mgmtAcct.c

@@ -16,49 +16,181 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
-#include "mnode.h"
+#include "ttime.h"
-#include "taccount.h"
+#include "tutil.h"
+#include "dnode.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
 #include "mgmtDb.h"
+#include "mgmtSdb.h"
 #include "mgmtUser.h"
 
-#ifndef _ACCOUNT
+static void *  tsAcctSdb = NULL;
+static int32_t tsAcctUpdateSize;
+static void    mgmtCreateRootAcct();
 
-static SAcctObj tsAcctObj = {0};
+static int32_t mgmtActionAcctDestroy(SSdbOperDesc *pOper) {
-
+  SAcctObj *pAcct = pOper->pObj;
-int32_t acctInit() {
+  pthread_mutex_destroy(&pAcct->mutex);
-  tsAcctObj.acctId = 0;
+  tfree(pOper->pObj);
-  strcpy(tsAcctObj.user, "root");
   return TSDB_CODE_SUCCESS;
 }
 
-void    acctCleanUp() {}
+static int32_t mgmtAcctActionInsert(SSdbOperDesc *pOper) {
-void   *acctGetAcct(char *acctName) { return &tsAcctObj; }
+  SAcctObj *pAcct = pOper->pObj;
-void    acctIncRef(struct _acct_obj *pAcct) {}
+  memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
-void    acctReleaseAcct(SAcctObj *pAcct) {}
+  pthread_mutex_init(&pAcct->mutex, NULL);
-int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
+  return TSDB_CODE_SUCCESS;
+}
 
-#endif
+static int32_t mgmtActionAcctDelete(SSdbOperDesc *pOper) {
+  SAcctObj *pAcct = pOper->pObj;
+  mgmtDropAllUsers(pAcct);
+  mgmtDropAllDbs(pAcct);
+  return TSDB_CODE_SUCCESS;
+}
 
-void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) {
+static int32_t mgmtActionAcctUpdate(SSdbOperDesc *pOper) {
+  SAcctObj *pAcct = pOper->pObj;
+  SAcctObj *pSaved = mgmtGetAcct(pAcct->user);
+  if (pAcct != pSaved) {
+    memcpy(pSaved, pAcct, tsAcctUpdateSize);
+    free(pAcct);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtActionActionEncode(SSdbOperDesc *pOper) {
+  SAcctObj *pAcct = pOper->pObj;
+  memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
+  pOper->rowSize = tsAcctUpdateSize;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtActionAcctDecode(SSdbOperDesc *pOper) {
+  SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj));
+  if (pAcct == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  memcpy(pAcct, pOper->rowData, tsAcctUpdateSize);
+  pOper->pObj = pAcct;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtActionAcctRestored() {
+  if (dnodeIsFirstDeploy()) {
+    mgmtCreateRootAcct();
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mgmtInitAccts() {
+  SAcctObj tObj;
+  tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+
+  SSdbTableDesc tableDesc = {
+    .tableId      = SDB_TABLE_ACCOUNT,
+    .tableName    = "accounts",
+    .hashSessions = TSDB_MAX_ACCOUNTS,
+    .maxRowSize   = tsAcctUpdateSize,
+    .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
+    .keyType      = SDB_KEY_STRING,
+    .insertFp     = mgmtAcctActionInsert,
+    .deleteFp     = mgmtActionAcctDelete,
+    .updateFp     = mgmtActionAcctUpdate,
+    .encodeFp     = mgmtActionActionEncode,
+    .decodeFp     = mgmtActionAcctDecode,
+    .destroyFp    = mgmtActionAcctDestroy,
+    .restoredFp   = mgmtActionAcctRestored
+  };
+
+  tsAcctSdb = sdbOpenTable(&tableDesc);
+  if (tsAcctSdb == NULL) {
+    mError("failed to init acct data");
+    return -1;
+  }
+
+  mTrace("account table is created");
+  return acctInit();
+}
+
+void mgmtCleanUpAccts() {
+  sdbCloseTable(tsAcctSdb);
+  acctCleanUp();
+}
+
+void *mgmtGetAcct(char *name) {
+  return sdbGetRow(tsAcctSdb, name);
+}
+
+void mgmtIncAcctRef(SAcctObj *pAcct) {
+  sdbIncRef(tsAcctSdb, pAcct);
+}
+
+void mgmtDecAcctRef(SAcctObj *pAcct) {
+  sdbDecRef(tsAcctSdb, pAcct);
+}
+
+void mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb) {
   atomic_add_fetch_32(&pAcct->acctInfo.numOfDbs, 1);
   pDb->pAcct = pAcct;
-  acctIncRef(pAcct);
+  mgmtIncAcctRef(pAcct);
 }
 
-void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) {
+void mgmtDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfDbs, 1);
   pDb->pAcct = NULL;
-  acctReleaseAcct(pAcct);
+  mgmtDecAcctRef(pAcct);
 }
 
-void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) {
+void mgmtAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser) {
   atomic_add_fetch_32(&pAcct->acctInfo.numOfUsers, 1);
   pUser->pAcct = pAcct;
-  acctIncRef(pAcct);
+  mgmtIncAcctRef(pAcct);
 }
 
-void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) {
+void mgmtDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfUsers, 1);
   pUser->pAcct = NULL;
-  acctReleaseAcct(pAcct);
+  mgmtDecAcctRef(pAcct);
 }
+
+static void mgmtCreateRootAcct() {
+  int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb);
+  if (numOfAccts != 0) return;
+
+  SAcctObj *pAcct = malloc(sizeof(SAcctObj));
+  memset(pAcct, 0, sizeof(SAcctObj));
+  strcpy(pAcct->user, "root");
+  taosEncryptPass((uint8_t*)"taosdata", strlen("taosdata"), pAcct->pass);
+  pAcct->cfg = (SAcctCfg){
+    .maxUsers           = 10,
+    .maxDbs             = 64,
+    .maxTimeSeries      = INT32_MAX,
+    .maxConnections     = 1024,
+    .maxStreams         = 1000,
+    .maxPointsPerSecond = 10000000,
+    .maxStorage         = INT64_MAX,
+    .maxQueryTime       = INT64_MAX,
+    .maxInbound         = 0,
+    .maxOutbound        = 0,
+    .accessState        = TSDB_VN_ALL_ACCCESS
+  };
+  pAcct->acctId = sdbGetId(tsAcctSdb);
+  pAcct->createdTime = taosGetTimestampMs();
+
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsAcctSdb,
+    .pObj = pAcct,
+  };
+  sdbInsertRow(&oper);
+}
+
+#ifndef _ACCT
+
+int32_t acctInit() { return TSDB_CODE_SUCCESS; }
+void    acctCleanUp() {}
+int32_t acctCheck(void *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
+
+#endif

src/mnode/src/mgmtDClient.c

@@ -19,12 +19,14 @@
 #include "tsched.h"
 #include "tsystem.h"
 #include "tutil.h"
+#include "tglobalcfg.h"
 #include "dnode.h"
-#include "mnode.h"
-#include "tbalance.h"
-#include "mgmtDb.h"
-#include "tcluster.h"
 #include "tgrant.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtMnode.h"
+#include "mgmtDb.h"
+#include "mgmtDnode.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"

src/mnode/src/mgmtDServer.c

@@ -20,12 +20,14 @@
 #include "tsched.h"
 #include "tsystem.h"
 #include "tutil.h"
+#include "tgrant.h"
+#include "treplica.h"
+#include "tglobalcfg.h"
 #include "dnode.h"
-#include "mnode.h"
+#include "mgmtDef.h"
-#include "tbalance.h"
+#include "mgmtLog.h"
 #include "mgmtDb.h"
 #include "mgmtDServer.h"
-#include "tgrant.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"

src/mnode/src/mgmtDb.c

@@ -17,14 +17,16 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tutil.h"
-#include "name.h"
-#include "mnode.h"
-#include "taccount.h"
-#include "tbalance.h"
-#include "mgmtDb.h"
-#include "tcluster.h"
 #include "tgrant.h"
-#include "mpeer.h"
+#include "tglobalcfg.h"
+#include "ttime.h"
+#include "name.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
+#include "mgmtDb.h"
+#include "mgmtDnode.h"
+#include "mgmtMnode.h"
 #include "mgmtShell.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
@@ -51,7 +53,7 @@ static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) {
 
 static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
   SDbObj *pDb = pOper->pObj;
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
 
   pDb->pHead = NULL;
   pDb->pTail = NULL;
@@ -60,7 +62,7 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
   pDb->numOfSuperTables = 0;
 
   if (pAcct != NULL) {
-    acctAddDb(pAcct, pDb);
+    mgmtAddDbToAcct(pAcct, pDb);
   }
   else {
     mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct);
@@ -72,9 +74,9 @@ static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) {
 
 static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
   SDbObj *pDb = pOper->pObj;
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
 
-  acctRemoveDb(pAcct, pDb);
+  mgmtDropDbFromAcct(pAcct, pDb);
   mgmtDropAllChildTables(pDb);
   mgmtDropAllSuperTables(pDb);
   mgmtDropAllVgroups(pDb);
@@ -83,6 +85,12 @@ static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) {
 }
 
 static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) {
+  SDbObj *pDb = pOper->pObj;
+  SDbObj *pSaved = mgmtGetDb(pDb->name);
+  if (pDb != pSaved) {
+    memcpy(pSaved, pDb, pOper->rowSize);
+    free(pDb);
+  }
   return TSDB_CODE_SUCCESS;
 }
 
@@ -150,7 +158,7 @@ void mgmtIncDbRef(SDbObj *pDb) {
   return sdbIncRef(tsDbSdb, pDb); 
 }
 
-void mgmtReleaseDb(SDbObj *pDb) { 
+void mgmtDecDbRef(SDbObj *pDb) { 
   return sdbDecRef(tsDbSdb, pDb); 
 }
 
@@ -282,14 +290,14 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) {
 }
 
 static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
-  int32_t code = acctCheck(pAcct, TSDB_ACCT_DB);
+  int32_t code = acctCheck(pAcct, ACCT_GRANT_DB);
   if (code != 0) {
     return code;
   }
 
   SDbObj *pDb = mgmtGetDb(pCreate->db);
   if (pDb != NULL) {
-    mgmtReleaseDb(pDb);
+    mgmtDecDbRef(pDb);
     return TSDB_CODE_DB_ALREADY_EXIST;
   }
 
@@ -635,7 +643,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
     cols++;
 
     numOfRows++;
-    mgmtReleaseDb(pDb);
+    mgmtDecDbRef(pDb);
   }
 
   pShow->numOfReads += numOfRows;
@@ -882,7 +890,7 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
       mgmtSetDbDropping(pDb);
       numOfDbs++;
     }
-    mgmtReleaseDb(pDb);
+    mgmtDecDbRef(pDb);
   }
 
   mTrace("acct:%s, all dbs is is set dirty", pAcct->user, numOfDbs);

src/mnode/src/mgmtDnode.c

@@ -15,93 +15,217 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tmodule.h"
+#include "tgrant.h"
-#include "tbalance.h"
+#include "treplica.h"
-#include "tcluster.h"
+#include "tglobalcfg.h"
-#include "mnode.h"
+#include "ttime.h"
-#include "mpeer.h"
+#include "tutil.h"
+#include "tsocket.h"
+#include "dnode.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
 #include "mgmtDClient.h"
-#include "mgmtShell.h"
 #include "mgmtDServer.h"
+#include "mgmtDnode.h"
+#include "mgmtMnode.h"
+#include "mgmtSdb.h"
+#include "mgmtShell.h"
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-static void    clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg);
+void   *tsDnodeSdb = NULL;
-static void    clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
+int32_t tsDnodeUpdateSize = 0;
-static void    clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
+extern void *  tsVgroupSdb;
-static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-#ifndef _CLUSTER
+static int32_t mgmtCreateDnode(uint32_t ip);
+static void    mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg);
+static void    mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg);
+static void    mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
+static void    mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
+static void    mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
+static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-static SDnodeObj  tsDnodeObj = {0};
+static int32_t mgmtDnodeActionDestroy(SSdbOperDesc *pOper) {
-
+  tfree(pOper->pObj);
-int32_t clusterInitDnodes() {
+  return TSDB_CODE_SUCCESS;
-  tsDnodeObj.dnodeId          = 1;
-  tsDnodeObj.privateIp        = inet_addr(tsPrivateIp);
-  tsDnodeObj.publicIp         = inet_addr(tsPublicIp);
-  tsDnodeObj.createdTime      = taosGetTimestampMs();
-  tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes;
-  tsDnodeObj.status           = TAOS_DN_STATUS_OFFLINE;
-  tsDnodeObj.lastReboot       = taosGetTimestampSec();
-  sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId);
-
-  tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MGMT);
-  if (tsEnableHttpModule) {
-    tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_HTTP);
 }
-  if (tsEnableMonitorModule) {
+
-    tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR);
+static int32_t mgmtDnodeActionInsert(SSdbOperDesc *pOper) {
+  SDnodeObj *pDnode = pOper->pObj;
+  if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
+    pDnode->status = TAOS_DN_STATUS_OFFLINE;
   }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtDnodeActionDelete(SSdbOperDesc *pOper) {
+  SDnodeObj *pDnode = pOper->pObj;
+  void *     pNode = NULL;
+  void *     pLastNode = NULL;
+  SVgObj *   pVgroup = NULL;
+  int32_t    numOfVgroups = 0;
+
+  while (1) {
+    pLastNode = pNode;
+    pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
+      SSdbOperDesc oper = {
+        .type = SDB_OPER_LOCAL,
+        .table = tsVgroupSdb,
+        .pObj = pVgroup,
+      };
+      sdbDeleteRow(&oper);
+      pNode = pLastNode;
+      numOfVgroups++;
+      continue;
+    }
+  }
+
+  mTrace("dnode:%d, all vgroups:%d is dropped from sdb", pDnode->dnodeId, numOfVgroups);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtDnodeActionUpdate(SSdbOperDesc *pOper) {
+  SDnodeObj *pDnode = pOper->pObj;
+  SDnodeObj *pSaved = mgmtGetDnode(pDnode->dnodeId);
+  if (pDnode != pSaved) {
+    memcpy(pSaved, pDnode, pOper->rowSize);
+    free(pDnode);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtDnodeActionEncode(SSdbOperDesc *pOper) {
+  SDnodeObj *pDnode = pOper->pObj;
+  memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize);
+  pOper->rowSize = tsDnodeUpdateSize;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtDnodeActionDecode(SSdbOperDesc *pOper) {
+  SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
+  if (pDnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  memcpy(pDnode, pOper->rowData, tsDnodeUpdateSize);
+  pOper->pObj = pDnode;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtDnodeActionRestored() {
+  int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
+  if (numOfRows <= 0 && strcmp(tsMasterIp, tsPrivateIp) == 0) {
+    uint32_t ip = inet_addr(tsPrivateIp);
+    mgmtCreateDnode(ip);
+    SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
+    mgmtAddMnode(pDnode->dnodeId);
+    mgmtReleaseDnode(pDnode);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mgmtInitDnodes() {
+  SDnodeObj tObj;
+  tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+
+  SSdbTableDesc tableDesc = {
+    .tableId      = SDB_TABLE_DNODE,
+    .tableName    = "dnodes",
+    .hashSessions = TSDB_MAX_DNODES,
+    .maxRowSize   = tsDnodeUpdateSize,
+    .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
+    .keyType      = SDB_KEY_AUTO,
+    .insertFp     = mgmtDnodeActionInsert,
+    .deleteFp     = mgmtDnodeActionDelete,
+    .updateFp     = mgmtDnodeActionUpdate,
+    .encodeFp     = mgmtDnodeActionEncode,
+    .decodeFp     = mgmtDnodeActionDecode,
+    .destroyFp    = mgmtDnodeActionDestroy,
+    .restoredFp   = mgmtDnodeActionRestored
+  };
+
+  tsDnodeSdb = sdbOpenTable(&tableDesc);
+  if (tsDnodeSdb == NULL) {
+    mError("failed to init dnodes data");
+    return -1;
+  }
+
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DNODE, mgmtProcessCreateDnodeMsg);
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DNODE, mgmtProcessDropDnodeMsg); 
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg);
+  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
+  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes);
+ 
+  mTrace("dnodes table is created");
   return 0;
 }
 
-void *clusterGetNextDnode(void *pNode, SDnodeObj **pDnode) {
+void mgmtCleanupDnodes() {
-  if (*pDnode == NULL) {
+  sdbCloseTable(tsDnodeSdb);
-    *pDnode = &tsDnodeObj;
-  } else {
-    *pDnode = NULL;
-  }
-  return *pDnode;
 }
 
-void    clusterCleanupDnodes() {}
+void *mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { 
-int32_t clusterGetDnodesNum() { return 1; }
+  return sdbFetchRow(tsDnodeSdb, pNode, (void **)pDnode); 
-void *  clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : NULL; }
-void *  clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; }
-void    clusterReleaseDnode(struct _dnode_obj *pDnode) {}
-void    clusterUpdateDnode(struct _dnode_obj *pDnode) {}
-void    clusterMonitorDnodeModule() {}
-
-#endif
-
-int32_t clusterInit() {
-  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, clusterProcessCfgDnodeMsg);
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, clusterProcessCfgDnodeMsgRsp);
-  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, clusterProcessDnodeStatusMsg);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, clusterGetModuleMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, clusterRetrieveModules);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, clusterGetConfigMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, clusterRetrieveConfigs);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, clusterGetVnodeMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, clusterRetrieveVnodes);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, clusterGetDnodeMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, clusterRetrieveDnodes);
- 
-  return clusterInitDnodes();
 }
 
-void clusterCleanUp() {
+int32_t mgmtGetDnodesNum() {
-  clusterCleanupDnodes();
+  return sdbGetNumOfRows(tsDnodeSdb);
 }
 
-void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
+void *mgmtGetDnode(int32_t dnodeId) {
+  return sdbGetRow(tsDnodeSdb, &dnodeId);
+}
+
+void *mgmtGetDnodeByIp(uint32_t ip) {
+  SDnodeObj *pDnode = NULL;
+  void *     pNode = NULL;
+
+  while (1) {
+    pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode);
+    if (pDnode == NULL) break;
+    if (ip == pDnode->privateIp) {
+      return pDnode;
+    }
+    mgmtReleaseDnode(pDnode);
+  }
+
+  return NULL;
+}
+
+void mgmtReleaseDnode(SDnodeObj *pDnode) {
+  sdbDecRef(tsDnodeSdb, pDnode);
+}
+
+void mgmtUpdateDnode(SDnodeObj *pDnode) {
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsDnodeSdb,
+    .pObj = pDnode,
+    .rowSize = tsDnodeUpdateSize
+  };
+
+  sdbUpdateRow(&oper);
+}
+
+void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
   SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont;
@@ -137,11 +261,11 @@ void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
-static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
+static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
   mPrint("cfg vnode rsp is received, result:%s", tstrerror(rpcMsg->code));
 }
 
-void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   SDMStatusMsg *pStatus = rpcMsg->pCont;
   pStatus->dnodeId = htonl(pStatus->dnodeId);
   pStatus->privateIp = htonl(pStatus->privateIp);
@@ -159,14 +283,14 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
 
   SDnodeObj *pDnode = NULL;
   if (pStatus->dnodeId == 0) {
-    pDnode = clusterGetDnodeByIp(pStatus->privateIp);
+    pDnode = mgmtGetDnodeByIp(pStatus->privateIp);
     if (pDnode == NULL) {
       mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp));
       mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
       return;
     }
   } else {
-    pDnode = clusterGetDnode(pStatus->dnodeId);
+    pDnode = mgmtGetDnode(pStatus->dnodeId);
     if (pDnode == NULL) {
       mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp));
       mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
@@ -180,7 +304,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   pDnode->numOfCores       = pStatus->numOfCores;
   pDnode->diskAvailable    = pStatus->diskAvailable;
   pDnode->alternativeRole  = pStatus->alternativeRole;
-  pDnode->numOfTotalVnodes = pStatus->numOfTotalVnodes; 
+  pDnode->totalVnodes      = pStatus->numOfTotalVnodes; 
   
   if (pStatus->dnodeId == 0) {
     mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName);
@@ -208,11 +332,10 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
     pDnode->status = TAOS_DN_STATUS_READY;
-    balanceNotify();
+    replicaNotify();
-    clusterMonitorDnodeModule();
   }
 
-  clusterReleaseDnode(pDnode);
+  mgmtReleaseDnode(pDnode);
 
   int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SVnodeAccess);
   SDMStatusRsp *pRsp = rpcMallocCont(contLen);
@@ -221,7 +344,7 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
     return;
   }
 
-  mpeerGetMpeerInfos(&pRsp->mpeers);
+  mgmtGetMnodeList(&pRsp->mpeers);
 
   pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId);
   pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus);
@@ -242,7 +365,123 @@ void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
-static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mgmtCreateDnode(uint32_t ip) {
+  int32_t grantCode = grantCheck(TSDB_GRANT_DNODE);
+  if (grantCode != TSDB_CODE_SUCCESS) {
+    return grantCode;
+  }
+
+  SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
+  if (pDnode != NULL) {
+    mError("dnode:%d is alredy exist, ip:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp));
+    return TSDB_CODE_DNODE_ALREADY_EXIST;
+  }
+
+  pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
+  pDnode->privateIp = ip;
+  pDnode->publicIp = ip;
+  pDnode->createdTime = taosGetTimestampMs();
+  pDnode->status = TAOS_DN_STATUS_OFFLINE; 
+  pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM; 
+  sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1);
+
+  if (pDnode->privateIp == inet_addr(tsMasterIp)) {
+    pDnode->moduleStatus |= (1 << TSDB_MOD_MGMT);
+  }
+  
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsDnodeSdb,
+    .pObj = pDnode,
+    .rowSize = sizeof(SDnodeObj)
+  };
+
+  int32_t code = sdbInsertRow(&oper);
+  if (code != TSDB_CODE_SUCCESS) {
+    tfree(pDnode);
+    code = TSDB_CODE_SDB_ERROR;
+  }
+
+  mPrint("dnode:%d is created, result:%s", pDnode->dnodeId, tstrerror(code));
+  return code;
+}
+
+int32_t mgmtDropDnode(SDnodeObj *pDnode) {
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsDnodeSdb,
+    .pObj = pDnode
+  };
+
+  int32_t code = sdbDeleteRow(&oper); 
+  if (code != TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_SDB_ERROR;
+  }
+
+  mLPrint("dnode:%d is dropped from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
+  return code;
+}
+
+static int32_t mgmtDropDnodeByIp(uint32_t ip) {
+  SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
+  if (pDnode == NULL) {
+    mError("dnode:%s, is not exist", taosIpStr(ip));
+    return TSDB_CODE_INVALID_VALUE;
+  }
+
+  if (pDnode->privateIp == dnodeGetMnodeMasteIp()) {
+    mError("dnode:%d, can't drop dnode which is master", pDnode->dnodeId);
+    return TSDB_CODE_NO_REMOVE_MASTER;
+  }
+
+#ifndef _VPEER
+  return mgmtDropDnode(pDnode);
+#else
+  return balanceDropDnode(pDnode);
+#endif
+}
+
+static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) {
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  
+  SCMCreateDnodeMsg *pCreate = pMsg->pCont;
+
+  if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) {
+    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+  } else {
+    uint32_t ip = inet_addr(pCreate->ip);
+    rpcRsp.code = mgmtCreateDnode(ip);
+    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+      SDnodeObj *pDnode = mgmtGetDnodeByIp(ip);
+      mLPrint("dnode:%d, ip:%s is created by %s", pDnode->dnodeId, pCreate->ip, pMsg->pUser->user);
+    } else {
+      mError("failed to create dnode:%s, reason:%s", pCreate->ip, tstrerror(rpcRsp.code));
+    }
+  }
+  rpcSendResponse(&rpcRsp);
+}
+
+
+static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  
+  SCMDropDnodeMsg *pDrop = pMsg->pCont;
+  if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) {
+    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+  } else {
+    uint32_t ip = inet_addr(pDrop->ip);
+    rpcRsp.code = mgmtDropDnodeByIp(ip);
+    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+      mLPrint("dnode:%s is dropped by %s", pDrop->ip, pMsg->pUser->user);
+    } else {
+      mError("failed to drop dnode:%s, reason:%s", pDrop->ip, tstrerror(rpcRsp.code));
+    }
+  }
+
+  rpcSendResponse(&rpcRsp);
+}
+
+static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
 
@@ -309,7 +548,7 @@ static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = clusterGetDnodesNum();
+  pShow->numOfRows = mgmtGetDnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
 
@@ -318,7 +557,7 @@ static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
   return 0;
 }
 
-static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   int32_t    cols      = 0;
   SDnodeObj *pDnode   = NULL;
@@ -326,7 +565,7 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows,
   char       ipstr[32];
 
   while (numOfRows < rows) {
-    pShow->pNode = clusterGetNextDnode(pShow->pNode, &pDnode);
+    pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode);
     if (pDnode == NULL) break;
 
     cols = 0;
@@ -350,7 +589,7 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows,
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
+    strcpy(pWrite, mgmtGetDnodeStatusStr(pDnode->status));
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -358,29 +597,29 @@ static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows,
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->numOfTotalVnodes;
+    *(int16_t *)pWrite = pDnode->totalVnodes;
     cols++;
 
 #ifdef _VPEER
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
+    strcpy(pWrite, mgmtGetDnodeStatusStr(pDnode->status));
     cols++;
 #endif    
 
     numOfRows++;
-    clusterReleaseDnode(pDnode);
+    mgmtReleaseDnode(pDnode);
   }
 
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
 
-bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
+bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
   uint32_t status = pDnode->moduleStatus & (1 << moduleType);
   return status > 0;
 }
 
-static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
@@ -390,21 +629,27 @@ static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
 
   SSchema *pSchema = pMeta->schema;
 
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = 16;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "IP");
+  strcpy(pSchema[cols].name, "ip");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pShow->bytes[cols] = 10;
+  pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "module type");
+  strcpy(pSchema[cols].name, "module");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pShow->bytes[cols] = 10;
+  pShow->bytes[cols] = 8;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "module status");
+  strcpy(pSchema[cols].name, "status");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -416,18 +661,7 @@ static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = 0;
+  pShow->numOfRows = mgmtGetDnodesNum() * TSDB_MOD_MAX;
-  SDnodeObj *pDnode = NULL;
-  while (1) {
-    pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
-    if (pDnode == NULL) break;
-    for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
-      if (clusterCheckModuleInDnode(pDnode, moduleType)) {
-        pShow->numOfRows++;
-      }
-    }
-  }
-
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
   mgmtReleaseUser(pUser);
@@ -435,53 +669,66 @@ static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   return 0;
 }
 
-int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t numOfRows = 0;
-  SDnodeObj *pDnode = NULL;
   char *  pWrite;
-  int32_t    cols = 0;
-  char       ipstr[20];
 
   while (numOfRows < rows) {
-    clusterReleaseDnode(pDnode);
+    SDnodeObj *pDnode = NULL;
-    pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
 
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
-      if (!clusterCheckModuleInDnode(pDnode, moduleType)) {
+      int32_t cols = 0;
-        continue;
-      }
 
-      cols = 0;
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      *(int16_t *)pWrite = pDnode->dnodeId;
+      cols++;
 
+      char ipstr[20];
       tinet_ntoa(ipstr, pDnode->privateIp);
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       strcpy(pWrite, ipstr);
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      strcpy(pWrite, tsModule[moduleType].name);
+      switch (moduleType) {
+        case TSDB_MOD_MGMT:
+          strcpy(pWrite, "mgmt");
+          break;
+        case TSDB_MOD_HTTP:
+          strcpy(pWrite, "http");
+          break;
+        case TSDB_MOD_MONITOR:
+          strcpy(pWrite, "monitor");
+          break;
+        default:
+          strcpy(pWrite, "unknown");
+      }
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
+      bool enable = mgmtCheckModuleInDnode(pDnode, moduleType);
+      strcpy(pWrite, enable ? "enable" : "disable");
       cols++;
 
       numOfRows++;
     }
+
+    mgmtReleaseDnode(pDnode);
   }
 
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
 
-static bool clusterCheckConfigShow(SGlobalConfig *cfg) {
+static bool mgmtCheckConfigShow(SGlobalConfig *cfg) {
   if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW))
     return false;
   return true;
 }
 
-static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
@@ -512,7 +759,7 @@ static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pShow->numOfRows = 0;
   for (int32_t i = tsGlobalConfigNum - 1; i >= 0; --i) {
     SGlobalConfig *cfg = tsGlobalConfig + i;
-    if (!clusterCheckConfigShow(cfg)) continue;
+    if (!mgmtCheckConfigShow(cfg)) continue;
     pShow->numOfRows++;
   }
 
@@ -523,12 +770,12 @@ static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   return 0;
 }
 
-static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t numOfRows = 0;
 
   for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) {
     SGlobalConfig *cfg = tsGlobalConfig + i;
-    if (!clusterCheckConfigShow(cfg)) continue;
+    if (!mgmtCheckConfigShow(cfg)) continue;
 
     char *pWrite;
     int32_t   cols = 0;
@@ -570,7 +817,7 @@ static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows,
   return numOfRows;
 }
 
-static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
@@ -599,7 +846,7 @@ static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
   SDnodeObj *pDnode = NULL;
   if (pShow->payloadLen > 0 ) {
     uint32_t ip = ip2uint(pShow->payload);
-    pDnode = clusterGetDnodeByIp(ip);
+    pDnode = mgmtGetDnodeByIp(ip);
     if (NULL == pDnode) {
       return TSDB_CODE_NODE_OFFLINE;
     }
@@ -616,7 +863,7 @@ static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
     pShow->pNode = pDnode;
   } else {
     while (true) {
-      pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+      pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
       if (pDnode == NULL) break;
       pShow->numOfRows += pDnode->openVnodes;
 
@@ -627,13 +874,13 @@ static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
   } 
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  clusterReleaseDnode(pDnode);
+  mgmtReleaseDnode(pDnode);
   mgmtReleaseUser(pUser);
 
   return 0;
 }
 
-static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   SDnodeObj *pDnode = NULL;
   char *     pWrite;
@@ -674,7 +921,7 @@ static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows,
   return numOfRows;
 }
 
-char* clusterGetDnodeStatusStr(int32_t dnodeStatus) {
+char* mgmtGetDnodeStatusStr(int32_t dnodeStatus) {
   switch (dnodeStatus) {
     case TAOS_DN_STATUS_OFFLINE:   return "offline";
     case TAOS_DN_STATUS_DROPPING:  return "dropping";

src/mnode/src/mgmtGrant.c

@@ -17,8 +17,8 @@
 #ifndef _GRANT
 #include "os.h"
 #include "taoserror.h"
-#include "tlog.h"
 #include "tgrant.h"
+#include "mgmtLog.h"
 
 int32_t grantInit() { return TSDB_CODE_SUCCESS; }
 void    grantCleanUp() {}

src/mnode/src/mgmtMain.c

@@ -16,14 +16,15 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
-#include "tmodule.h"
 #include "tsched.h"
-#include "mnode.h"
+#include "treplica.h"
-#include "taccount.h"
-#include "tbalance.h"
-#include "tcluster.h"
 #include "tgrant.h"
-#include "mpeer.h"
+#include "ttimer.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
+#include "mgmtDnode.h"
+#include "mgmtMnode.h"
 #include "mgmtDb.h"
 #include "mgmtDClient.h"
 #include "mgmtDServer.h"
@@ -33,8 +34,91 @@
 #include "mgmtTable.h"
 #include "mgmtShell.h"
 
-static int32_t mgmtCheckMgmtRunning();
 void *tsMgmtTmr = NULL;
+static bool tsMgmtIsRunning = false;
+
+int32_t mgmtStartSystem() {
+  if (tsMgmtIsRunning) {
+    mPrint("TDengine mgmt module already started...");
+    return 0;
+  }
+
+  mPrint("starting to initialize TDengine mgmt ...");
+  struct stat dirstat;
+  if (stat(tsMnodeDir, &dirstat) < 0) {
+    mkdir(tsMnodeDir, 0755);
+  }
+
+  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
+  if (tsMgmtTmr == NULL) {
+    mError("failed to init timer");
+    return -1;
+  }
+
+  if (mgmtInitAccts() < 0) {
+    mError("failed to init accts");
+    return -1;
+  }
+
+  if (grantInit() < 0) {
+    mError("failed to init grants");
+    return -1;
+  }
+
+  if (mgmtInitUsers() < 0) {
+    mError("failed to init users");
+    return -1;
+  }
+
+  if (mgmtInitDnodes() < 0) {
+    mError("failed to init dnodes");
+    return -1;
+  }
+
+  if (mgmtInitDbs() < 0) {
+    mError("failed to init dbs");
+    return -1;
+  }
+
+  if (mgmtInitVgroups() < 0) {
+    mError("failed to init vgroups");
+    return -1;
+  }
+
+  if (mgmtInitTables() < 0) {
+    mError("failed to init tables");
+    return -1;
+  }
+
+  if (mgmtInitMnodes() < 0) {
+    mError("failed to init mpeers");
+    return -1;
+  }
+
+  if (sdbInit() < 0) {
+    mError("failed to init sdb");
+    return -1;
+  }
+
+  if (mgmtInitDClient() < 0) {
+    return -1;
+  }
+
+  if (mgmtInitDServer() < 0) {
+    return -1;
+  }
+
+  if (replicaInit() < 0) {
+    mError("failed to init dnode balance")
+  }
+
+  grantReset(TSDB_GRANT_ALL, 0);
+  tsMgmtIsRunning = true;
+
+  mPrint("TDengine mgmt is initialized successfully");
+
+  return 0;
+}
 
 int32_t mgmtInitSystem() {
   if (mgmtInitShell() != 0) {
@@ -55,124 +139,34 @@ int32_t mgmtInitSystem() {
   return 0;
 }
 
-int32_t mgmtStartSystem() {
-  mPrint("starting to initialize TDengine mgmt ...");
-
-  struct stat dirstat;
-  if (stat(tsMnodeDir, &dirstat) < 0) {
-    mkdir(tsMnodeDir, 0755);
-  }
-
-  if (mgmtCheckMgmtRunning() != 0) {
-    mPrint("TDengine mgmt module already started...");
-    return 0;
-  }
-
-  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
-  if (tsMgmtTmr == NULL) {
-    mError("failed to init timer");
-    return -1;
-  }
-
-  if (acctInit() < 0) {
-    mError("failed to init accts");
-    return -1;
-  }
-
-  if (grantInit() < 0) {
-    mError("failed to init grants");
-    return -1;
-  }
-
-  if (mgmtInitUsers() < 0) {
-    mError("failed to init users");
-    return -1;
-  }
-
-  if (clusterInit() < 0) {
-    mError("failed to init dnodes");
-    return -1;
-  }
-
-  if (mgmtInitDbs() < 0) {
-    mError("failed to init dbs");
-    return -1;
-  }
-
-  if (mgmtInitVgroups() < 0) {
-    mError("failed to init vgroups");
-    return -1;
-  }
-
-  if (mgmtInitTables() < 0) {
-    mError("failed to init tables");
-    return -1;
-  }
-
-  if (mpeerInit() < 0) {
-    mError("failed to init mpeers");
-    return -1;
-  }
-
-  if (sdbInit() < 0) {
-    mError("failed to init sdb");
-    return -1;
-  }
-
-  if (mgmtInitDClient() < 0) {
-    return -1;
-  }
-
-  if (mgmtInitDServer() < 0) {
-    return -1;
-  }
-
-  if (balanceInit() < 0) {
-    mError("failed to init dnode balance")
-  }
-
-  grantReset(TSDB_GRANT_ALL, 0);
-
-  mPrint("TDengine mgmt is initialized successfully");
-
-  return 0;
-}
-
-
-void mgmtStopSystem() {
-  if (mpeerIsMaster()) {
-    mTrace("it is a master mgmt node, it could not be stopped");
-    return;
-  }
-
-  mgmtCleanUpSystem();
-  remove(tsMnodeDir);
-}
-
 void mgmtCleanUpSystem() {
   mPrint("starting to clean up mgmt");
   grantCleanUp();
-  mpeerCleanup();
+  mgmtCleanupMnodes();
-  balanceCleanUp();
+  replicaCleanUp();
   mgmtCleanUpShell();
   mgmtCleanupDClient();
   mgmtCleanupDServer();
   mgmtCleanUpTables();
   mgmtCleanUpVgroups();
   mgmtCleanUpDbs();
-  clusterCleanUp();
+  mgmtCleanupDnodes();
   mgmtCleanUpUsers();
-  acctCleanUp();
+  mgmtCleanUpAccts();
   sdbCleanUp();
   taosTmrCleanUp(tsMgmtTmr);
+  tsMgmtIsRunning = false;
   mPrint("mgmt is cleaned up");
 }
 
-static int32_t mgmtCheckMgmtRunning() {
+void mgmtStopSystem() {
-  if (tsModuleStatus & (1 << TSDB_MOD_MGMT)) {
+  if (mgmtIsMaster()) {
-    return -1;
+    mTrace("it is a master mgmt node, it could not be stopped");
+    return;
   }
 
-  tsetModuleStatus(TSDB_MOD_MGMT);
+  mgmtCleanUpSystem();
-  return 0;
+
+  mPrint("mgmt file is removed");
+  remove(tsMnodeDir);
 }

src/mnode/src/mgmtMnode.c

@@ -18,89 +18,132 @@
 #include "taoserror.h"
 #include "trpc.h"
 #include "tsync.h"
-#include "mpeer.h"
+#include "treplica.h"
+#include "tutil.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtMnode.h"
+#include "mgmtDnode.h"
+#include "mgmtSdb.h"
 #include "mgmtShell.h"
 #include "mgmtUser.h"
 
+static void *  tsMnodeSdb = NULL;
+static int32_t tsMnodeUpdateSize = 0;
+static int32_t tsMnodeIsMaster = true;
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-#ifndef _MPEER
+static int32_t mgmtMnodeActionDestroy(SSdbOperDesc *pOper) {
+  tfree(pOper->pObj);
+  return TSDB_CODE_SUCCESS;
+}
 
-static SMnodeObj tsMnodeObj = {0};
+static int32_t mgmtMnodeActionInsert(SSdbOperDesc *pOper) {
+  SMnodeObj *pMnode = pOper->pObj;
+  SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+  if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
 
-int32_t mpeerInitMnodes() {
+  pMnode->pDnode = pDnode;
-  tsMnodeObj.mnodeId     = 1;
+  pDnode->isMgmt = true;
-  tsMnodeObj.privateIp   = inet_addr(tsPrivateIp);
+  mgmtReleaseDnode(pDnode);
-  tsMnodeObj.publicIp    = inet_addr(tsPublicIp);
-  tsMnodeObj.createdTime = taosGetTimestampMs();
-  tsMnodeObj.role        = TAOS_SYNC_ROLE_MASTER;
-  tsMnodeObj.port        = tsMnodeDnodePort;
-  sprintf(tsMnodeObj.mnodeName, "m%d", tsMnodeObj.mnodeId);
   
   return TSDB_CODE_SUCCESS;
 }
 
-void    mpeerCleanupMnodes() {}
+static int32_t mgmtMnodeActionDelete(SSdbOperDesc *pOper) {
-int32_t mpeerAddMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
+  SMnodeObj *pMnode = pOper->pObj;
-int32_t mpeerRemoveMnode(int32_t dnodeId) { return TSDB_CODE_SUCCESS; }
+  mTrace("mnode:%d, is dropped from sdb", pMnode->mnodeId);
-void *  mpeerGetMnode(int32_t mnodeId) { return &tsMnodeObj; }
-int32_t mpeerGetMnodesNum() { return 1; }
-void    mpeerReleaseMnode(struct _mnode_obj *pMnode) {}
-bool    mpeerIsMaster() { return tsMnodeObj.role == TAOS_SYNC_ROLE_MASTER; }
-void    mpeerUpdateSync() {}
-
-void *mpeerGetNextMnode(void *pNode, SMnodeObj **pMnode) {
-  if (*pMnode == NULL) {
-    *pMnode = &tsMnodeObj;
-  } else {
-    *pMnode = NULL;
-  }
-
-  return *pMnode;
-}
-
-void mpeerGetPrivateIpList(SRpcIpSet *ipSet) {
-  ipSet->inUse = 0;
-  ipSet->numOfIps = 1;
-  ipSet->port = htons(tsMnodeObj.port);
-  ipSet->ip[0] = htonl(tsMnodeObj.privateIp);
-}
-
-void mpeerGetPublicIpList(SRpcIpSet *ipSet) {
-  ipSet->inUse = 0;
-  ipSet->numOfIps = 1;
-  ipSet->port = htons(tsMnodeObj.port);
-  ipSet->ip[0] = htonl(tsMnodeObj.publicIp);
-}
-
-void mpeerGetMpeerInfos(void *param) {
-  SDMNodeInfos *mpeers = param;
-  mpeers->inUse = 0;
-  mpeers->nodeNum = 1;
-  mpeers->nodeInfos[0].nodeId = htonl(tsMnodeObj.mnodeId);
-  mpeers->nodeInfos[0].nodeIp = htonl(tsMnodeObj.privateIp);
-  mpeers->nodeInfos[0].nodePort = htons(tsMnodeObj.port);
-  strcpy(mpeers->nodeInfos[0].nodeName, tsMnodeObj.mnodeName);
-}
-
-int32_t mpeerForwardReqToPeer(void *pHead) {
   return TSDB_CODE_SUCCESS;
 }
 
-#endif
+static int32_t mgmtMnodeActionUpdate(SSdbOperDesc *pOper) {
+  SMnodeObj *pMnode = pOper->pObj;
+  SMnodeObj *pSaved = mgmtGetMnode(pMnode->mnodeId);
+  if (pMnode != pSaved) {
+    memcpy(pSaved, pMnode, pOper->rowSize);
+    free(pMnode);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtMnodeActionEncode(SSdbOperDesc *pOper) {
+  SMnodeObj *pMnode = pOper->pObj;
+  memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize);
+  pOper->rowSize = tsMnodeUpdateSize;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtMnodeActionDecode(SSdbOperDesc *pOper) {
+  SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
+  if (pMnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  memcpy(pMnode, pOper->rowData, tsMnodeUpdateSize);
+  pOper->pObj = pMnode;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mgmtMnodeActionRestored() {
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mgmtInitMnodes() {
+  SMnodeObj tObj;
+  tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
+
+  SSdbTableDesc tableDesc = {
+    .tableId      = SDB_TABLE_MNODE,
+    .tableName    = "mnodes",
+    .hashSessions = TSDB_MAX_MNODES,
+    .maxRowSize   = tsMnodeUpdateSize,
+    .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
+    .keyType      = SDB_KEY_INT,
+    .insertFp     = mgmtMnodeActionInsert,
+    .deleteFp     = mgmtMnodeActionDelete,
+    .updateFp     = mgmtMnodeActionUpdate,
+    .encodeFp     = mgmtMnodeActionEncode,
+    .decodeFp     = mgmtMnodeActionDecode,
+    .destroyFp    = mgmtMnodeActionDestroy,
+    .restoredFp   = mgmtMnodeActionRestored
+  };
+
+  tsMnodeSdb = sdbOpenTable(&tableDesc);
+  if (tsMnodeSdb == NULL) {
+    mError("failed to init mnodes data");
+    return -1;
+  }
 
-int32_t mpeerInit() {
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
-  return mpeerInitMnodes();
+
+  mTrace("mnodes table is created");
+  return TSDB_CODE_SUCCESS;
 }
 
-void mpeerCleanup() {
+void mgmtCleanupMnodes() {
-  mpeerCleanupMnodes();
+  sdbCloseTable(tsMnodeSdb);
 }
 
-static char *mpeerGetMnodeRoleStr(int32_t role) {
+int32_t mgmtGetMnodesNum() { 
+  return sdbGetNumOfRows(tsMnodeSdb); 
+}
+
+void *mgmtGetMnode(int32_t mnodeId) {
+  return sdbGetRow(tsMnodeSdb, &mnodeId);
+}
+
+void mgmtReleaseMnode(SMnodeObj *pMnode) {
+  sdbDecRef(tsMnodeSdb, pMnode);
+}
+
+void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { 
+  return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode); 
+}
+
+static char *mgmtGetMnodeRoleStr(int32_t role) {
   switch (role) {
     case TAOS_SYNC_ROLE_OFFLINE:
       return "offline";
@@ -115,6 +158,101 @@ static char *mpeerGetMnodeRoleStr(int32_t role) {
   }
 }
 
+bool mgmtIsMaster() { return tsMnodeIsMaster; }
+
+void mgmtGetMnodeIpList(SRpcIpSet *ipSet, bool usePublicIp) {
+  void *pNode = NULL;
+  while (1) {
+    SMnodeObj *pMnode = NULL;
+    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    if (pMnode == NULL) break;
+
+    if (usePublicIp) {
+      ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->publicIp);
+    } else {
+      ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->privateIp);
+    }
+
+    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+      ipSet->inUse = ipSet->numOfIps;
+    }
+
+    ipSet->numOfIps++;
+    ipSet->port = htons(pMnode->pDnode->mnodeShellPort);
+    
+    mgmtReleaseMnode(pMnode);
+  }
+}
+
+void mgmtGetMnodeList(void *param) {
+  SDMNodeInfos *mnodes = param;
+  mnodes->inUse = 0;
+  
+  int32_t index = 0;
+  void *pNode = NULL;
+  while (1) {
+    SMnodeObj *pMnode = NULL;
+    pNode = mgmtGetNextMnode(pNode, &pMnode);
+    if (pMnode == NULL) break;
+
+    mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
+    mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp);
+    mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort);
+    strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName);
+    mPrint("node:%d role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
+    if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
+      mnodes->inUse = index;
+      mPrint("node:%d inUse:%d", pMnode->mnodeId, mnodes->inUse);
+    }
+
+    index++;
+    mgmtReleaseMnode(pMnode);
+  }
+
+  mnodes->nodeNum = index;
+}
+
+int32_t mgmtAddMnode(int32_t dnodeId) {
+  SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
+  pMnode->mnodeId = dnodeId;
+  pMnode->createdTime = taosGetTimestampMs();
+
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsMnodeSdb,
+    .pObj = pMnode,
+  };
+
+  int32_t code = sdbInsertRow(&oper);
+  if (code != TSDB_CODE_SUCCESS) {
+    tfree(pMnode);
+    code = TSDB_CODE_SDB_ERROR;
+  }
+
+  return code;
+}
+
+int32_t mgmtDropMnode(int32_t dnodeId) {
+  SMnodeObj *pMnode = sdbGetRow(tsMnodeSdb, &dnodeId);
+  if (pMnode == NULL) {
+    return TSDB_CODE_DNODE_NOT_EXIST;
+  }
+  
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_GLOBAL,
+    .table = tsMnodeSdb,
+    .pObj = pMnode
+  };
+
+  int32_t code = sdbDeleteRow(&oper);
+  if (code != TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_SDB_ERROR;
+  }
+
+  sdbDecRef(tsMnodeSdb, pMnode);
+  return code;
+}
+
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
@@ -162,7 +300,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = mpeerGetMnodesNum();
+  pShow->numOfRows = mgmtGetMnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
   mgmtReleaseUser(pUser);
@@ -178,7 +316,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char       ipstr[32];
 
   while (numOfRows < rows) {
-    pShow->pNode = mpeerGetNextMnode(pShow->pNode, &pMnode);
+    pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode);
     if (pMnode == NULL) break;
 
     cols = 0;
@@ -187,12 +325,12 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     *(int16_t *)pWrite = pMnode->mnodeId;
     cols++;
 
-    tinet_ntoa(ipstr, pMnode->privateIp);
+    tinet_ntoa(ipstr, pMnode->pDnode->privateIp);
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     strcpy(pWrite, ipstr);
     cols++;
 
-    tinet_ntoa(ipstr, pMnode->publicIp);
+    tinet_ntoa(ipstr, pMnode->pDnode->publicIp);
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     strcpy(pWrite, ipstr);
     cols++;
@@ -202,12 +340,12 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, mpeerGetMnodeRoleStr(pMnode->role));
+    strcpy(pWrite, mgmtGetMnodeRoleStr(pMnode->role));
     cols++;
 
     numOfRows++;
 
-    mpeerReleaseMnode(pMnode);
+    mgmtReleaseMnode(pMnode);
   }
 
   pShow->numOfReads += numOfRows;

src/mnode/src/mgmtProfile.c

@@ -16,10 +16,14 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
-#include "taccount.h"
+#include "taoserror.h"
-#include "tcluster.h"
+#include "tutil.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
+#include "mgmtDnode.h"
 #include "mgmtDb.h"
-#include "mpeer.h"
+#include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"
@@ -787,11 +791,11 @@ void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
   if (pMsg != NULL) {
     rpcFreeCont(pMsg->pCont);
     if (pMsg->pUser) mgmtReleaseUser(pMsg->pUser);
-    if (pMsg->pDb) mgmtReleaseDb(pMsg->pDb);
+    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
     if (pMsg->pVgroup) mgmtReleaseVgroup(pMsg->pVgroup);
     if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
-    if (pMsg->pAcct) acctReleaseAcct(pMsg->pAcct);
+    if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
-    if (pMsg->pDnode) clusterReleaseDnode(pMsg->pDnode);
+    if (pMsg->pDnode) mgmtReleaseDnode(pMsg->pDnode);
     free(pMsg);
   }
 }

src/mnode/src/mgmtReplica.c

@@ -14,34 +14,41 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "tbalance.h"
+#include "os.h"
-#include "mnode.h"
+#include "trpc.h"
-#include "tcluster.h"
+#include "treplica.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtMnode.h"
+#include "mgmtDnode.h"
 #include "mgmtVgroup.h"
 
-#ifndef _VPEER
+#ifndef _SYNC
-int32_t balanceInit() { return 0; }
-void    balanceCleanUp() {}
-void    balanceNotify() {}
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+int32_t replicaInit() { return TSDB_CODE_SUCCESS; }
+void    replicaCleanUp() {}
+void    replicaNotify() {}
+void    replicaReset() {}
+int32_t replicaForwardReqToPeer(void *pHead) { return TSDB_CODE_SUCCESS; }
+
+int32_t replicaAllocVnodes(SVgObj *pVgroup) {
   void *     pNode = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
   float      vnodeUsage = 1.0;
 
   while (1) {
-    pNode = clusterGetNextDnode(pNode, &pDnode);
+    pNode = mgmtGetNextDnode(pNode, &pDnode);
     if (pDnode == NULL) break;
 
-    if (pDnode->numOfTotalVnodes > 0 && pDnode->openVnodes < pDnode->numOfTotalVnodes) {
+    if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) {
-      float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes;
+      float usage = (float)pDnode->openVnodes / pDnode->totalVnodes;
       if (usage <= vnodeUsage) {
         pSelDnode = pDnode;
         vnodeUsage = usage;
       }
     }
-    clusterReleaseDnode(pDnode);
+    mgmtReleaseDnode(pDnode);
   }
 
   if (pSelDnode == NULL) {

src/mnode/src/mgmtSdb.c

@@ -18,11 +18,13 @@
 #include "taoserror.h"
 #include "tlog.h"
 #include "trpc.h"
+#include "treplica.h"
 #include "tqueue.h"
 #include "twal.h"
 #include "hashint.h"
 #include "hashstr.h"
-#include "mpeer.h"
+#include "mgmtLog.h"
+#include "mgmtMnode.h"
 #include "mgmtSdb.h"
 
 typedef struct _SSdbTable {
@@ -131,7 +133,7 @@ int32_t sdbInit() {
 
   sdbTrace("sdb is initialized, version:%d totalRows:%d numOfTables:%d", tsSdbObj->version, totalRows, numOfTables);
   
-  mpeerUpdateSync();
+  replicaNotify();
 
   return TSDB_CODE_SUCCESS;
 }
@@ -264,7 +266,7 @@ static int32_t sdbProcessWriteFromApp(SSdbTable *pTable, SWalHead *pHead, int32_
   tsSdbObj->version++;
   pHead->version = tsSdbObj->version;
 
-  code = mpeerForwardReqToPeer(pHead);
+  code = replicaForwardReqToPeer(pHead);
   if (code != TSDB_CODE_SUCCESS) {
     pthread_mutex_unlock(&tsSdbObj->mutex);
     sdbError("table:%s, failed to forward %s record:%s from file, version:%" PRId64 ", reason:%s", pTable->tableName,
@@ -335,25 +337,19 @@ static int32_t sdbProcessWriteFromWal(SSdbTable *pTable, SWalHead *pHead, int32_
     SRowMeta *rowMeta = sdbGetRowMeta(pTable, pHead->cont);
     assert(rowMeta != NULL && rowMeta->row != NULL);
 
-    SSdbOperDesc oper1 = {
+    SSdbOperDesc oper = {
-        .table = pTable,
-        .pObj = rowMeta->row,
-    };
-    sdbDeleteLocal(pTable, &oper1);
-
-    SSdbOperDesc oper2 = {
         .rowSize = pHead->len,
         .rowData = pHead->cont,
         .table = pTable,
     };
-    code = (*pTable->decodeFp)(&oper2);
+    code = (*pTable->decodeFp)(&oper);
     if (code < 0) {
       sdbTrace("table:%s, failed to decode %s record:%s from file, version:%" PRId64, pTable->tableName,
                sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version);
       pthread_mutex_unlock(&tsSdbObj->mutex);
       return code;
     }
-    code = sdbInsertLocal(pTable, &oper2);
+    code = sdbUpdateLocal(pTable, &oper);
   }
 
   pthread_mutex_unlock(&tsSdbObj->mutex);

src/mnode/src/mgmtShell.c

@@ -20,14 +20,16 @@
 #include "tlog.h"
 #include "trpc.h"
 #include "tsched.h"
+#include "tutil.h"
+#include "ttimer.h"
 #include "dnode.h"
-#include "mnode.h"
+#include "mgmtDef.h"
-#include "taccount.h"
+#include "mgmtLog.h"
-#include "tbalance.h"
+#include "mgmtAcct.h"
 #include "mgmtDb.h"
-#include "tcluster.h"
+#include "mgmtDnode.h"
 #include "tgrant.h"
-#include "mpeer.h"
+#include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
 #include "mgmtShell.h"
@@ -48,6 +50,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
 
+extern void *tsMgmtTmr;
 static void *tsMgmtShellRpc = NULL;
 static void *tsMgmtTranQhandle = NULL;
 static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
@@ -141,7 +144,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
     return;
   }
 
-  if (!mpeerIsMaster()) {
+  if (!mgmtIsMaster()) {
     // rpcSendRedirectRsp(rpcMsg->handle, mgmtGetMnodeIpListForRedirect());
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NO_MASTER);
     rpcFreeCont(rpcMsg->pCont);
@@ -329,11 +332,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  if (pMsg->usePublicIp) {
+  mgmtGetMnodeIpList(&pHBRsp->ipList, pMsg->usePublicIp);
-    mpeerGetPublicIpList(&pHBRsp->ipList);
-  } else {
-    mpeerGetPrivateIpList(&pHBRsp->ipList);
-  }
   
   /*
    * TODO
@@ -415,11 +414,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
   pConnectRsp->writeAuth = pUser->writeAuth;
   pConnectRsp->superAuth = pUser->superAuth;
 
-  if (pMsg->usePublicIp) {
+  mgmtGetMnodeIpList(&pConnectRsp->ipList, pMsg->usePublicIp);
-    mpeerGetPublicIpList(&pConnectRsp->ipList);
-  } else {
-    mpeerGetPrivateIpList(&pConnectRsp->ipList);
-  }
   
 connect_over:
   rpcRsp.code = code;

src/mnode/src/mgmtTable.c

@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-
 #include "taosmsg.h"
 #include "ttime.h"
 #include "tutil.h"
@@ -23,13 +22,16 @@
 #include "taosmsg.h"
 #include "tscompression.h"
 #include "name.h"
-#include "taccount.h"
+#include "tidpool.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
 #include "mgmtDClient.h"
 #include "mgmtDb.h"
-#include "tcluster.h"
+#include "mgmtDnode.h"
 #include "mgmtDServer.h"
 #include "tgrant.h"
-#include "mpeer.h"
+#include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
 #include "mgmtShell.h"
@@ -45,6 +47,9 @@ static int32_t tsSuperTableUpdateSize;
 static void *  mgmtGetChildTable(char *tableId);
 static void *  mgmtGetSuperTable(char *tableId);
 static void    mgmtDropAllChildTablesInStable(SSuperTableObj *pStable);
+static void    mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
+static void    mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable);
+
 static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -98,18 +103,18 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
     mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct == NULL) {
     mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
     return TSDB_CODE_INVALID_ACCT;
   }
-  acctReleaseAcct(pAcct);
+  mgmtDecAcctRef(pAcct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
-    pTable->superTable->numOfTables++;
+    mgmtAddTableIntoStable(pTable->superTable, pTable);
     grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
     pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1);
   } else {
@@ -140,19 +145,19 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
     mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct == NULL) {
     mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
     return TSDB_CODE_INVALID_ACCT;
   }
-  acctReleaseAcct(pAcct);
+  mgmtDecAcctRef(pAcct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
     pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
-    pTable->superTable->numOfTables--;
+    mgmtRemoveTableFromStable(pTable->superTable, pTable);
     mgmtDecTableRef(pTable->superTable);
   } else {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
@@ -165,6 +170,19 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
 }
 
 static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) {
+  SChildTableObj *pNew = pOper->pObj;
+  SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId);
+  if (pTable != pNew) {
+    void *oldSql = pTable->sql;
+    void *oldSchema = pTable->schema;
+    memcpy(pTable, pNew, pOper->rowSize);
+    pTable->sql = pNew->sql;
+    pTable->schema = pNew->schema;
+    free(pNew);
+    free(oldSql);
+    free(oldSchema);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -242,7 +260,7 @@ static int32_t mgmtChildTableActionRestored() {
       pNode = pLastNode;
       continue;
     }
-    mgmtReleaseDb(pDb);
+    mgmtDecDbRef(pDb);
 
     SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
     if (pVgroup == NULL) {
@@ -337,8 +355,40 @@ static void mgmtCleanUpChildTables() {
   sdbCloseTable(tsChildTableSdb);
 }
 
+static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+  if (pStable->vgLen == 0) {
+    pStable->vgLen = 10;
+    pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t));
+  }
+  
+  bool find = false;
+  int32_t pos = 0;
+  for (int pos = 0; pos < pStable->vgLen; ++pos) {
+    if (pStable->vgList[pos] == 0) break;
+    if (pStable->vgList[pos] == pCtable->vgId) {
+      find = true;
+      break;
+    }
+  }
+
+  if (!find) {
+    if (pos >= pStable->vgLen) {
+      pStable->vgLen *= 2;
+      pStable->vgList = realloc(pStable->vgList, pStable->vgLen * sizeof(int32_t));
+    }
+    pStable->vgList[pos] = pCtable->vgId;
+  }
+
+  pStable->numOfTables++;
+}
+
+static void mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
+  pStable->numOfTables--;
+}
+
 static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
   tfree(pStable->schema);
+  tfree(pStable->vgList)
   tfree(pStable);
 }
 
@@ -353,7 +403,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
   if (pDb != NULL) {
     mgmtAddSuperTableIntoDb(pDb);
   }
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -365,12 +415,23 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
     mgmtRemoveSuperTableFromDb(pDb);
     mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable);
   }
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) {
+  SSuperTableObj *pNew = pOper->pObj;
+  SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId);
+  if (pTable != pNew) {
+    void *oldSchema = pTable->schema;
+    memcpy(pTable, pNew, pOper->rowSize);
+    pTable->schema = pNew->schema;
+    free(pNew);
+    free(pNew->vgList);
+    free(oldSchema);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -864,10 +925,10 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
   pStable->numOfColumns += ncols;
   pStable->sversion++;
 
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables);
-    acctReleaseAcct(pAcct);
+    mgmtDecAcctRef(pAcct);
   }
 
   SSdbOperDesc oper = {
@@ -901,10 +962,10 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
   int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns);
   pStable->schema = realloc(pStable->schema, schemaSize);
 
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables;
-    acctReleaseAcct(pAcct);
+    mgmtDecAcctRef(pAcct);
   }
 
   SSdbOperDesc oper = {
@@ -970,7 +1031,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
   pShow->numOfRows = pDb->numOfSuperTables;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
   return 0;
 }
 
@@ -1035,7 +1096,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
   }
 
   pShow->numOfReads += numOfRows;
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   return numOfRows;
 }
@@ -1107,40 +1168,49 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
 
 static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
   SCMSTableVgroupMsg *pInfo = pMsg->pCont;
-  pMsg->pTable = mgmtGetSuperTable(pInfo->tableId);
+  SSuperTableObj *pTable = mgmtGetSuperTable(pInfo->tableId);
+
+  pMsg->pTable = (STableObj *)pTable;
   if (pMsg->pTable == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
     return;
   }
 
-  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * clusterGetDnodesNum());
+  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * pTable->vgLen;
+  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
-    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
     return;
   }
   
-  int32_t numOfVgroups = 1;
+  int32_t vg = 0;
-  int32_t numOfDnodes  = 1;
+  for (; vg < pTable->vgLen; ++vg) {
+    int32_t vgId = pTable->vgList[vg];
+    if (vgId == 0) break;
 
-  pRsp->numOfDnodes = htonl(numOfDnodes);
+    SVgObj *pVgroup = mgmtGetVgroup(vgId);
-  STableDnodeVgroupInfo* pVgroupInfo = pRsp->dnodeVgroups;
+    if (pVgroup == NULL) break;
-  pVgroupInfo->ipAddr.ip = htonl(inet_addr(tsPrivateIp));
 
-  pVgroupInfo->ipAddr.port = htons(0);      // todo fix it
+    pRsp->vgroups[vg].vgId = htonl(vgId);
-  pVgroupInfo->numOfVgroups = htonl(numOfVgroups); // todo fix it
+    for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) {
-  int32_t* vgIdList = pVgroupInfo->vgId;
+      SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[vn].dnodeId);
+      if (pDnode == NULL) break;
 
-  for(int32_t i = 0; i < numOfVgroups; ++i) {
+      pRsp->vgroups[vg].ipAddr[vn].ip = htonl(pDnode->privateIp);
-    vgIdList[i] = htonl(2);  // todo fix it
+      pRsp->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
+      pRsp->vgroups[vg].numOfIps++;
+
+      mgmtReleaseDnode(pDnode);
     }
 
-  assert(numOfDnodes == 1);  // this size is valid only when numOfDnodes equals 1
+    mgmtReleaseVgroup(pVgroup);
-  int32_t msgLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(STableDnodeVgroupInfo) + numOfVgroups * sizeof(int32_t);
+  }
+  pRsp->numOfVgroups = htonl(vg);
 
   SRpcMsg rpcRsp = {0};
   rpcRsp.handle = pMsg->thandle;
   rpcRsp.pCont = pRsp;
-  rpcRsp.contLen = msgLen;
+  rpcRsp.contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * vg;
   rpcSendResponse(&rpcRsp);
 }
 
@@ -1432,10 +1502,10 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
   pTable->numOfColumns += ncols;
   pTable->sversion++;
   
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
      pAcct->acctInfo.numOfTimeSeries += ncols;
-    acctReleaseAcct(pAcct);
+    mgmtDecAcctRef(pAcct);
   }
  
   SSdbOperDesc oper = {
@@ -1466,10 +1536,10 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
   pTable->numOfColumns--;
   pTable->sversion++;
 
-  SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
+  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries--;
-    acctReleaseAcct(pAcct);
+    mgmtDecAcctRef(pAcct);
   }
  
   SSdbOperDesc oper = {
@@ -1508,7 +1578,6 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
 
   pMeta->uid       = htobe64(pTable->uid);
   pMeta->sid       = htonl(pTable->sid);
-  pMeta->vgId      = htonl(pTable->vgId);
   pMeta->precision = pDb->cfg.precision;
   pMeta->tableType = pTable->info.type;
   strncpy(pMeta->tableId, pTable->info.tableId, tListLen(pTable->info.tableId));
@@ -1532,16 +1601,20 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
 
-  for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) {
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    if (pDnode == NULL) break;
     if (usePublicIp) {
-      pMeta->vpeerDesc[i].ip = htonl(pVgroup->vnodeGid[i].publicIp);
+      pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->publicIp);
+      pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
     } else {
-      pMeta->vpeerDesc[i].ip = htonl(pVgroup->vnodeGid[i].privateIp);
+      pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->privateIp);
+      pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort);
     }
-//    pMeta->vpeerDesc[i].vgId = htonl(pVgroup->vgId);
+    pMeta->vgroup.numOfIps++;
-    pMeta->vpeerDesc[i].dnodeId = htonl(pVgroup->vnodeGid[i].dnodeId);
+    mgmtReleaseDnode(pDnode);
   }
-  pMeta->numOfVpeers = pVgroup->numOfVnodes;
+  pMeta->vgroup.vgId = htonl(pVgroup->vgId);
 
   mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
 
@@ -1659,7 +1732,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
 }
 
 static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) {
-  SDnodeObj *pObj = clusterGetDnode(dnodeId);
+  SDnodeObj *pObj = mgmtGetDnode(dnodeId);
   SVgObj *pVgroup = mgmtGetVgroup(vnode);
 
   if (pObj == NULL || pVgroup == NULL) {
@@ -1897,7 +1970,7 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pShow->numOfRows = pDb->numOfTables;
   pShow->rowSize   = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
   return 0;
 }
 
@@ -1974,7 +2047,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
   const int32_t NUM_OF_COLUMNS = 4;
 
   mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   return numOfRows;
 }

src/mnode/src/mgmtUser.c

@@ -18,9 +18,11 @@
 #include "trpc.h"
 #include "ttime.h"
 #include "tutil.h"
-#include "taccount.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
+#include "mgmtAcct.h"
 #include "tgrant.h"
-#include "mpeer.h"
+#include "mgmtMnode.h"
 #include "mgmtSdb.h"
 #include "mgmtShell.h"
 #include "mgmtUser.h"
@@ -40,10 +42,10 @@ static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) {
 
 static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
   SUserObj *pUser = pOper->pObj;
-  SAcctObj *pAcct = acctGetAcct(pUser->acct);
+  SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
 
   if (pAcct != NULL) {
-    acctAddUser(pAcct, pUser);
+    mgmtAddUserToAcct(pAcct, pUser);
   }
   else {
     mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct);
@@ -55,16 +57,22 @@ static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) {
 
 static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) {
   SUserObj *pUser = pOper->pObj;
-  SAcctObj *pAcct = acctGetAcct(pUser->acct);
+  SAcctObj *pAcct = mgmtGetAcct(pUser->acct);
 
   if (pAcct != NULL) {
-    acctRemoveUser(pAcct, pUser);
+    mgmtDropUserFromAcct(pAcct, pUser);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) {
+  SUserObj *pUser = pOper->pObj;
+  SUserObj *pSaved = mgmtGetUser(pUser->user);
+  if (pUser != pSaved) {
+    memcpy(pSaved, pUser, pOper->rowSize);
+    free(pUser);
+  }
   return TSDB_CODE_SUCCESS;
 }
 
@@ -86,11 +94,11 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) {
 
 static int32_t mgmtUserActionRestored() {
   if (strcmp(tsMasterIp, tsPrivateIp) == 0) {
-    SAcctObj *pAcct = acctGetAcct("root");
+    SAcctObj *pAcct = mgmtGetAcct("root");
     mgmtCreateUser(pAcct, "root", "taosdata");
     mgmtCreateUser(pAcct, "monitor", tsInternalPass);
     mgmtCreateUser(pAcct, "_root", tsInternalPass);
-    acctReleaseAcct(pAcct);
+    mgmtDecAcctRef(pAcct);
   }
 
   return 0;
@@ -161,7 +169,7 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
 }
 
 int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
-  int32_t code = acctCheck(pAcct, TSDB_ACCT_USER);
+  int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
   if (code != 0) {
     return code;
   }

src/mnode/src/mgmtVgroup.c

@@ -17,14 +17,19 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tlog.h"
-#include "tbalance.h"
+#include "tutil.h"
+#include "tsocket.h"
+#include "tidpool.h"
 #include "tsync.h"
-#include "tcluster.h"
+#include "ttime.h"
-#include "mnode.h"
+#include "treplica.h"
+#include "mgmtDef.h"
+#include "mgmtLog.h"
 #include "mgmtDb.h"
 #include "mgmtDClient.h"
 #include "mgmtDServer.h"
-#include "mpeer.h"
+#include "mgmtDnode.h"
+#include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
 #include "mgmtShell.h"
@@ -53,14 +58,6 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
     tfree(pVgroup->tableList);
   }
 
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId);
-    if (pDnode) {
-      atomic_sub_fetch_32(&pDnode->openVnodes, 1);
-    }
-    clusterReleaseDnode(pDnode);
-  }
-
   tfree(pOper->pObj);
   return TSDB_CODE_SUCCESS;
 }
@@ -71,7 +68,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
   if (pDb == NULL) {
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   pVgroup->pDb = pDb;
   pVgroup->prev = NULL;
@@ -92,12 +89,12 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
   }
 
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
     if (pDnode != NULL) {
       pVgroup->vnodeGid[i].privateIp = pDnode->privateIp;
       pVgroup->vnodeGid[i].publicIp = pDnode->publicIp;
       atomic_add_fetch_32(&pDnode->openVnodes, 1);    
-      clusterReleaseDnode(pDnode);
+      mgmtReleaseDnode(pDnode);
     }     
   }
 
@@ -114,14 +111,28 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
     mgmtRemoveVgroupFromDb(pVgroup);
   }
 
-  mgmtReleaseDb(pVgroup->pDb);
+  mgmtDecDbRef(pVgroup->pDb);
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    if (pDnode) {
+      atomic_sub_fetch_32(&pDnode->openVnodes, 1);
+    }
+    mgmtReleaseDnode(pDnode);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) {
-  SVgObj  *pVgroup  = pOper->pObj;
+  SVgObj *pNew = pOper->pObj;
-  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
+  SVgObj *pVgroup = mgmtGetVgroup(pNew->vgId);
+  if (pVgroup != pNew) {
+    memcpy(pVgroup, pNew, pOper->rowSize);
+    free(pNew);
+  }
 
+  int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool);
   SDbObj *pDb = pVgroup->pDb;
   if (pDb != NULL) {
     if (pDb->cfg.maxSessions != oldTables) {
@@ -238,7 +249,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
   strcpy(pVgroup->dbName, pDb->name);
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
-  if (balanceAllocVnodes(pVgroup) != 0) {
+  if (replicaAllocVnodes(pVgroup) != 0) {
     mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
     free(pVgroup);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
@@ -375,18 +386,18 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
     pShow->pNode = pVgroup;
   }
 
-   mgmtReleaseDb(pDb);
+   mgmtDecDbRef(pDb);
 
   return 0;
 }
 
 char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) {
-  SDnodeObj *pDnode = clusterGetDnode(pVnode->dnodeId);
+  SDnodeObj *pDnode = mgmtGetDnode(pVnode->dnodeId);
   if (pDnode == NULL) {
     mError("vgroup:%d, not exist in dnode:%d", pVgroup->vgId, pDnode->dnodeId);
     return "null";
   }
-  clusterReleaseDnode(pDnode);
+  mgmtReleaseDnode(pDnode);
 
   if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     return "offline";
@@ -461,7 +472,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
   }
 
   pShow->numOfReads += numOfRows;
-  mgmtReleaseDb(pDb);
+  mgmtDecDbRef(pDb);
 
   return numOfRows;
 }
@@ -670,13 +681,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
   pCfg->dnodeId = htonl(pCfg->dnodeId);
   pCfg->vgId    = htonl(pCfg->vgId);
 
-  SDnodeObj *pDnode = clusterGetDnode(pCfg->dnodeId);
+  SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId);
   if (pDnode == NULL) {
     mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE);
     return;
   }
-  clusterReleaseDnode(pDnode);
+  mgmtReleaseDnode(pDnode);
 
   SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId);
   if (pVgroup == NULL) {

src/rpc/CMakeLists.txt

@@ -24,7 +24,7 @@ ELSEIF (TD_WINDOWS_64)
 ENDIF ()
 
 ADD_LIBRARY(trpc ${SRC})
-TARGET_LINK_LIBRARIES(trpc tutil lz4)
+TARGET_LINK_LIBRARIES(trpc tutil lz4 common)
 
 ADD_SUBDIRECTORY(test)
 

src/tsdb/src/tsdbMain.c

@@ -387,7 +387,7 @@ int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
 /**
  * Initialize a table configuration
  */
-int tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid) {
+int tsdbInitTableCfg(STableCfg *config, ETableType type, int64_t uid, int32_t tid) {
   if (config == NULL) return -1;
   if (type != TSDB_NORMAL_TABLE && type != TSDB_CHILD_TABLE) return -1;
 
@@ -732,7 +732,6 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key;
   
   pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData);
-//  pTable->mem->numOfPoints++;
 
   return 0;
 }

src/util/inc/tlog.h

@@ -177,26 +177,6 @@ extern uint32_t cdebugFlag;
     tprintf("DND QRY ", qdebugFlag, __VA_ARGS__); \
   }
 
-// mnode log function
-#define mError(...)                          \
-  if (mdebugFlag & DEBUG_ERROR) {            \
-    tprintf("ERROR MND ", 255, __VA_ARGS__); \
-  }
-#define mWarn(...)                                  \
-  if (mdebugFlag & DEBUG_WARN) {                    \
-    tprintf("WARN  MND ", mdebugFlag, __VA_ARGS__); \
-  }
-#define mTrace(...)                           \
-  if (mdebugFlag & DEBUG_TRACE) {             \
-    tprintf("MND ", mdebugFlag, __VA_ARGS__); \
-  }
-#define mPrint(...) \
-  { tprintf("MND ", 255, __VA_ARGS__); }
-
-#define mLError(...) taosLogError(__VA_ARGS__) mError(__VA_ARGS__)
-#define mLWarn(...) taosLogWarn(__VA_ARGS__) mWarn(__VA_ARGS__)
-#define mLPrint(...) taosLogPrint(__VA_ARGS__) mPrint(__VA_ARGS__)
-
 #define httpError(...)                       \
   if (httpDebugFlag & DEBUG_ERROR) {         \
     tprintf("ERROR HTP ", 255, __VA_ARGS__); \
@@ -239,25 +219,6 @@ extern uint32_t cdebugFlag;
 #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__)
 #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__)
 
-#define sdbError(...)                            \
-  if (sdbDebugFlag & DEBUG_ERROR) {              \
-    tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \
-  }
-#define sdbWarn(...)                                      \
-  if (sdbDebugFlag & DEBUG_WARN) {                        \
-    tprintf("WARN  MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
-  }
-#define sdbTrace(...)                               \
-  if (sdbDebugFlag & DEBUG_TRACE) {                 \
-    tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \
-  }
-#define sdbPrint(...) \
-  { tprintf("MND-SDB ", 255, __VA_ARGS__); }
-
-#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__)
-#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__)
-#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__)
-
 #ifdef __cplusplus
 }
 #endif

src/util/src/hash.c

@@ -101,8 +101,6 @@ static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) {
   if (pNode->next) {
     (pNode->next)->prev = pNode;
   }
-
-  pTrace("key:%s %p update hash table", pNode->key, pNode);
 }
 
 /**
@@ -153,18 +151,18 @@ static void taosHashTableResize(SHashObj *pHashObj) {
   SHashNode *pNode = NULL;
   SHashNode *pNext = NULL;
 
-  int32_t newSize = pHashObj->capacity << 1U;
+  int32_t newSize = pHashObj->capacity << 1u;
   if (newSize > HASH_MAX_CAPACITY) {
-    pTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached",
+//    pTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached",
-           pHashObj->capacity, HASH_MAX_CAPACITY);
+//           pHashObj->capacity, HASH_MAX_CAPACITY);
     return;
   }
 
-  int64_t st = taosGetTimestampUs();
+//  int64_t st = taosGetTimestampUs();
 
   SHashEntry **pNewEntry = realloc(pHashObj->hashList, sizeof(SHashEntry *) * newSize);
   if (pNewEntry == NULL) {
-    pTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
+//    pTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
     return;
   }
 
@@ -230,10 +228,9 @@ static void taosHashTableResize(SHashObj *pHashObj) {
     }
   }
 
-  int64_t et = taosGetTimestampUs();
+//  int64_t et = taosGetTimestampUs();
-
+//  pTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity,
-  pTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity,
+//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
-         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
 }
 
 /**

src/util/src/ttimer.c

@@ -562,7 +562,7 @@ void taosTmrCleanUp(void* handle) {
   pthread_mutex_unlock(&tmrCtrlMutex);
 
   if (numOfTmrCtrl <=0) {
-    pthread_cancel(athread);
+//    pthread_cancel(athread);
     
     for (int i = 0; i < tListLen(wheels); i++) {
       time_wheel_t* wheel = wheels + i;

tests/examples/c/demo.c

@@ -75,6 +75,7 @@ int main(int argc, char *argv[]) {
   
   doQuery(taos, "create database if not exists test");
   doQuery(taos, "use test");
+  doQuery(taos, "insert into tm99 values('2020-01-01 1:1:1', 99);");
 //  doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);");
 //  doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);");
 //  doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);");

tests/tsim/inc/sim.h

@@ -23,7 +23,6 @@
 #include "taos.h"
 #include "tidpool.h"
 #include "tlog.h"
-#include "tmodule.h"
 #include "tutil.h"
 
 #define MAX_MAIN_SCRIPT_NUM 10