Merge branch 'develop' into feature/query

.travis.yml

@@ -41,12 +41,34 @@ addons:
     branch_pattern: coverity_scan
 
 before_script:
-  - mkdir build
+  - mkdir debug
-  - cd build
+  - cd debug
 
 script:
   - cmake .. 
-  - cmake --build .
+  - cmake --build . || exit $?
+  - |-
+    case $TRAVIS_OS_NAME in
+      linux)
+        cd ../tests/script
+        sudo ./test.sh 2>&1 | grep 'success\|failed' | tee out.txt
+
+        total_success=`grep success out.txt | wc -l`
+
+        if [ "$total_success" -gt "0" ]; then
+          total_success=`expr $total_success - 1`
+        fi
+
+        echo "Total $total_success success"
+
+        total_failed=`grep failed out.txt | wc -l`
+        echo "Total $total_failed failed"
+
+        if [ "$total_failed" -ne "0" ]; then
+          exit $total_failed
+        fi
+        ;;
+    esac
 
 #
 # Build Matrix
@@ -58,6 +80,7 @@ matrix:
         packages:
           - build-essential
           - cmake
+          - net-tools
 
 #  - os: osx
 #    addons:

cmake/platform.inc

@@ -107,12 +107,12 @@ IF (TD_LINUX_64)
     SET(RELEASE_FLAGS "-O0")
     IF (NOT TD_ARM)
       IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
       ELSE ()
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
       ENDIF ()
     ELSE ()
-      SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
     ENDIF ()
     ADD_DEFINITIONS(-DLINUX)
     ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
@@ -128,7 +128,7 @@ IF (TD_LINUX_64)
     ENDIF ()
     SET(DEBUG_FLAGS "-O0 -DDEBUG")
     SET(RELEASE_FLAGS "-O0")
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
     ADD_DEFINITIONS(-DLINUX)
     ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
     ADD_DEFINITIONS(-DUSE_LIBICONV)
@@ -141,7 +141,7 @@ IF (TD_LINUX_64)
   ELSEIF (TD_WINDOWS_64)
     SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
     IF (NOT TD_GODLL)
-      SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+      SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
       SET(DEBUG_FLAGS "/Zi /W3 /GL")
       SET(RELEASE_FLAGS "/W0 /GL")
     ENDIF ()
@@ -151,7 +151,7 @@ IF (TD_LINUX_64)
     ADD_DEFINITIONS(-DPTW32_BUILD)
     ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
   ELSEIF (TD_DARWIN_64)
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -fPIC -malign-double -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
     SET(DEBUG_FLAGS "-O0 -DDEBUG")
     SET(RELEASE_FLAGS "-O0")
     ADD_DEFINITIONS(-DDARWIN)
@@ -159,4 +159,4 @@ IF (TD_LINUX_64)
   ELSE ()
     MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
     EXIT ()
-  ENDIF ()
+  ENDIF ()

documentation/webdocs/markdowndocs/administrator-ch.md

@@ -102,6 +102,11 @@ RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接
 
 日志文件目录，客户端和服务器的运行日志将写入该目录。
 
+**shellActivityTimer**
+- 默认值：3
+
+系统在服务端保持结果集的最长时间，范围[1-120]。
+
 **maxUsers**
 - 默认值：10,000
 
@@ -411,4 +416,4 @@ TDengine启动后，会自动创建一个监测数据库`LOG`，并自动将服
 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项`monitor`将其关闭或打开。
 
 
-[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData
+[1]: https://github.com/taosdata/TDengine/tree/develop/importSampleData

src/client/CMakeLists.txt

@@ -22,7 +22,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
 
   # generate dynamic library (*.so)
   ADD_LIBRARY(taos SHARED ${SRC})
-  TARGET_LINK_LIBRARIES(taos common trpc tutil pthread m rt)
+  TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt)
   SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
   #set version of .so

src/client/src/tscAsync.c

@@ -342,8 +342,8 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) {
   (*pSql->fp)(pSql->param, taosres, code);
 
   if (shouldFree) {
-    tscFreeSqlObj(pSql);
     tscTrace("%p Async sql is automatically freed in async res", pSql);
+    tscFreeSqlObj(pSql);
   }
 }
 

src/client/src/tscProfile.c

@@ -292,7 +292,6 @@ void tscKillConnection(STscObj *pObj) {
 
   pthread_mutex_unlock(&pObj->mutex);
 
-  taos_close(pObj);
-
   tscTrace("connection:%p is killed", pObj);
+  taos_close(pObj);
 }

src/client/src/tscSchemaUtil.c

@@ -210,7 +210,7 @@ char* tsGetTagsValue(STableMeta* pTableMeta) {
 }
 
 // todo refactor
-static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
+__attribute__ ((unused))static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     while (*input != 0 && *input++ != delim) {
     };
@@ -218,7 +218,7 @@ static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
   return input;
 }
 
-static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
+__attribute__ ((unused)) static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
   size_t len = 0;
   while (*src != delimiter && *src != 0) {
     *dst++ = *src++;

src/client/src/tscServer.c

@@ -364,8 +364,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
     (*pSql->fp)(pSql->param, taosres, rpcMsg->code);
 
     if (shouldFree) {
-      tscFreeSqlObj(pSql);
       tscTrace("%p Async sql is automatically freed", pSql);
+      tscFreeSqlObj(pSql);
     }
   }
 
@@ -522,7 +522,8 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pRetrieveMsg->free = htons(pQueryInfo->type);
   pMsg += sizeof(pQueryInfo->type);
 
-  pRetrieveMsg->header.vgId = htonl(1);
+  STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta;
+  pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId);
   pMsg += sizeof(SRetrieveTableMsg);
   
   pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen);
@@ -1801,6 +1802,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   size = tscEstimateHeartBeatMsgLength(pSql);
   if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) {
+    pthread_mutex_unlock(&pObj->mutex);
     tscError("%p failed to malloc for heartbeat msg", pSql);
     return -1;
   }

src/client/src/tscSql.c

@@ -329,7 +329,10 @@ int taos_num_fields(TAOS_RES *res) {
   }
 
   SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo;
-  return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols);
+  if (pFieldsInfo)
+    return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols);
+  else
+    return 0;
 }
 
 int taos_field_count(TAOS *taos) {
@@ -351,7 +354,11 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   if (pSql == NULL || pSql->signature != pSql) return 0;
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
-  return pQueryInfo->fieldsInfo.pFields;
+
+  if (pQueryInfo)
+    return pQueryInfo->fieldsInfo.pFields;
+  else
+    return NULL;
 }
 
 int taos_retrieve(TAOS_RES *res) {
@@ -401,6 +408,9 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
   }
 
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  if (pQueryInfo == NULL)
+    return 0;
+
   for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
     pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i);
   }
@@ -757,8 +767,8 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) {
     tscTrace("%p qhandle is null, abort free, fp:%p", pSql, pSql->fp);
     
     if (tscShouldFreeAsyncSqlObj(pSql)) {
-      tscFreeSqlObj(pSql);
       tscTrace("%p Async SqlObj is freed by app", pSql);
+      tscFreeSqlObj(pSql);
     } else {
       if (keepCmd) {
         tscFreeSqlResult(pSql);

src/client/src/tscStream.c

@@ -582,10 +582,12 @@ void taos_close_stream(TAOS_STREAM *handle) {
     tscRemoveFromStreamList(pStream, pSql);
 
     taosTmrStopA(&(pStream->pTimer));
+
+    tscTrace("%p stream:%p is closed", pSql, pStream);
+
     tscFreeSqlObj(pSql);
     pStream->pSql = NULL;
 
-    tscTrace("%p stream:%p is closed", pSql, pStream);
     tfree(pStream);
   }
 }

src/client/src/tscSub.c

@@ -108,7 +108,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
   if (pSql == NULL) {
     terrno = TSDB_CODE_CLI_OUT_OF_MEMORY;
     tscError("failed to allocate SSqlObj for subscription");
-    goto failed;
+    goto _pSql_failed;
   }
 
   pSql->signature = pSql;
@@ -137,13 +137,11 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char*
   return pSub;
 
 failed:
-  if (sqlstr != NULL) {
+  tfree(sqlstr);
-    free(sqlstr);
+
-  }
+_pSql_failed:
-  if (pSql != NULL) {
+  tfree(pSql);
-    free(pSql);
+  tfree(pSub);
-  }
-  free(pSub);
   return NULL;
 }
 

src/client/src/tscSubquery.c

@@ -1380,6 +1380,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
   } else { // all data has been retrieved to client
     tscAllDataRetrievedFromDnode(trsupport, pSql);
   }
+  pthread_mutex_unlock(&trsupport->queryMutex);
 }
 
 static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) {

src/common/src/dataformat.c

@@ -141,7 +141,7 @@ STSchema *tdDupSchema(STSchema *pSchema) {
  * Free the SSchema object created by tdNewSchema or tdDupSchema
  */
 void tdFreeSchema(STSchema *pSchema) {
-  if (pSchema == NULL) free(pSchema);
+  if (pSchema != NULL) free(pSchema);
 }
 
 /**

src/common/src/name.c

@@ -6,7 +6,7 @@
 #include "ttokendef.h"
 
 // todo refactor
-static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) {
+__attribute__((unused)) static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     while (*input != 0 && *input++ != delim) {
     };
@@ -14,7 +14,7 @@ static FORCE_INLINE const char* skipSegments(const char* input, char delim, int3
   return input;
 }
 
-static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
+__attribute__((unused)) static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
   size_t len = 0;
   while (*src != delimiter && *src != 0) {
     *dst++ = *src++;

src/dnode/CMakeLists.txt

@@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   ENDIF ()
   
   IF (TD_VPEER)
-    TARGET_LINK_LIBRARIES(taosd balance)
+    TARGET_LINK_LIBRARIES(taosd balance sync)
   ENDIF ()
 
   SET(PREPARE_ENV_CMD "prepare_env_cmd")

src/dnode/src/dnodeMain.c

@@ -28,7 +28,7 @@
 #include "dnodeRead.h"
 #include "dnodeShell.h"
 #include "dnodeWrite.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 
 static int32_t dnodeInitSystem();
 static int32_t dnodeInitStorage();
@@ -220,7 +220,6 @@ static int32_t dnodeInitStorage() {
   sprintf(tsMnodeDir, "%s/mnode", dataDir);
   sprintf(tsVnodeDir, "%s/vnode", dataDir);
   sprintf(tsDnodeDir, "%s/dnode", dataDir);
-  mkdir(tsMnodeDir, 0755);
   mkdir(tsVnodeDir, 0755);
   mkdir(tsDnodeDir, 0755);
 

src/dnode/src/dnodeMgmt.c

@@ -20,7 +20,6 @@
 #include "taosmsg.h"
 #include "tlog.h"
 #include "trpc.h"
-#include "tstatus.h"
 #include "tsdb.h"
 #include "ttime.h"
 #include "ttimer.h"
@@ -132,7 +131,7 @@ static int32_t dnodeOpenVnodes() {
   char vnodeDir[TSDB_FILENAME_LEN * 3];
   int32_t failed = 0;
 
-  int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000);
+  int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES);
   int32_t  numOfVnodes = dnodeGetVnodeList(vnodeList);
 
   for (int32_t i = 0; i < numOfVnodes; ++i) {
@@ -147,7 +146,7 @@ static int32_t dnodeOpenVnodes() {
 }
 
 static void dnodeCloseVnodes() {
-  int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * 10000);
+  int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES);
   int32_t  numOfVnodes = dnodeGetVnodeList(vnodeList);
 
   for (int32_t i = 0; i < numOfVnodes; ++i) {

src/dnode/src/dnodeRead.c

@@ -15,38 +15,25 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tlog.h"
 #include "tqueue.h"
 #include "trpc.h"
-
 #include "twal.h"
 #include "dnodeMgmt.h"
 #include "dnodeRead.h"
-#include "queryExecutor.h"
 #include "vnode.h"
 
 typedef struct {
-  int32_t  code;
+  SRspRet  rspRet;
-  int32_t  count;
+  void    *pCont;
-  int32_t  numOfVnodes;
+  int32_t  contLen;
-} SRpcContext;
+  SRpcMsg  rpcMsg;
-
-typedef struct {
-  void        *pCont;
-  int32_t      contLen;
-  SRpcMsg      rpcMsg;
-  SRpcContext *pRpcContext;  // RPC message context
 } SReadMsg;
 
 static void *dnodeProcessReadQueue(void *param);
-static void  dnodeProcessReadResult(void *pVnode, SReadMsg *pRead);
 static void  dnodeHandleIdleReadWorker();
-static void  dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg);
-static void  dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg);
-static void(*dnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(void *pVnode, SReadMsg *pNode);
 
 // module global variable
 static taos_qset readQset;
@@ -55,14 +42,11 @@ static int32_t   maxThreads;
 static int32_t   minThreads;
 
 int32_t dnodeInitRead() {
-  dnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY]    = dnodeProcessQueryMsg;
-  dnodeProcessReadMsgFp[TSDB_MSG_TYPE_RETRIEVE] = dnodeProcessRetrieveMsg;
-
   readQset = taosOpenQset();
 
   minThreads = 3;
-  maxThreads = tsNumOfCores*tsNumOfThreadsPerCore;
+  maxThreads = tsNumOfCores * tsNumOfThreadsPerCore;
-  if (maxThreads <= minThreads*2) maxThreads = 2*minThreads;
+  if (maxThreads <= minThreads * 2) maxThreads = 2 * minThreads;
 
   dPrint("dnode read is opened");
   return 0;
@@ -77,7 +61,6 @@ void dnodeRead(SRpcMsg *pMsg) {
   int32_t     queuedMsgNum = 0;
   int32_t     leftLen      = pMsg->contLen;
   char        *pCont       = (char *) pMsg->pCont;
-  SRpcContext *pRpcContext = NULL;
   void        *pVnode;  
 
   dTrace("dnode %s msg incoming, thandle:%p", taosMsg[pMsg->msgType], pMsg->handle);
@@ -105,7 +88,6 @@ void dnodeRead(SRpcMsg *pMsg) {
     pRead->rpcMsg      = *pMsg;
     pRead->pCont       = pCont;
     pRead->contLen     = pHead->contLen;
-    pRead->pRpcContext = pRpcContext;
 
     taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
 
@@ -156,6 +138,34 @@ void dnodeFreeRqueue(void *rqueue) {
   // dynamically adjust the number of threads
 }
 
+static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) {  
+  SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
+  pRead->rpcMsg      = pMsg->rpcMsg;
+  pRead->pCont       = qhandle;
+  pRead->contLen     = 0;
+  pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+  
+  taos_queue queue = vnodeGetRqueue(pVnode);
+  taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
+}
+
+void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) {
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+  if (code == TSDB_CODE_ACTION_NEED_REPROCESSED) {
+    dnodeContinueExecuteQuery(pVnode, pRead->rspRet.qhandle, pRead);
+  }
+
+  SRpcMsg rpcRsp = {
+    .handle  = pRead->rpcMsg.handle,
+    .pCont   = pRead->rspRet.rsp,
+    .contLen = pRead->rspRet.len,
+    .code    = pRead->rspRet.code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+  rpcFreeCont(pRead->rpcMsg.pCont);
+}
+
 static void *dnodeProcessReadQueue(void *param) {
   taos_qset  qset = (taos_qset)param;
   SReadMsg  *pReadMsg;
@@ -168,13 +178,8 @@ static void *dnodeProcessReadQueue(void *param) {
       continue;
     }
 
-    terrno = 0;
+    int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet);
-    if (dnodeProcessReadMsgFp[pReadMsg->rpcMsg.msgType]) {
+    dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
-      (*dnodeProcessReadMsgFp[pReadMsg->rpcMsg.msgType]) (pVnode, pReadMsg);
-    } else {
-      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-    }
-
     taosFreeQitem(pReadMsg);
   }
 
@@ -193,118 +198,3 @@ static void dnodeHandleIdleReadWorker() {
   }
 }
 
-UNUSED_FUNC
-static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) {
-  SRpcContext *pRpcContext = pRead->pRpcContext;
-  int32_t      code = 0;
-
-  if (pRpcContext) {
-    if (terrno) {
-      if (pRpcContext->code == 0) pRpcContext->code = terrno;
-    }
-
-    int32_t count = atomic_add_fetch_32(&pRpcContext->count, 1);
-    if (count < pRpcContext->numOfVnodes) {
-      // not over yet, multiple vnodes
-      return;
-    }
-
-    // over, result can be merged now
-    code = pRpcContext->code;
-  } else {
-    code = terrno;
-  }
-
-  //TODO: query handle is returned by dnodeProcessQueryMsg
-  if (0) {
-    SRpcMsg rsp;
-    rsp.handle = pRead->rpcMsg.handle;
-    rsp.code   = code;
-    rsp.pCont  = NULL;
-    rpcSendResponse(&rsp);
-  }
-  
-  rpcFreeCont(pRead->rpcMsg.pCont);  // free the received message
-}
-
-static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMsg) {
-  
-  SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg));
-  pRead->rpcMsg      = pMsg->rpcMsg;
-  pRead->pCont       = qhandle;
-  pRead->contLen     = 0;
-  pRead->pRpcContext = pMsg->pRpcContext;
-  pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
-  
-  taos_queue queue = vnodeGetRqueue(pVnode);
-  taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
-}
-
-static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) {
-  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont;
-  
-  SQInfo* pQInfo = NULL;
-  if (pMsg->contLen != 0) {
-    void* tsdb = vnodeGetTsdb(pVnode);
-    int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo);
-  
-    SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
-    pRsp->code    = code;
-    pRsp->qhandle = htobe64((uint64_t) (pQInfo));
-  
-    SRpcMsg rpcRsp = {
-        .handle = pMsg->rpcMsg.handle,
-        .pCont = pRsp,
-        .contLen = sizeof(SQueryTableRsp),
-        .code = code,
-        .msgType = 0
-    };
-  
-    rpcSendResponse(&rpcRsp);
-    dTrace("dnode query msg disposed, thandle:%p", pMsg->rpcMsg.handle);
-  } else {
-    pQInfo = pMsg->pCont;
-  }
-  
-  qTableQuery(pQInfo); // do execute query
-}
-
-static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) {
-  SRetrieveTableMsg *pRetrieve = pMsg->pCont;
-  void *pQInfo = (void*) htobe64(pRetrieve->qhandle);
-
-  dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId);
-  int32_t contLen = 0;
-  
-  SRetrieveTableRsp *pRsp = NULL;
-  
-  int32_t code = qRetrieveQueryResultInfo(pQInfo);
-  if (code != TSDB_CODE_SUCCESS) {
-    contLen = sizeof(SRetrieveTableRsp);
-
-    pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen);
-    memset(pRsp, 0, sizeof(SRetrieveTableRsp));
-  } else {
-    // todo check code and handle error in build result set
-    code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen);
-    
-    if (qHasMoreResultsToRetrieve(pQInfo)) {
-      dnodeContinueExecuteQuery(pVnode, pQInfo, pMsg);
-    } else {  // no further execution invoked, release the ref to vnode
-      qDestroyQueryInfo(pQInfo);
-//      dnodeProcessReadResult(pVnode, pMsg);
-      vnodeRelease(pVnode);
-    }
-  }
-  
-  SRpcMsg rpcRsp = (SRpcMsg) {
-      .handle = pMsg->rpcMsg.handle,
-      .pCont = pRsp,
-      .contLen = contLen,
-      .code = code,
-      .msgType = 0
-  };
-
-  rpcSendResponse(&rpcRsp);
-  dTrace("dnode retrieve msg disposed, thandle:%p", pMsg->rpcMsg.handle);
-}

src/inc/dnode.h

@@ -42,7 +42,7 @@ void *dnodeAllocateWqueue(void *pVnode);
 void  dnodeFreeWqueue(void *queue);
 void *dnodeAllocateRqueue(void *pVnode);
 void  dnodeFreeRqueue(void *rqueue);
-void  dnodeSendWriteResponse(void *pVnode, void *param, int32_t code);
+void  dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code);
 
 #ifdef __cplusplus
 }

src/inc/mnode.h

@@ -21,7 +21,6 @@ extern "C" {
 #endif
 
 #include "os.h"
-
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "taoserror.h"
@@ -40,7 +39,8 @@ extern "C" {
 
 struct _vg_obj;
 struct _db_obj;
-struct _acctObj;
+struct _acct_obj;
+struct _user_obj;
 
 typedef struct {
   int32_t  mnodeId;
@@ -65,7 +65,7 @@ typedef struct {
   void    *pSync;
 } SMnodeObj;
 
-typedef struct {
+typedef struct _dnode_obj {
   int32_t    dnodeId;
   uint32_t   privateIp;
   uint32_t   publicIp;
@@ -79,16 +79,15 @@ typedef struct {
   uint16_t   slot;
   uint16_t   numOfCores;       // from dnode status msg
   int8_t     alternativeRole;  // from dnode status msg, 0-any, 1-mgmt, 2-dnode
-  int8_t     lbStatus;         // set in balance function
+  int8_t     status;           // set in balance function
-  float      lbScore;          // calc in balance function
   int32_t    customScore;      // config by user
   char       dnodeName[TSDB_DNODE_NAME_LEN + 1];
   int8_t     reserved[15];
   int8_t     updateEnd[1];
   int32_t    refCount;
   SVnodeLoad vload[TSDB_MAX_VNODES];
-  int32_t    status;
   uint32_t   lastReboot;       // time stamp for last reboot
+  float      score;          // calc in balance function
   float      diskAvailable;    // from dnode status msg
   int16_t    diskAvgUsage;     // calc from sys.disk
   int16_t    cpuAvgUsage;      // calc from sys.cpu
@@ -105,10 +104,10 @@ typedef struct {
 typedef struct {
   char   tableId[TSDB_TABLE_ID_LEN + 1];
   int8_t type;
-} STableInfo;
+} STableObj;
 
 typedef struct SSuperTableObj {
-  STableInfo info;
+  STableObj  info;
   uint64_t   uid;
   int64_t    createdTime;
   int32_t    sversion;
@@ -123,7 +122,7 @@ typedef struct SSuperTableObj {
 } SSuperTableObj;
 
 typedef struct {
-  STableInfo info;
+  STableObj  info;
   uint64_t   uid;
   int64_t    createdTime;
   int32_t    sversion;     //used by normal table
@@ -147,10 +146,11 @@ typedef struct _vg_obj {
   int64_t         createdTime;
   SVnodeGid       vnodeGid[TSDB_VNODES_SUPPORT];
   int32_t         numOfVnodes;
-  int32_t         lbIp;
+  int32_t         lbDnodeId;
   int32_t         lbTime;
-  int8_t          lbStatus;
+  int8_t          status;
-  int8_t          reserved[14];
+  int8_t          inUse;
+  int8_t          reserved[13];
   int8_t          updateEnd[1];
   int32_t         refCount;
   struct _vg_obj *prev, *next;
@@ -162,7 +162,7 @@ typedef struct _vg_obj {
 
 typedef struct _db_obj {
   char    name[TSDB_DB_NAME_LEN + 1];
-  int8_t  dirty;
+  int8_t  status;
   int64_t createdTime;
   SDbCfg  cfg;
   int8_t  reserved[15];
@@ -173,7 +173,7 @@ typedef struct _db_obj {
   int32_t numOfSuperTables;
   SVgObj *pHead;
   SVgObj *pTail;
-  struct _acctObj *pAcct;
+  struct _acct_obj *pAcct;
 } SDbObj;
 
 typedef struct _user_obj {
@@ -186,7 +186,7 @@ typedef struct _user_obj {
   int8_t            reserved[13];
   int8_t            updateEnd[1];
   int32_t           refCount;
-  struct _acctObj * pAcct;
+  struct _acct_obj * pAcct;
   SQqueryList *     pQList;  // query list
   SStreamList *     pSList;  // stream list
 } SUserObj;
@@ -209,7 +209,7 @@ typedef struct {
   int8_t  accessState;   // Checked by mgmt heartbeat message
 } SAcctInfo;
 
-typedef struct _acctObj {
+typedef struct _acct_obj {
   char      user[TSDB_USER_LEN + 1];
   char      pass[TSDB_KEY_LEN + 1];
   SAcctCfg  cfg;
@@ -244,6 +244,8 @@ typedef struct {
   int8_t   received;
   int8_t   successed;
   int8_t   expected;
+  int8_t   retry;
+  int8_t   maxRetry;
   int32_t  contLen;
   int32_t  code;
   void     *ahandle;
@@ -254,7 +256,7 @@ typedef struct {
   SUserObj *pUser;
   SDbObj   *pDb;
   SVgObj   *pVgroup;
-  STableInfo *pTable;
+  STableObj *pTable;
 } SQueuedMsg;
 
 int32_t mgmtInitSystem();

src/inc/taccount.h

@@ -13,31 +13,34 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_ACCT_H
+#ifndef TDENGINE_ACCT_H
-#define TDENGINE_MGMT_ACCT_H
+#define TDENGINE_ACCT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
 
+struct _acct_obj;
+struct _user_obj;
+struct _db_obj;
+ 
 typedef enum {
   TSDB_ACCT_USER,
   TSDB_ACCT_DB,
   TSDB_ACCT_TABLE
 } EAcctGrantType;
 
-int32_t   acctInit();
+int32_t acctInit();
-void      acctCleanUp();
+void    acctCleanUp();
-SAcctObj *acctGetAcct(char *acctName);
+void   *acctGetAcct(char *acctName);
-void      acctIncRef(SAcctObj *pAcct);
+void    acctIncRef(struct _acct_obj *pAcct);
-void      acctDecRef(SAcctObj *pAcct);
+void    acctReleaseAcct(struct _acct_obj *pAcct);
-int32_t   acctCheck(SAcctObj *pAcct, EAcctGrantType type);
+int32_t acctCheck(struct _acct_obj *pAcct, EAcctGrantType type);
 
-void      acctAddDb(SAcctObj *pAcct, SDbObj *pDb);
+void    acctAddDb(struct _acct_obj *pAcct, struct _db_obj *pDb);
-void      acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb);
+void    acctRemoveDb(struct _acct_obj *pAcct, struct _db_obj *pDb);
-void      acctAddUser(SAcctObj *pAcct, SUserObj *pUser);
+void    acctAddUser(struct _acct_obj *pAcct, struct _user_obj *pUser);
-void      acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser);
+void    acctRemoveUser(struct _acct_obj *pAcct, struct _user_obj *pUser);
 
 #ifdef __cplusplus
 }

src/inc/tadmin.h

@@ -13,18 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_BALANCE_H
+#ifndef TDENGINE_ADMIN_H
-#define TDENGINE_MGMT_BALANCE_H
+#define TDENGINE_ADMIN_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
 
-int32_t mgmtInitBalance();
+#include <stdint.h>
-void    mgmtCleanupBalance();
+#include <stdbool.h>
-void    mgmtBalanceNotify() ;
+
-int32_t mgmtAllocVnodes(SVgObj *pVgroup);
+void adminInit();
+
+struct _http_server_obj_;
+
+extern void (*adminInitHandleFp)(struct _http_server_obj_* pServer);
+extern void (*opInitHandleFp)(struct _http_server_obj_* pServer);
 
 #ifdef __cplusplus
 }

src/inc/taoserror.h

@@ -47,6 +47,7 @@ static STaosError errors[] = {
 
 // rpc
 TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS,         0, 1, "action in progress")
+TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_NEED_REPROCESSED,    0, 3, "action need to be reprocessed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED,          0, 4, "message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_ALREADY_PROCESSED,          0, 5, "message already processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_REDIRECT,                   0, 6, "redirect")
@@ -75,89 +76,89 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE,               0, 28, "node offline")
 TAOS_DEFINE_ERROR(TSDB_CODE_NETWORK_UNAVAIL,            0, 29, "network unavailable")
 
 // db & user
-TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED,            0, 30, "db not selected")
+TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED,            0, 100, "db not selected")
-TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST,           0, 31, "database aleady exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_DB_ALREADY_EXIST,           0, 101, "database aleady exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB,                 0, 32, "invalid database")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DB,                 0, 102, "invalid database")
-TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN,       0, 33, "monitor db forbidden")
+TAOS_DEFINE_ERROR(TSDB_CODE_MONITOR_DB_FORBIDDEN,       0, 103, "monitor db forbidden")
-TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST,         0, 34, "user already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_USER_ALREADY_EXIST,         0, 104, "user already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER,               0, 35, "invalid user")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_USER,               0, 105, "invalid user")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS,               0, 36, "invalid password")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PASS,               0, 106, "invalid password")
 
 // table
-TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST,        0, 41, "table already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ALREADY_EXIST,        0, 200, "table already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID,           0, 42, "invalid table id")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_ID,           0, 201, "invalid table id")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE,         0, 43, "invalid table typee")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE_TYPE,         0, 202, "invalid table typee")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE,              0, 44, "invalid table name")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE,              0, 203, "invalid table name")
-TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE,            0, 45, "no super table")           // operation only available for super table
+TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE,            0, 204, "no super table")           // operation only available for super table
-TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE,           0, 46, "not active table")
+TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE,           0, 205, "not active table")
-TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH,          0, 47, "table id mismatch")
+TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH,          0, 206, "table id mismatch")
 
 // dnode & mnode
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES,           0, 50, "no enough dnodes")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES,           0, 300, "no enough dnodes")
-TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST,        0, 51, "dnode already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST,        0, 301, "dnode already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST,            0, 52, "dnode not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST,            0, 302, "dnode not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER,                  0, 53, "no master")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER,                  0, 303, "no master")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER,           0, 54, "no remove master")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER,           0, 304, "no remove master")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID,           0, 55, "invalid query id")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID,           0, 305, "invalid query id")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID,          0, 56, "invalid stream id")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID,          0, 306, "invalid stream id")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION,         0, 57, "invalid connection")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION,         0, 307, "invalid connection")
-TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR,                  0, 58, "sdb error")
+TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR,                  0, 308, "sdb error")
 
 // acct
-TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST,         0, 60, "accounts already exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST,         0, 400, "accounts already exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT,               0, 61, "invalid account")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT,               0, 401, "invalid account")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER,     0, 62, "invalid account parameter")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER,     0, 402, "invalid account parameter")
-TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS,             0, 63, "too many accounts")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS,             0, 403, "too many accounts")
-TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS,             0, 64, "too many users")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS,             0, 404, "too many users")
-TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES,            0, 65, "too many tables")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES,            0, 405, "too many tables")
-TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES,         0, 66, "too many databases")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES,         0, 406, "too many databases")
-TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES,       0, 67, "not enough time series")
+TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES,       0, 407, "not enough time series")
 
 // grant
-TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE,               0, 70, "auth failure")
+TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE,               0, 400, "auth failure")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS,                  0, 71, "no rights")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS,                  0, 401, "no rights")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS,            0, 72, "no write access")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS,            0, 402, "no write access")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS,             0, 73, "no read access")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS,             0, 403, "no read access")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,              0, 74, "grant expired")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,              0, 404, "grant expired")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED,        0, 75, "grant dnode limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED,        0, 405, "grant dnode limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED,         0, 76, "grant account limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED,         0, 406, "grant account limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED,   0, 77, "grant timeseries limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED,   0, 407, "grant timeseries limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED,           0, 78, "grant db limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED,           0, 408, "grant db limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED,         0, 79, "grant user limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED,         0, 409, "grant user limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED,         0, 80, "grant conn limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED,         0, 410, "grant conn limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED,       0, 81, "grant stream limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED,       0, 411, "grant stream limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED,        0, 82, "grant speed limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED,        0, 412, "grant speed limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED,      0, 83, "grant storage limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED,      0, 413, "grant storage limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED,    0, 84, "grant query time limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED,    0, 414, "grant query time limited")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,          0, 85, "grant cpu limited")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,          0, 415, "grant cpu limited")
 
 // server
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID,          0, 90, "invalid vgroup id")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID,          0, 500, "invalid vgroup id")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID,           0, 91, "invalid vnode id")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID,           0, 501, "invalid vnode id")
-TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE,           0, 92, "not active vnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE,           0, 502, "not active vnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED,             0, 93, "vg init failed")
+TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED,             0, 503, "vg init failed")
-TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE,          0, 94, "server no diskspace")
+TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE,          0, 504, "server no diskspace")
-TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY,         0, 95, "server out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY,         0, 505, "server out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS,        0, 96, "no disk permissions")
+TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS,        0, 506, "no disk permissions")
-TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,             0, 97, "file corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,             0, 507, "file corrupted")
-TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,           0, 98, "memory corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,           0, 508, "memory corrupted")
 
 // client
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION,     0, 101, "invalid client version")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION,     0, 601, "invalid client version")
-TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY,          0, 102, "client out of memory")
+TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY,          0, 602, "client out of memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE,           0, 103, "client no disk space")
+TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE,           0, 603, "client no disk space")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP,         0, 104, "invalid timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP,         0, 604, "invalid timestamp")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL,                0, 105, "invalid sql")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL,                0, 605, "invalid sql")
-TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED,         0, 106, "query cache erased")
+TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED,         0, 606, "query cache erased")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG,          0, 107, "invalid query message")    // failed to validate the sql expression msg by vnode
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG,          0, 607, "invalid query message")    // failed to validate the sql expression msg by vnode
-TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY,        0, 108, "sorted res too many")      // too many result for ordered super table projection query
+TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY,        0, 608, "sorted res too many")      // too many result for ordered super table projection query
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE,            0, 109, "invalid handle")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE,            0, 609, "invalid handle")
-TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED,            0, 110, "query cancelled")
+TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED,            0, 610, "query cancelled")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE,                 0, 111, "invalid ie")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE,                 0, 611, "invalid ie")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE,              0, 112, "invalid value")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE,              0, 612, "invalid value")
 
 // others
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT,        0, 120, "invalid file format")
+TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT,        0, 700, "invalid file format")
 
 
 #ifdef TAOS_ERROR_C

src/inc/taosmsg.h

@@ -508,12 +508,11 @@ typedef struct SRetrieveTableRsp {
 
 typedef struct {
   int32_t vgId;
-  int32_t vnode;
   int64_t totalStorage;
   int64_t compStorage;
   int64_t pointsWritten;
   uint8_t status;
-  uint8_t syncStatus;
+  uint8_t role;
   uint8_t accessState;
   uint8_t reserved[5];
 } SVnodeLoad;
@@ -751,12 +750,12 @@ typedef struct {
 
 typedef struct {
   int32_t    numOfQueries;
-  SQueryDesc qdesc[];
+  SQueryDesc *qdesc;
 } SQqueryList;
 
 typedef struct {
   int32_t     numOfStreams;
-  SStreamDesc sdesc[];
+  SStreamDesc *sdesc;
 } SStreamList;
 
 typedef struct {

src/inc/tbalance.h

@@ -13,22 +13,26 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DNODE_H
+#ifndef TDENGINE_BALANCE_H
-#define TDENGINE_MGMT_DNODE_H
+#define TDENGINE_BALANCE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mnode.h"
 
-int32_t mgmtInitDnodes();
+#include <stdint.h>
-void    mgmtCleanUpDnodes();
+#include <stdbool.h>
-int32_t mgmtGetDnodesNum();
+#include <pthread.h>
-void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
+
-void    mgmtIncDnodeRef(SDnodeObj *pDnode);
+struct _db_obj;
-void    mgmtDecDnodeRef(SDnodeObj *pDnode);
+struct _vg_obj;
-SDnodeObj* mgmtGetDnode(int32_t dnodeId);
+struct _dnode_obj;
-SDnodeObj* mgmtGetDnodeByIp(uint32_t ip);
+
+int32_t balanceInit();
+void    balanceCleanUp();
+void    balanceNotify();
+int32_t balanceAllocVnodes(struct _vg_obj *pVgroup);
+int32_t balanceDropDnode(struct _dnode_obj *pDnode);
 
 #ifdef __cplusplus
 }

src/inc/tcluster.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_CLUSTER_H
+#define TDENGINE_CLUSTER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <pthread.h>
+
+struct _dnode_obj;
+
+enum _TAOS_DN_STATUS {
+  TAOS_DN_STATUS_OFFLINE,
+  TAOS_DN_STATUS_DROPPING,
+  TAOS_DN_STATUS_BALANCING,
+  TAOS_DN_STATUS_READY
+};
+
+int32_t clusterInit();
+void    clusterCleanUp();
+char*   clusterGetDnodeStatusStr(int32_t dnodeStatus);
+bool    clusterCheckModuleInDnode(struct _dnode_obj *pDnode, int moduleType);
+
+int32_t clusterInitDnodes();
+void    clusterCleanupDnodes();
+int32_t clusterGetDnodesNum();
+void *  clusterGetNextDnode(void *pNode, struct _dnode_obj **pDnode);
+void    clusterReleaseDnode(struct _dnode_obj *pDnode);
+void *  clusterGetDnode(int32_t dnodeId);
+void *  clusterGetDnodeByIp(uint32_t ip);
+void    clusterUpdateDnode(struct _dnode_obj *pDnode);
+int32_t clusterDropDnode(struct _dnode_obj *pDnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/inc/tgrant.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_GRANT_H
+#ifndef TDENGINE_GTANT_H
-#define TDENGINE_MGMT_GTANT_H
+#define TDENGINE_GTANT_H
 
 #ifdef __cplusplus
 "C" {

src/inc/tsync.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_SYNC_H
+#define TDENGINE_SYNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TAOS_SYNC_MAX_REPLICA 5
+
+typedef enum _TAOS_SYNC_ROLE {
+  TAOS_SYNC_ROLE_OFFLINE,
+  TAOS_SYNC_ROLE_UNSYNCED,
+  TAOS_SYNC_ROLE_SLAVE,
+  TAOS_SYNC_ROLE_MASTER,
+} ESyncRole;
+
+typedef enum _TAOS_SYNC_STATUS {
+  TAOS_SYNC_STATUS_INIT,
+  TAOS_SYNC_STATUS_START,
+  TAOS_SYNC_STATUS_FILE,
+  TAOS_SYNC_STATUS_CACHE,
+} ESyncStatus;
+
+typedef struct {
+  uint32_t  nodeId;    // node ID assigned by TDengine
+  uint32_t  nodeIp;    // node IP address
+  char      name[TSDB_FILENAME_LEN]; // external node name 
+} SNodeInfo;
+
+typedef struct {
+  uint32_t   arbitratorIp;  // arbitrator IP address
+  int8_t     quorum;    // number of confirms required, >=1 
+  int8_t     replica;   // number of replications, >=1
+  SNodeInfo  nodeInfo[TAOS_SYNC_MAX_REPLICA];
+} SSyncCfg;
+
+typedef struct {
+  int       selfIndex;
+  uint32_t  nodeId[TAOS_SYNC_MAX_REPLICA];
+  int       role[TAOS_SYNC_MAX_REPLICA];  
+} SNodesRole;
+ 
+typedef struct {
+  int32_t    vgId;      // vgroup ID
+  uint64_t   version;   // initial version
+  SSyncCfg   syncCfg;   // configuration from mgmt
+  char       path[128]; // path to the file
+ 
+  void      *ahandle;   // handle provided by APP 
+
+  // if name is null, get the file from index or after, used by master
+  // if name is provided, get the named file at the specified index, used by unsynced node
+  // it returns the file magic number and size, if file not there, magic shall be 0.
+  uint32_t   (*getFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); 
+
+  // get the wal file from index or after
+  // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
+  int        (*getWalInfo)(void *ahandle, char *name, uint32_t *index); 
+ 
+  // when a forward pkt is received, call this to handle data 
+  int        (*writeToCache)(void *ahandle, void *pHead, int type);
+
+  // when forward is confirmed by peer, master call this API to notify app
+  void       (*confirmForward)(void *ahandle, void *mhandle, int32_t code);
+
+  // when role is changed, call this to notify app
+  void       (*notifyRole)(void *ahandle, int8_t role);
+} SSyncInfo;
+
+typedef void* tsync_h;
+
+tsync_h syncStart(SSyncInfo *);
+void    syncStop(tsync_h shandle);
+int     syncReconfig(tsync_h shandle, SSyncCfg *);
+int     syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle);
+void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
+void    syncRecover(tsync_h shandle);      // recover from other nodes:
+int     syncGetNodesRole(tsync_h shandle, SNodesRole *);
+
+extern  char *syncRole[];
+
+//global configurable parameters
+extern  int   tsMaxSyncNum;
+extern  int   tsSyncTcpThreads;
+extern  int   tsMaxWatchFiles;
+extern  short tsSyncPort;
+extern  int   tsSyncTimer;
+extern  int   tsMaxFwdInfo; 
+extern  int   sDebugFlag;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SYNC_H

src/inc/twal.h

@@ -33,14 +33,19 @@ typedef struct {
   char      cont[];
 } SWalHead;
 
+typedef struct {
+  int8_t    commitLog; // commitLog
+  int8_t    wals;      // number of WAL files;
+} SWalCfg;
+
 typedef void* twal_h;  // WAL HANDLE
 
-twal_h  walOpen(char *path, int max, int level);
+twal_h  walOpen(char *path, SWalCfg *pCfg);
 void    walClose(twal_h);
 int     walRenew(twal_h);
 int     walWrite(twal_h, SWalHead *);
 void    walFsync(twal_h);
-int     walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, SWalHead *pHead, int type));
+int     walRestore(twal_h, void *pVnode, int (*writeFp)(void *ahandle, void *pHead, int type));
 int     walGetWalFile(twal_h, char *name, uint32_t *index);
 
 extern int wDebugFlag;

src/inc/vnode.h

@@ -20,9 +20,19 @@
 extern "C" {
 #endif
 
+typedef enum _VN_STATUS {
+  TAOS_VN_STATUS_INIT,
+  TAOS_VN_STATUS_CREATING,
+  TAOS_VN_STATUS_READY,
+  TAOS_VN_STATUS_CLOSING,
+  TAOS_VN_STATUS_DELETING,
+} EVnStatus;
+
 typedef struct {
   int   len;
+  int   code;
   void *rsp;
+  void *qhandle; //used by query and retrieve msg
 } SRspRet;
 
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg);
@@ -39,9 +49,11 @@ void*   vnodeGetWqueue(int32_t vgId);
 void*   vnodeGetWal(void *pVnode);
 void*   vnodeGetTsdb(void *pVnode);
 
-int32_t vnodeProcessWrite(void *pVnode, int qtype, SWalHead *pHead, void *item);
+int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
 void    vnodeBuildStatusMsg(void * param);
 
+int32_t vnodeProcessRead(void *pVnode, int msgType, void *pCont, int32_t contLen, SRspRet *ret);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtDb.h

@@ -22,13 +22,18 @@ extern "C" {
 
 #include "mnode.h"
 
+enum _TSDB_DB_STATUS {
+  TSDB_DB_STATUS_READY,
+  TSDB_DB_STATUS_DROPPING
+};
+
 // api
 int32_t mgmtInitDbs();
 void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
 void    mgmtIncDbRef(SDbObj *pDb);
-void    mgmtDecDbRef(SDbObj *pDb);
+void    mgmtReleaseDb(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);
 void    mgmtDropAllDbs(SAcctObj *pAcct);
 

src/mnode/inc/mgmtMnode.h

@@ -20,6 +20,19 @@
 extern "C" {
 #endif
 
+enum _TSDB_MN_STATUS {
+  TSDB_MN_STATUS_OFFLINE,
+  TSDB_MN_STATUS_UNSYNCED,
+  TSDB_MN_STATUS_SYNCING,
+  TSDB_MN_STATUS_SERVING
+};
+
+enum _TSDB_MN_ROLE {
+  TSDB_MN_ROLE_UNDECIDED,
+  TSDB_MN_ROLE_SLAVE,
+  TSDB_MN_ROLE_MASTER
+};
+
 int32_t mgmtInitMnodes();
 void    mgmtCleanupMnodes();
 

src/mnode/inc/mgmtShell.h

@@ -31,6 +31,7 @@ void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp);
 void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
 
 void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
+void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg);
 void mgmtSendSimpleResp(void *thandle, int32_t code);
 
 #ifdef __cplusplus

src/mnode/inc/mgmtTable.h

@@ -27,7 +27,7 @@ extern "C" {
 
 int32_t     mgmtInitTables();
 void        mgmtCleanUpTables();
-STableInfo* mgmtGetTable(char* tableId);
+STableObj*  mgmtGetTable(char* tableId);
 void        mgmtIncTableRef(void *pTable); 
 void        mgmtDecTableRef(void *pTable);
 void        mgmtDropAllChildTables(SDbObj *pDropDb);

src/mnode/inc/mgmtUser.h

@@ -24,8 +24,7 @@ extern "C" {
 int32_t   mgmtInitUsers();
 void      mgmtCleanUpUsers();
 SUserObj *mgmtGetUser(char *name);
-void      mgmtIncUserRef(SUserObj *pUser);
+void      mgmtReleaseUser(SUserObj *pUser);
-void      mgmtDecUserRef(SUserObj *pUser);
 SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
 int32_t   mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
 void      mgmtDropAllUsers(SAcctObj *pAcct);

src/mnode/inc/mgmtVgroup.h

@@ -24,13 +24,21 @@ extern "C" {
 #include <stdbool.h>
 #include "mnode.h"
 
+enum _TSDB_VG_STATUS {
+  TSDB_VG_STATUS_READY,
+  TSDB_VG_STATUS_UPDATE
+};
+
 int32_t mgmtInitVgroups();
 void    mgmtCleanUpVgroups();
 SVgObj *mgmtGetVgroup(int32_t vgId);
-void    mgmtIncVgroupRef(SVgObj *pVgroup);
+void    mgmtReleaseVgroup(SVgObj *pVgroup);
-void    mgmtDecVgroupRef(SVgObj *pVgroup);
 void    mgmtDropAllVgroups(SDbObj *pDropDb);
 
+void *  mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
+void    mgmtUpdateVgroup(SVgObj *pVgroup);
+void    mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload);
+
 void    mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb);
 void    mgmtDropVgroup(SVgObj *pVgroup, void *ahandle);
 void    mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle);

src/mnode/src/mgmtAcct.c

@@ -17,9 +17,10 @@
 #include "os.h"
 #include "taoserror.h"
 #include "mnode.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
 #include "mgmtDb.h"
 #include "mgmtUser.h"
+
 #ifndef _ACCOUNT
 
 static SAcctObj tsAcctObj = {0};
@@ -30,11 +31,12 @@ int32_t acctInit() {
   return TSDB_CODE_SUCCESS;
 }
 
-void      acctCleanUp() {}
+void    acctCleanUp() {}
-SAcctObj *acctGetAcct(char *acctName) { return &tsAcctObj; }
+void   *acctGetAcct(char *acctName) { return &tsAcctObj; }
-void      acctIncRef(SAcctObj *pAcct) {}
+void    acctIncRef(struct _acct_obj *pAcct) {}
-void      acctDecRef(SAcctObj *pAcct) {}
+void    acctReleaseAcct(SAcctObj *pAcct) {}
-int32_t   acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
+int32_t acctCheck(SAcctObj *pAcct, EAcctGrantType type) { return TSDB_CODE_SUCCESS; }
+
 #endif
 
 void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) {
@@ -46,7 +48,7 @@ void acctAddDb(SAcctObj *pAcct, SDbObj *pDb) {
 void acctRemoveDb(SAcctObj *pAcct, SDbObj *pDb) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfDbs, 1);
   pDb->pAcct = NULL;
-  acctIncRef(pAcct);
+  acctReleaseAcct(pAcct);
 }
 
 void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) {
@@ -58,5 +60,5 @@ void acctAddUser(SAcctObj *pAcct, SUserObj *pUser) {
 void acctRemoveUser(SAcctObj *pAcct, SUserObj *pUser) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfUsers, 1);
   pUser->pAcct = NULL;
-  acctIncRef(pAcct);
+  acctReleaseAcct(pAcct);
 }

src/mnode/src/mgmtBalance.c

@@ -14,56 +14,34 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "tstatus.h"
+#include "tbalance.h"
-#include "mgmtBalance.h"
+#include "mnode.h"
-#include "mgmtDnode.h"
+#include "tcluster.h"
+#include "mgmtVgroup.h"
 
-extern int32_t balanceInit();
+#ifndef _VPEER
-extern void    balanceCleanUp();
+int32_t balanceInit() { return 0; }
-extern void    balanceNotify();
+void    balanceCleanUp() {}
-extern int32_t balanceAllocVnodes(SVgObj *pVgroup);
+void    balanceNotify() {}
 
-int32_t mgmtInitBalance() {
+int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-#ifdef _VPEER
-  return balanceInit();
-#else
-  return 0;
-#endif
-}
-
-void mgmtCleanupBalance() {
-#ifdef _VPEER
-  balanceCleanUp();
-#endif
-}
-
-void mgmtBalanceNotify() {
-#ifdef _VPEER
-  balanceNotify();
-#endif
-}
-
-int32_t mgmtAllocVnodes(SVgObj *pVgroup) {
-#ifdef _VPEER
-  return balanceAllocVnodes(pVgroup);
-#else
   void *     pNode = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
   float      vnodeUsage = 1.0;
 
   while (1) {
-    mgmtDecDnodeRef(pDnode);
+    pNode = clusterGetNextDnode(pNode, &pDnode);
-    pNode = mgmtGetNextDnode(pNode, &pDnode);
     if (pDnode == NULL) break;
-    if (pDnode->numOfTotalVnodes <= 0) continue;
-    if (pDnode->openVnodes == pDnode->numOfTotalVnodes) continue;
 
-    float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes;
+    if (pDnode->numOfTotalVnodes > 0 && pDnode->openVnodes < pDnode->numOfTotalVnodes) {
-    if (usage <= vnodeUsage) {
+      float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes;
-      pSelDnode = pDnode;
+      if (usage <= vnodeUsage) {
-      vnodeUsage = usage;
+        pSelDnode = pDnode;
+        vnodeUsage = usage;
+      }
     }
+    clusterReleaseDnode(pDnode);
   }
 
   if (pSelDnode == NULL) {
@@ -77,5 +55,6 @@ int32_t mgmtAllocVnodes(SVgObj *pVgroup) {
 
   mTrace("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
   return TSDB_CODE_SUCCESS;
-#endif  
 }
+
+#endif 

src/mnode/src/mgmtDClient.c

@@ -17,15 +17,14 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tsched.h"
-#include "tstatus.h"
 #include "tsystem.h"
 #include "tutil.h"
 #include "dnode.h"
 #include "mnode.h"
-#include "mgmtBalance.h"
+#include "tbalance.h"
 #include "mgmtDb.h"
-#include "mgmtDnode.h"
+#include "tcluster.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"

src/mnode/src/mgmtDServer.c

@@ -18,15 +18,14 @@
 #include "taoserror.h"
 #include "trpc.h"
 #include "tsched.h"
-#include "tstatus.h"
 #include "tsystem.h"
 #include "tutil.h"
 #include "dnode.h"
 #include "mnode.h"
-#include "mgmtBalance.h"
+#include "tbalance.h"
 #include "mgmtDb.h"
 #include "mgmtDServer.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"

src/mnode/src/mgmtDb.c

@@ -16,15 +16,14 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
-#include "tstatus.h"
 #include "tutil.h"
 #include "name.h"
 #include "mnode.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
-#include "mgmtBalance.h"
+#include "tbalance.h"
 #include "mgmtDb.h"
-#include "mgmtDnode.h"
+#include "tcluster.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtShell.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
@@ -38,7 +37,7 @@ static int32_t tsDbUpdateSize;
 
 static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
 static void    mgmtDropDb(SQueuedMsg *newMsg);
-static int32_t mgmtSetDbDirty(SDbObj *pDb);
+static int32_t mgmtSetDbDropping(SDbObj *pDb);
 static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static void    mgmtProcessCreateDbMsg(SQueuedMsg *pMsg);
@@ -146,11 +145,11 @@ SDbObj *mgmtGetDb(char *db) {
   return (SDbObj *)sdbGetRow(tsDbSdb, db);
 }
 
-void mgmtIncDbRef(SDbObj *pDb) { 
+void mgmtIncDbRef(SDbObj *pDb) {
   return sdbIncRef(tsDbSdb, pDb); 
 }
 
-void mgmtDecDbRef(SDbObj *pDb) { 
+void mgmtReleaseDb(SDbObj *pDb) { 
   return sdbDecRef(tsDbSdb, pDb); 
 }
 
@@ -289,7 +288,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) {
 
   SDbObj *pDb = mgmtGetDb(pCreate->db);
   if (pDb != NULL) {
-    mgmtDecDbRef(pDb);
+    mgmtReleaseDb(pDb);
     return TSDB_CODE_DB_ALREADY_EXIST;
   }
 
@@ -519,7 +518,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->numOfRows = pUser->pAcct->acctInfo.numOfDbs;
 
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
   return 0;
 }
 
@@ -631,15 +630,15 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, pDb->dirty != TSDB_DB_STATUS_READY ? "dropping" : "ready");
+    strcpy(pWrite, pDb->status != TSDB_DB_STATUS_READY ? "dropping" : "ready");
     cols++;
 
     numOfRows++;
-    mgmtDecDbRef(pDb);
+    mgmtReleaseDb(pDb);
   }
 
   pShow->numOfReads += numOfRows;
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
   return numOfRows;
 }
 
@@ -659,10 +658,10 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) {
   atomic_add_fetch_32(&pDb->numOfTables, -1);
 }
 
-static int32_t mgmtSetDbDirty(SDbObj *pDb) {
+static int32_t mgmtSetDbDropping(SDbObj *pDb) {
-  if (pDb->dirty) return TSDB_CODE_SUCCESS;
+  if (pDb->status) return TSDB_CODE_SUCCESS;
 
-  pDb->dirty = true;
+  pDb->status = true;
   SSdbOperDesc oper = {
     .type = SDB_OPER_TYPE_GLOBAL,
     .table = tsDbSdb,
@@ -850,7 +849,7 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  int32_t code = mgmtSetDbDirty(pDb);
+  int32_t code = mgmtSetDbDropping(pDb);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
     mgmtSendSimpleResp(pMsg->thandle, code);
@@ -881,11 +880,11 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
     if (pDb == NULL) break;
 
     if (pDb->pAcct == pAcct) {
-      mgmtSetDbDirty(pDb);
+      mgmtSetDbDropping(pDb);
       numOfDbs++;
     }
-    mgmtDecDbRef(pDb);
+    mgmtReleaseDb(pDb);
   }
 
   mTrace("acct:%s, all dbs is is set dirty", pAcct->user, numOfDbs);
-}
+}

src/mnode/src/mgmtDnode.c

@@ -16,9 +16,9 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tmodule.h"
-#include "tstatus.h"
+#include "tbalance.h"
-#include "mgmtBalance.h"
+#include "tcluster.h"
-#include "mgmtDnode.h"
+#include "mnode.h"
 #include "mgmtDClient.h"
 #include "mgmtMnode.h"
 #include "mgmtShell.h"
@@ -26,52 +26,29 @@
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-static void    mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
+static void    clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg);
-static void    mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
+static void    clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ;
-static void    mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
+static void    clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
-static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-extern int32_t clusterInit();
-extern void    clusterCleanUp();
-extern int32_t clusterGetDnodesNum();
-extern void *  clusterGetNextDnode(void *pNode, SDnodeObj **pDnode);
-extern void    clusterIncDnodeRef(SDnodeObj *pDnode);
-extern void    clusterDecDnodeRef(SDnodeObj *pDnode);
-extern SDnodeObj* clusterGetDnode(int32_t dnodeId);
-extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip);
 #ifndef _CLUSTER
-static SDnodeObj  tsDnodeObj = {0};
-#endif
 
-int32_t mgmtInitDnodes() {
+static SDnodeObj  tsDnodeObj = {0};
-  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, mgmtProcessCfgDnodeMsg);
+
-  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, mgmtProcessCfgDnodeMsgRsp);
+int32_t clusterInitDnodes() {
-  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mgmtProcessDnodeStatusMsg);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mgmtGetModuleMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mgmtRetrieveModules);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtGetConfigMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mgmtRetrieveConfigs);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mgmtGetVnodeMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mgmtRetrieveVnodes);
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mgmtGetDnodeMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mgmtRetrieveDnodes);
- 
-#ifdef _CLUSTER
-  return clusterInit();
-#else
   tsDnodeObj.dnodeId          = 1;
   tsDnodeObj.privateIp        = inet_addr(tsPrivateIp);
   tsDnodeObj.publicIp         = inet_addr(tsPublicIp);
   tsDnodeObj.createdTime      = taosGetTimestampMs();
   tsDnodeObj.numOfTotalVnodes = tsNumOfTotalVnodes;
-  tsDnodeObj.status           = TSDB_DN_STATUS_OFFLINE;
+  tsDnodeObj.status           = TAOS_DN_STATUS_OFFLINE;
   tsDnodeObj.lastReboot       = taosGetTimestampSec();
   sprintf(tsDnodeObj.dnodeName, "%d", tsDnodeObj.dnodeId);
 
@@ -83,69 +60,47 @@ int32_t mgmtInitDnodes() {
     tsDnodeObj.moduleStatus |= (1 << TSDB_MOD_MONITOR);
   }
   return 0;
-#endif
 }
 
-void mgmtCleanUpDnodes() {
+void *clusterGetNextDnode(void *pNode, SDnodeObj **pDnode) {
-#ifdef _CLUSTER
-  clusterCleanUp();
-#endif
-}
-
-SDnodeObj *mgmtGetDnode(int32_t dnodeId) {
-#ifdef _CLUSTER
-  return clusterGetDnode(dnodeId);
-#else
-  if (dnodeId == 1) {
-    return &tsDnodeObj;
-  } else {
-    return NULL;
-  }
-#endif
-}
-
-SDnodeObj *mgmtGetDnodeByIp(uint32_t ip) {
-#ifdef _CLUSTER
-  return clusterGetDnodeByIp(ip);
-#else
-  return &tsDnodeObj;
-#endif
-}
-
-int32_t mgmtGetDnodesNum() {
-#ifdef _CLUSTER
-  return clusterGetDnodesNum();
-#else
-  return 1;
-#endif
-}
-
-void mgmtIncDnodeRef(SDnodeObj *pDnode) {
-#ifdef _CLUSTER
-  return clusterIncDnodeRef(pDnode);
-#endif
-}
-
-void mgmtDecDnodeRef(SDnodeObj *pDnode) {
-#ifdef _CLUSTER
-  return clusterDecDnodeRef(pDnode);
-#endif
-}
-
-void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) {
-#ifdef _CLUSTER
-  return clusterGetNextDnode(pNode, pDnode);
-#else
   if (*pDnode == NULL) {
     *pDnode = &tsDnodeObj;
   } else {
     *pDnode = NULL;
   }
   return *pDnode;
-#endif
 }
 
-void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
+void    clusterCleanupDnodes() {}
+int32_t clusterGetDnodesNum() { return 1; }
+void *  clusterGetDnode(int32_t dnodeId) { return dnodeId == 1 ? &tsDnodeObj : NULL; }
+void *  clusterGetDnodeByIp(uint32_t ip) { return &tsDnodeObj; }
+void    clusterReleaseDnode(struct _dnode_obj *pDnode) {}
+void    clusterUpdateDnode(struct _dnode_obj *pDnode) {}
+
+#endif
+
+int32_t clusterInit() {
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONFIG_DNODE, clusterProcessCfgDnodeMsg);
+  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP, clusterProcessCfgDnodeMsgRsp);
+  mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, clusterProcessDnodeStatusMsg);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MODULE, clusterGetModuleMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, clusterRetrieveModules);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, clusterGetConfigMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, clusterRetrieveConfigs);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VNODES, clusterGetVnodeMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, clusterRetrieveVnodes);
+  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_DNODE, clusterGetDnodeMeta);
+  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, clusterRetrieveDnodes);
+ 
+  return clusterInitDnodes();
+}
+
+void clusterCleanUp() {
+  clusterCleanupDnodes();
+}
+
+void clusterProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
   SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont;
@@ -181,11 +136,11 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
-static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
+static void clusterProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) {
   mPrint("cfg vnode rsp is received, result:%s", tstrerror(rpcMsg->code));
 }
 
-void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
+void clusterProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   if (mgmtCheckRedirect(rpcMsg->handle)) return;
 
   SDMStatusMsg *pStatus = rpcMsg->pCont;
@@ -205,14 +160,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
 
   SDnodeObj *pDnode = NULL;
   if (pStatus->dnodeId == 0) {
-    pDnode = mgmtGetDnodeByIp(pStatus->privateIp);
+    pDnode = clusterGetDnodeByIp(pStatus->privateIp);
     if (pDnode == NULL) {
       mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp));
       mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
       return;
     }
   } else {
-    pDnode = mgmtGetDnode(pStatus->dnodeId);
+    pDnode = clusterGetDnode(pStatus->dnodeId);
     if (pDnode == NULL) {
       mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp));
       mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST);
@@ -234,27 +189,30 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
  
   int32_t openVnodes = htons(pStatus->openVnodes);
   for (int32_t j = 0; j < openVnodes; ++j) {
-    pDnode->vload[j].vgId          = htonl(pStatus->load[j].vgId);
+    SVnodeLoad *pVload = &pStatus->load[j];
-    pDnode->vload[j].totalStorage  = htobe64(pStatus->load[j].totalStorage);
+    pDnode->vload[j].vgId          = htonl(pVload->vgId);
-    pDnode->vload[j].compStorage   = htobe64(pStatus->load[j].compStorage);
+    pDnode->vload[j].totalStorage  = htobe64(pVload->totalStorage);
-    pDnode->vload[j].pointsWritten = htobe64(pStatus->load[j].pointsWritten);
+    pDnode->vload[j].compStorage   = htobe64(pVload->compStorage);
+    pDnode->vload[j].pointsWritten = htobe64(pVload->pointsWritten);
     
     SVgObj *pVgroup = mgmtGetVgroup(pDnode->vload[j].vgId);
     if (pVgroup == NULL) {
       SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp);
       mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pDnode->vload[j].vgId);
       mgmtSendDropVnodeMsg(pDnode->vload[j].vgId, &ipSet, NULL);
+    } else {
+      mgmtUpdateVgroupStatus(pVgroup, pDnode->dnodeId, pVload);
+      mgmtReleaseVgroup(pVgroup);
     }
-    mgmtDecVgroupRef(pVgroup);
   }
 
-  if (pDnode->status != TSDB_DN_STATUS_READY) {
+  if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     mTrace("dnode:%d, from offline to online", pDnode->dnodeId);
-    pDnode->status = TSDB_DN_STATUS_READY;
+    pDnode->status = TAOS_DN_STATUS_READY;
-    mgmtBalanceNotify();
+    balanceNotify();
   }
 
-  mgmtDecDnodeRef(pDnode);
+  clusterReleaseDnode(pDnode);
 
   int32_t contLen = sizeof(SDMStatusRsp) + TSDB_MAX_VNODES * sizeof(SVnodeAccess);
   SDMStatusRsp *pRsp = rpcMallocCont(contLen);
@@ -284,7 +242,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
   rpcSendResponse(&rpcRsp);
 }
 
-static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t clusterGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
 
@@ -351,16 +309,16 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = mgmtGetDnodesNum();
+  pShow->numOfRows = clusterGetDnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
 
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 
   return 0;
 }
 
-static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t clusterRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   int32_t    cols      = 0;
   SDnodeObj *pDnode   = NULL;
@@ -368,8 +326,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
   char       ipstr[32];
 
   while (numOfRows < rows) {
-    mgmtDecDnodeRef(pDnode);
+    pShow->pNode = clusterGetNextDnode(pShow->pNode, &pDnode);
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
 
     cols = 0;
@@ -393,7 +350,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) );
+    strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -406,23 +363,24 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
 #ifdef _VPEER
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, taosGetDnodeLbStatusStr(pDnode->lbStatus));
+    strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
     cols++;
 #endif    
 
     numOfRows++;
+    clusterReleaseDnode(pDnode);
   }
 
   pShow->numOfReads += numOfRows;
   return numOfRows;
 }
 
-static bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
+bool clusterCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
   uint32_t status = pDnode->moduleStatus & (1 << moduleType);
   return status > 0;
 }
 
-static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t clusterGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
@@ -461,7 +419,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   pShow->numOfRows = 0;
   SDnodeObj *pDnode = NULL;
   while (1) {
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+    pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
       if (clusterCheckModuleInDnode(pDnode, moduleType)) {
@@ -472,12 +430,12 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 
   return 0;
 }
 
-int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+int32_t clusterRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   SDnodeObj *pDnode = NULL;
   char *     pWrite;
@@ -485,8 +443,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
   char       ipstr[20];
 
   while (numOfRows < rows) {
-    mgmtDecDnodeRef(pDnode);
+    clusterReleaseDnode(pDnode);
-    pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+    pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
     if (pDnode == NULL) break;
 
     for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) {
@@ -506,7 +464,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo
       cols++;
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      strcpy(pWrite, taosGetDnodeStatusStr(pDnode->status) );
+      strcpy(pWrite, clusterGetDnodeStatusStr(pDnode->status));
       cols++;
 
       numOfRows++;
@@ -523,7 +481,7 @@ static bool clusterCheckConfigShow(SGlobalConfig *cfg) {
   return true;
 }
 
-static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t clusterGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
@@ -560,12 +518,12 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 
   return 0;
 }
 
-static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t clusterRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t numOfRows = 0;
 
   for (int32_t i = tsGlobalConfigNum - 1; i >= 0 && numOfRows < rows; --i) {
@@ -612,7 +570,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo
   return numOfRows;
 }
 
-static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t clusterGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
@@ -632,12 +590,6 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pShow->bytes[cols] = 12;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "sync_status");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
   pMeta->numOfColumns = htons(cols);
   pShow->numOfColumns = cols;
 
@@ -647,7 +599,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   SDnodeObj *pDnode = NULL;
   if (pShow->payloadLen > 0 ) {
     uint32_t ip = ip2uint(pShow->payload);
-    pDnode = mgmtGetDnodeByIp(ip);
+    pDnode = clusterGetDnodeByIp(ip);
     if (NULL == pDnode) {
       return TSDB_CODE_NODE_OFFLINE;
     }
@@ -664,7 +616,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
     pShow->pNode = pDnode;
   } else {
     while (true) {
-      pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
+      pShow->pNode = clusterGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode);
       if (pDnode == NULL) break;
       pShow->numOfRows += pDnode->openVnodes;
 
@@ -675,13 +627,13 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   } 
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  mgmtDecDnodeRef(pDnode);
+  clusterReleaseDnode(pDnode);
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 
   return 0;
 }
 
-static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t clusterRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   SDnodeObj *pDnode = NULL;
   char *     pWrite;
@@ -707,11 +659,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
         cols++;
         
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        strcpy(pWrite, taosGetVnodeStatusStr(pVnode->status));
+        strcpy(pWrite, pVnode->status ? "ready" : "offline");
-        cols++;
-        
-        pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-        strcpy(pWrite, taosGetVnodeSyncStatusStr(pVnode->syncStatus));
         cols++;
         
         numOfRows++;
@@ -724,4 +672,14 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi
   
   pShow->numOfReads += numOfRows;
   return numOfRows;
-}
+}
+
+char* clusterGetDnodeStatusStr(int32_t dnodeStatus) {
+  switch (dnodeStatus) {
+    case TAOS_DN_STATUS_OFFLINE:   return "offline";
+    case TAOS_DN_STATUS_DROPPING:  return "dropping";
+    case TAOS_DN_STATUS_BALANCING: return "balancing";
+    case TAOS_DN_STATUS_READY:     return "ready";
+    default:                       return "undefined";
+  }
+}

src/mnode/src/mgmtGrant.c

@@ -18,7 +18,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tlog.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 
 int32_t grantInit() { return TSDB_CODE_SUCCESS; }
 void    grantCleanUp() {}

src/mnode/src/mgmtMain.c

@@ -19,13 +19,13 @@
 #include "tmodule.h"
 #include "tsched.h"
 #include "mnode.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
-#include "mgmtBalance.h"
+#include "tbalance.h"
+#include "tcluster.h"
 #include "mgmtDb.h"
 #include "mgmtDClient.h"
-#include "mgmtDnode.h"
 #include "mgmtDServer.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
 #include "mgmtVgroup.h"
@@ -89,7 +89,7 @@ int32_t mgmtStartSystem() {
     return -1;
   }
 
-  if (mgmtInitDnodes() < 0) {
+  if (clusterInit() < 0) {
     mError("failed to init dnodes");
     return -1;
   }
@@ -122,7 +122,7 @@ int32_t mgmtStartSystem() {
     return -1;
   }
 
-  if (mgmtInitBalance() < 0) {
+  if (balanceInit() < 0) {
     mError("failed to init dnode balance")
   }
 
@@ -148,14 +148,14 @@ void mgmtCleanUpSystem() {
   mPrint("starting to clean up mgmt");
   grantCleanUp();
   mgmtCleanupMnodes();
-  mgmtCleanupBalance();
+  balanceCleanUp();
   mgmtCleanUpShell();
   mgmtCleanupDClient();
   mgmtCleanupDServer();
   mgmtCleanUpTables();
   mgmtCleanUpVgroups();
   mgmtCleanUpDbs();
-  mgmtCleanUpDnodes();
+  clusterCleanUp();
   mgmtCleanUpUsers();
   acctCleanUp();
   taosTmrCleanUp(tsMgmtTmr);

src/mnode/src/mgmtMnode.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
-#include "tstatus.h"
 #include "trpc.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
@@ -64,6 +63,25 @@ static void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) {
   return *pMnode;
 }
 
+char *taosGetMnodeStatusStr(int32_t mnodeStatus) {
+  switch (mnodeStatus) {
+    case TSDB_MN_STATUS_OFFLINE:   return "offline";
+    case TSDB_MN_STATUS_UNSYNCED:  return "unsynced";
+    case TSDB_MN_STATUS_SYNCING:   return "syncing";
+    case TSDB_MN_STATUS_SERVING:   return "serving";
+    default:                       return "undefined";
+  }
+}
+
+char *taosGetMnodeRoleStr(int32_t mnodeRole) {
+  switch (mnodeRole) {
+    case TSDB_MN_ROLE_UNDECIDED: return "undicided";
+    case TSDB_MN_ROLE_SLAVE:     return "slave";
+    case TSDB_MN_ROLE_MASTER:    return "master";
+    default:                     return "undefined";
+  }
+}
+
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
   if (pUser == NULL) return 0;
@@ -120,7 +138,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
   pShow->numOfRows = mgmtGetMnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pNode = NULL;
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 
   return 0;
 }

src/mnode/src/mgmtProfile.c

@@ -16,6 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
+#include "taccount.h"
+#include "tcluster.h"
 #include "mgmtDb.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
@@ -787,12 +789,12 @@ void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
 void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
   if (pMsg != NULL) {
     rpcFreeCont(pMsg->pCont);
-    if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
+    if (pMsg->pUser) mgmtReleaseUser(pMsg->pUser);
-    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
+    if (pMsg->pDb) mgmtReleaseDb(pMsg->pDb);
-    if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
+    if (pMsg->pVgroup) mgmtReleaseVgroup(pMsg->pVgroup);
     if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
-    // if (pMsg->pAcct) acctDecRef(pMsg->pAcct);
+    if (pMsg->pAcct) acctReleaseAcct(pMsg->pAcct);
-    // if (pMsg->pDnode) mgmtDecTableRef(pMsg->pDnode);
+    if (pMsg->pDnode) clusterReleaseDnode(pMsg->pDnode);
     free(pMsg);
   }
 }
@@ -804,6 +806,8 @@ void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
   pDestMsg->msgType = pSrcMsg->msgType;
   pDestMsg->pCont   = pSrcMsg->pCont;
   pDestMsg->contLen = pSrcMsg->contLen;
+  pDestMsg->retry   = pSrcMsg->retry;
+  pDestMsg->maxRetry= pSrcMsg->maxRetry;
   pDestMsg->pUser   = pSrcMsg->pUser;
   pDestMsg->usePublicIp = pSrcMsg->usePublicIp;
 

src/mnode/src/mgmtSdb.c

@@ -435,7 +435,7 @@ void sdbIncRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     atomic_add_fetch_32(pRefCount, 1);
-    if (0) {
+    if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
       sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount);
     }
   }
@@ -446,7 +446,7 @@ void sdbDecRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     int32_t  refCount = atomic_sub_fetch_32(pRefCount, 1);
-    if (0) {
+    if (0 && strcmp(pTable->tableName, "dnodes") == 0) {
       sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount);
     }
     int8_t* updateEnd = pRow + pTable->refCountPos - 1;
@@ -520,6 +520,11 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) {
   
   if (pTable->keyType == SDB_KEY_TYPE_AUTO) {
     *((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
+
+    // let vgId increase from 2
+    if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) {
+      *((uint32_t *)pOper->pObj) = ++pTable->autoIndex;
+    }
   }
   pTable->version++;
   sdbVersion++;
@@ -695,6 +700,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) {
   int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
   SRowHead *rowHead = (SRowHead *)calloc(1, total_size);
   if (rowHead == NULL) {
+    pthread_mutex_unlock(&pTable->mutex);
     sdbError("table:%s, failed to allocate row head memory", pTable->tableName);
     return -1;
   }

src/mnode/src/mgmtShell.c

@@ -19,15 +19,14 @@
 #include "taoserror.h"
 #include "tlog.h"
 #include "trpc.h"
-#include "tstatus.h"
 #include "tsched.h"
 #include "dnode.h"
 #include "mnode.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
-#include "mgmtBalance.h"
+#include "tbalance.h"
 #include "mgmtDb.h"
-#include "mgmtDnode.h"
+#include "tcluster.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
@@ -129,6 +128,15 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) {
   taosScheduleTask(tsMgmtTranQhandle, &schedMsg);
 }
 
+static void mgmtDoDealyedAddToShellQueue(void *param, void *tmrId) {
+  mgmtAddToShellQueue(param);
+}
+
+void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) {
+  void *unUsed = NULL;
+  taosTmrReset(mgmtDoDealyedAddToShellQueue, 1000, queuedMsg, tsMgmtTmr, &unUsed);
+}
+
 static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
   if (rpcMsg == NULL || rpcMsg->pCont == NULL) {
     return;
@@ -179,6 +187,28 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
   }
 }
 
+char *mgmtGetShowTypeStr(int32_t showType) {
+  switch (showType) {
+    case TSDB_MGMT_TABLE_ACCT:    return "show accounts";
+    case TSDB_MGMT_TABLE_USER:    return "show users";
+    case TSDB_MGMT_TABLE_DB:      return "show databases";
+    case TSDB_MGMT_TABLE_TABLE:   return "show tables";
+    case TSDB_MGMT_TABLE_DNODE:   return "show dnodes";
+    case TSDB_MGMT_TABLE_MNODE:   return "show mnodes";
+    case TSDB_MGMT_TABLE_VGROUP:  return "show vgroups";
+    case TSDB_MGMT_TABLE_METRIC:  return "show stables";
+    case TSDB_MGMT_TABLE_MODULE:  return "show modules";
+    case TSDB_MGMT_TABLE_QUERIES: return "show queries";
+    case TSDB_MGMT_TABLE_STREAMS: return "show streams";
+    case TSDB_MGMT_TABLE_CONFIGS: return "show configs";
+    case TSDB_MGMT_TABLE_CONNS:   return "show connections";
+    case TSDB_MGMT_TABLE_SCORES:  return "show scores";
+    case TSDB_MGMT_TABLE_GRANTS:  return "show grants";
+    case TSDB_MGMT_TABLE_VNODES:  return "show vnodes";
+    default:                      return "undefined";
+  }
+}
+
 static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
   SCMShowMsg *pShowMsg = pMsg->pCont;
   if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
@@ -187,7 +217,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
   }
 
   if (!tsMgmtShowMetaFp[pShowMsg->type] || !tsMgmtShowRetrieveFp[pShowMsg->type]) {
-    mError("show type:%s is not support", taosGetShowTypeStr(pShowMsg->type));
+    mError("show type:%s is not support", mgmtGetShowTypeStr(pShowMsg->type));
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OPS_NOT_SUPPORT);
     return;
   }
@@ -209,7 +239,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
   mgmtSaveQhandle(pShow);
   pShowRsp->qhandle = htobe64((uint64_t) pShow);
 
-  mTrace("show:%p, type:%s, start to get meta", pShow, taosGetShowTypeStr(pShowMsg->type));
+  mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type));
   int32_t code = (*tsMgmtShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->thandle);
   if (code == 0) {
     SRpcMsg rpcRsp = {
@@ -220,7 +250,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
     };
     rpcSendResponse(&rpcRsp);
   } else {
-    mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, taosGetShowTypeStr(pShowMsg->type), tstrerror(code));
+    mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code));
     mgmtFreeQhandle(pShow);
     SRpcMsg rpcRsp = {
       .handle  = pMsg->thandle,
@@ -248,7 +278,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
   }
 
   SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
-  mTrace("show:%p, type:%s, retrieve data", pShow, taosGetShowTypeStr(pShow->type));
+  mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type));
 
   if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
     mError("pShow:%p, query memory is corrupted", pShow);
@@ -338,11 +368,11 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secr
   SUserObj *pUser = mgmtGetUser(user);
   if (pUser == NULL) {
     *secret = 0;
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
     return TSDB_CODE_INVALID_USER;
   } else {
     memcpy(secret, pUser->pass, TSDB_KEY_LEN);
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
     return TSDB_CODE_SUCCESS;
   }
 }

src/mnode/src/mgmtTable.c

@@ -19,7 +19,6 @@
 #include "tscompression.h"
 #include "tskiplist.h"
 #include "ttime.h"
-#include "tstatus.h"
 #include "tutil.h"
 #include "qast.h"
 #include "qextbuffer.h"
@@ -28,15 +27,14 @@
 #include "tscompression.h"
 #include "tskiplist.h"
 #include "tsqlfunction.h"
-#include "tstatus.h"
 #include "ttime.h"
 #include "name.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
 #include "mgmtDClient.h"
 #include "mgmtDb.h"
-#include "mgmtDnode.h"
+#include "tcluster.h"
 #include "mgmtDServer.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
 #include "mgmtSdb.h"
@@ -98,21 +96,21 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) {
     mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId);
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
-  mgmtDecVgroupRef(pVgroup);
+  mgmtReleaseVgroup(pVgroup);
 
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
     mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct == NULL) {
     mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
     return TSDB_CODE_INVALID_ACCT;
   }
-  acctDecRef(pAcct);
+  acctReleaseAcct(pAcct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
@@ -140,21 +138,21 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) {
   if (pVgroup == NULL) {
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
-  mgmtDecVgroupRef(pVgroup);
+  mgmtReleaseVgroup(pVgroup);
 
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
     mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct == NULL) {
     mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct);
     return TSDB_CODE_INVALID_ACCT;
   }
-  acctDecRef(pAcct);
+  acctReleaseAcct(pAcct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
@@ -272,7 +270,7 @@ static int32_t mgmtInitChildTables() {
       pNode = pLastNode;
       continue;
     }
-    mgmtDecDbRef(pDb);
+    mgmtReleaseDb(pDb);
 
     SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
     if (pVgroup == NULL) {
@@ -286,7 +284,7 @@ static int32_t mgmtInitChildTables() {
       pNode = pLastNode;
       continue;
     }
-    mgmtDecVgroupRef(pVgroup);
+    mgmtReleaseVgroup(pVgroup);
 
     if (strcmp(pVgroup->dbName, pDb->name) != 0) {
       mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
@@ -354,7 +352,7 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) {
   if (pDb != NULL) {
     mgmtAddSuperTableIntoDb(pDb);
   }
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -366,7 +364,7 @@ static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) {
     mgmtRemoveSuperTableFromDb(pDb);
     mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable);
   }
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -486,8 +484,8 @@ static void *mgmtGetSuperTable(char *tableId) {
   return sdbGetRow(tsSuperTableSdb, tableId);
 }
 
-STableInfo *mgmtGetTable(char *tableId) {
+STableObj *mgmtGetTable(char *tableId) {
-  STableInfo *tableInfo = sdbGetRow(tsSuperTableSdb, tableId);
+  STableObj *tableInfo = sdbGetRow(tsSuperTableSdb, tableId);
   if (tableInfo != NULL) {
     return tableInfo;
   }
@@ -501,7 +499,7 @@ STableInfo *mgmtGetTable(char *tableId) {
 }
 
 void mgmtIncTableRef(void *p1) {
-  STableInfo *pTable = (STableInfo *)p1;
+  STableObj *pTable = (STableObj *)p1;
   if (pTable->type == TSDB_SUPER_TABLE) {
     sdbIncRef(tsSuperTableSdb, pTable);
   } else {
@@ -512,7 +510,7 @@ void mgmtIncTableRef(void *p1) {
 void mgmtDecTableRef(void *p1) {
   if (p1 == NULL) return;
 
-  STableInfo *pTable = (STableInfo *)p1;
+  STableObj *pTable = (STableObj *)p1;
   if (pTable->type == TSDB_SUPER_TABLE) {
     sdbDecRef(tsSuperTableSdb, pTable);
   } else {
@@ -542,7 +540,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
   SCMCreateTableMsg *pCreate = pMsg->pCont;
   
   pMsg->pTable = mgmtGetTable(pCreate->tableId);
-  if (pMsg->pTable != NULL) {
+  if (pMsg->pTable != NULL && pMsg->retry == 0) {
     if (pCreate->igExists) {
       mTrace("table:%s, is already exist", pCreate->tableId);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@@ -554,7 +552,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
   }
 
   pMsg->pDb = mgmtGetDb(pCreate->db);
-  if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
+  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to create, db not selected", pCreate->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
     return;
@@ -572,7 +570,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
 static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
   SCMDropTableMsg *pDrop = pMsg->pCont;
   pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
-  if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
+  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
     return;
@@ -611,7 +609,7 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
   mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle);
 
   pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
-  if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
+  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to get table meta, db not selected", pInfo->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
     return;
@@ -860,7 +858,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables);
-    acctDecRef(pAcct);
+    acctReleaseAcct(pAcct);
   }
 
   SSdbOperDesc oper = {
@@ -897,7 +895,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables;
-    acctDecRef(pAcct);
+    acctReleaseAcct(pAcct);
   }
 
   SSdbOperDesc oper = {
@@ -963,7 +961,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
   pShow->numOfRows = pDb->numOfSuperTables;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
   return 0;
 }
 
@@ -1028,7 +1026,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
   }
 
   pShow->numOfReads += numOfRows;
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   return numOfRows;
 }
@@ -1106,7 +1104,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum());
+  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * clusterGetDnodesNum());
   if (pRsp == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
     return;
@@ -1302,7 +1300,11 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  pMsg->pTable = (STableInfo *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
+  if (pMsg->retry == 0) {
+    pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
+  } else {
+    pMsg->pTable = mgmtGetTable(pCreate->tableId);
+  }
   if (pMsg->pTable == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, terrno);
     return;
@@ -1317,6 +1319,7 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
   SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   newMsg->ahandle = pMsg->pTable;
+  newMsg->maxRetry = 5;
   mgmtIncTableRef(pMsg->pTable);
   SRpcMsg rpcMsg = {
       .handle  = newMsg,
@@ -1409,7 +1412,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
      pAcct->acctInfo.numOfTimeSeries += ncols;
-    acctDecRef(pAcct);
+    acctReleaseAcct(pAcct);
   }
  
   SSdbOperDesc oper = {
@@ -1443,7 +1446,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
   SAcctObj *pAcct = acctGetAcct(pDb->cfg.acct);
   if (pAcct != NULL) {
     pAcct->acctInfo.numOfTimeSeries--;
-    acctDecRef(pAcct);
+    acctReleaseAcct(pAcct);
   }
  
   SSdbOperDesc oper = {
@@ -1633,7 +1636,7 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
 }
 
 static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) {
-  SDnodeObj *pObj = mgmtGetDnode(dnodeId);
+  SDnodeObj *pObj = clusterGetDnode(dnodeId);
   SVgObj *pVgroup = mgmtGetVgroup(vnode);
 
   if (pObj == NULL || pVgroup == NULL) {
@@ -1641,8 +1644,8 @@ static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_
   }
 
   SChildTableObj *pTable = pVgroup->tableList[sid];
-  mgmtIncTableRef((STableInfo *)pTable);
+  mgmtIncTableRef((STableObj *)pTable);
-  mgmtDecVgroupRef(pVgroup);
+  mgmtReleaseVgroup(pVgroup);
   return pTable;
 }
 
@@ -1739,30 +1742,40 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
   queueMsg->received++;
 
   SChildTableObj *pTable = queueMsg->ahandle;
-  mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle,
+  mTrace("table:%s, create table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle,
-         rpcMsg->handle, tstrerror(rpcMsg->code));
+         tstrerror(rpcMsg->code));
 
   if (rpcMsg->code != TSDB_CODE_SUCCESS) {
-    SSdbOperDesc oper = {
+    if (queueMsg->retry++ < queueMsg->maxRetry) {
-      .type = SDB_OPER_TYPE_GLOBAL,
+      mTrace("table:%s, create table rsp received, retry:%d thandle:%p result:%s", pTable->info.tableId,
-      .table = tsChildTableSdb,
+             queueMsg->retry, queueMsg->thandle, tstrerror(rpcMsg->code));
-      .pObj = pTable
+      mgmtDealyedAddToShellQueue(queueMsg);
-    };
+    } else {
-    sdbDeleteRow(&oper);
+      mError("table:%s, failed to create in dnode, thandle:%p result:%s", pTable->info.tableId,
+             queueMsg->thandle, tstrerror(rpcMsg->code));
+      
+      SSdbOperDesc oper = {
+        .type = SDB_OPER_TYPE_GLOBAL,
+        .table = tsChildTableSdb,
+        .pObj = pTable
+      };
+      sdbDeleteRow(&oper);
     
-    mError("table:%s, failed to create in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code));
+      mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
-    mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
+      mgmtFreeQueuedMsg(queueMsg);
+    }    
   } else {
-    mTrace("table:%s, created in dnode", pTable->info.tableId);
+    mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle,
+           tstrerror(rpcMsg->code));
+
     if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) {
       mTrace("table:%s, start to get meta", pTable->info.tableId);
-      mgmtAddToShellQueue(mgmtCloneQueuedMsg(queueMsg));
+      mgmtAddToShellQueue(queueMsg);
     } else {
       mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
+      mgmtFreeQueuedMsg(queueMsg);
     }
   }
-
-  mgmtFreeQueuedMsg(queueMsg);
 }
 
 // not implemented yet
@@ -1863,7 +1876,7 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   pShow->numOfRows = pDb->numOfTables;
   pShow->rowSize   = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
   return 0;
 }
 
@@ -1940,7 +1953,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
   const int32_t NUM_OF_COLUMNS = 4;
 
   mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   return numOfRows;
 }
@@ -1950,7 +1963,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
   mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle);
 
   pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
-  if (pMsg->pDb == NULL || pMsg->pDb->dirty) {
+  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to alter table, db not selected", pAlter->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
     return;

src/mnode/src/mgmtUser.c

@@ -18,8 +18,8 @@
 #include "trpc.h"
 #include "ttime.h"
 #include "tutil.h"
-#include "mgmtAcct.h"
+#include "taccount.h"
-#include "mgmtGrant.h"
+#include "tgrant.h"
 #include "mgmtMnode.h"
 #include "mgmtSdb.h"
 #include "mgmtShell.h"
@@ -117,7 +117,7 @@ int32_t mgmtInitUsers() {
   mgmtCreateUser(pAcct, "root", "taosdata");
   mgmtCreateUser(pAcct, "monitor", tsInternalPass);
   mgmtCreateUser(pAcct, "_root", tsInternalPass);
-  acctDecRef(pAcct);
+  acctReleaseAcct(pAcct);
 
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_USER, mgmtProcessCreateUserMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_USER, mgmtProcessAlterUserMsg);
@@ -137,11 +137,7 @@ SUserObj *mgmtGetUser(char *name) {
   return (SUserObj *)sdbGetRow(tsUserSdb, name);
 }
 
-void mgmtIncUserRef(SUserObj *pUser) { 
+void mgmtReleaseUser(SUserObj *pUser) { 
-  return sdbIncRef(tsUserSdb, pUser); 
-}
-
-void mgmtDecUserRef(SUserObj *pUser) { 
   return sdbDecRef(tsUserSdb, pUser); 
 }
 
@@ -174,7 +170,7 @@ int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) {
   SUserObj *pUser = mgmtGetUser(name);
   if (pUser != NULL) {
     mTrace("user:%s is already there", name);
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
     return TSDB_CODE_USER_ALREADY_EXIST;
   }
 
@@ -264,7 +260,7 @@ static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCon
   pShow->numOfRows = pUser->pAcct->acctInfo.numOfUsers;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
 
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
   return 0;
 }
 
@@ -299,7 +295,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
     cols++;
 
     numOfRows++;
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
   }
   pShow->numOfReads += numOfRows;
   return numOfRows;
@@ -351,7 +347,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
 
   if (strcmp(pUser->user, "monitor") == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
     return;
   }
 
@@ -427,7 +423,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
   }
 
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 }
 
 static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
@@ -446,7 +442,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
   if (strcmp(pUser->user, "monitor") == 0 || strcmp(pUser->user, pUser->acct) == 0 ||
     (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
     return ;
   }
 
@@ -475,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
   }
 
   mgmtSendSimpleResp(pMsg->thandle, code);
-  mgmtDecUserRef(pUser);
+  mgmtReleaseUser(pUser);
 }
 
 void  mgmtDropAllUsers(SAcctObj *pAcct)  {
@@ -501,7 +497,7 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
       numOfUsers++;
     }
 
-    mgmtDecUserRef(pUser);
+    mgmtReleaseUser(pUser);
   }
 
   mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);

src/mnode/src/mgmtVgroup.c

@@ -17,12 +17,12 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tlog.h"
-#include "tstatus.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "tcluster.h"
 #include "mnode.h"
-#include "mgmtBalance.h"
 #include "mgmtDb.h"
 #include "mgmtDClient.h"
-#include "mgmtDnode.h"
 #include "mgmtDServer.h"
 #include "mgmtMnode.h"
 #include "mgmtProfile.h"
@@ -54,11 +54,11 @@ static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) {
   }
 
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId);
     if (pDnode) {
       atomic_sub_fetch_32(&pDnode->openVnodes, 1);
     }
-    mgmtDecDnodeRef(pDnode);
+    clusterReleaseDnode(pDnode);
   }
 
   tfree(pOper->pObj);
@@ -71,7 +71,7 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
   if (pDb == NULL) {
     return TSDB_CODE_INVALID_DB;
   }
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   pVgroup->pDb = pDb;
   pVgroup->prev = NULL;
@@ -92,12 +92,12 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) {
   }
 
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    SDnodeObj *pDnode = clusterGetDnode(pVgroup->vnodeGid[i].dnodeId);
     if (pDnode != NULL) {
       pVgroup->vnodeGid[i].privateIp = pDnode->privateIp;
       pVgroup->vnodeGid[i].publicIp = pDnode->publicIp;
       atomic_add_fetch_32(&pDnode->openVnodes, 1);    
-      mgmtDecDnodeRef(pDnode);
+      clusterReleaseDnode(pDnode);
     }     
   }
 
@@ -114,7 +114,7 @@ static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) {
     mgmtRemoveVgroupFromDb(pVgroup);
   }
 
-  mgmtDecDbRef(pVgroup->pDb);
+  mgmtReleaseDb(pVgroup->pDb);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -190,11 +190,7 @@ int32_t mgmtInitVgroups() {
   return 0;
 }
 
-void mgmtIncVgroupRef(SVgObj *pVgroup) { 
+void mgmtReleaseVgroup(SVgObj *pVgroup) { 
-  return sdbIncRef(tsVgroupSdb, pVgroup); 
-}
-
-void mgmtDecVgroupRef(SVgObj *pVgroup) { 
   return sdbDecRef(tsVgroupSdb, pVgroup); 
 }
 
@@ -202,16 +198,44 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
   return (SVgObj *)sdbGetRow(tsVgroupSdb, &vgId);
 }
 
+void mgmtUpdateVgroup(SVgObj *pVgroup) {
+  SSdbOperDesc oper = {
+    .type = SDB_OPER_TYPE_GLOBAL,
+    .table = tsVgroupSdb,
+    .pObj = pVgroup,
+    .rowSize = tsVgUpdateSize
+  };
+
+  sdbUpdateRow(&oper);
+  mgmtSendCreateVgroupMsg(pVgroup, NULL);
+}
+
+void mgmtUpdateVgroupStatus(SVgObj *pVgroup, int32_t dnodeId, SVnodeLoad *pVload) {
+  if (pVload->role == TAOS_SYNC_ROLE_MASTER) {
+    for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+      SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
+      if (pVgid->dnodeId == dnodeId) {
+        pVgroup->inUse = i;
+        break;
+      }
+    }
+  }
+}
+
 SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) {
   return pDb->pHead;
 }
 
+void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { 
+  return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); 
+}
+
 void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
   SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
   strcpy(pVgroup->dbName, pDb->name);
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
-  if (mgmtAllocVnodes(pVgroup) != 0) {
+  if (balanceAllocVnodes(pVgroup) != 0) {
     mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes);
     free(pVgroup);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
@@ -293,7 +317,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
 
   int32_t maxReplica = 0;
   SVgObj  *pVgroup   = NULL;
-  STableInfo *pTable = NULL;
+  STableObj *pTable = NULL;
   if (pShow->payloadLen > 0 ) {
     pTable = mgmtGetTable(pShow->payload);
     if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
@@ -302,7 +326,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
     mgmtDecTableRef(pTable);
     pVgroup = mgmtGetVgroup(((SChildTableObj*)pTable)->vgId);
     if (NULL == pVgroup) return TSDB_CODE_INVALID_TABLE_ID;
-    mgmtDecVgroupRef(pVgroup);
+    mgmtReleaseVgroup(pVgroup);
     maxReplica = pVgroup->numOfVnodes > maxReplica ? pVgroup->numOfVnodes : maxReplica;
   } else {
     SVgObj *pVgroup = pDb->pHead;
@@ -348,26 +372,26 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
     pShow->pNode = pVgroup;
   }
 
-   mgmtDecDbRef(pDb);
+   mgmtReleaseDb(pDb);
 
   return 0;
 }
 
 char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) {
-  SDnodeObj *pDnode = mgmtGetDnode(pVnode->dnodeId);
+  SDnodeObj *pDnode = clusterGetDnode(pVnode->dnodeId);
   if (pDnode == NULL) {
     mError("vgroup:%d, not exist in dnode:%d", pVgroup->vgId, pDnode->dnodeId);
     return "null";
   }
-  mgmtDecDnodeRef(pDnode);
+  clusterReleaseDnode(pDnode);
 
-  if (pDnode->status == TSDB_DN_STATUS_OFFLINE) {
+  if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
     return "offline";
   }
 
   for (int i = 0; i < pDnode->openVnodes; ++i) {
     if (pDnode->vload[i].vgId == pVgroup->vgId) {
-       return (char*)taosGetVnodeStatusStr(pDnode->vload[i].status);
+       return pDnode->vload[i].status ? "ready" : "offline";
     }
   }
   
@@ -407,7 +431,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    strcpy(pWrite, taosGetVgroupLbStatusStr(pVgroup->lbStatus));
+    strcpy(pWrite, pVgroup->status ? "updating" : "ready");
     cols++;
 
     for (int32_t i = 0; i < maxReplica; ++i) {
@@ -434,7 +458,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
   }
 
   pShow->numOfReads += numOfRows;
-  mgmtDecDbRef(pDb);
+  mgmtReleaseDb(pDb);
 
   return numOfRows;
 }
@@ -645,13 +669,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
   pCfg->dnodeId = htonl(pCfg->dnodeId);
   pCfg->vgId    = htonl(pCfg->vgId);
 
-  SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId);
+  SDnodeObj *pDnode = clusterGetDnode(pCfg->dnodeId);
   if (pDnode == NULL) {
     mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE);
     return;
   }
-  mgmtDecDnodeRef(pDnode);
+  clusterReleaseDnode(pDnode);
 
   SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId);
   if (pVgroup == NULL) {
@@ -659,7 +683,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE);
     return;
   }
-  mgmtDecVgroupRef(pVgroup);
+  mgmtReleaseVgroup(pVgroup);
 
   mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS);
 
@@ -675,7 +699,7 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
   SVgObj *pVgroup = NULL;
 
   while (1) {
-    mgmtDecVgroupRef(pVgroup);
+    mgmtReleaseVgroup(pVgroup);
     pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
     if (pVgroup == NULL) break;
 
@@ -704,4 +728,5 @@ void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) {
   } else {
     mgmtAddToShellQueue(ahandle);
   }
-}
+}
+

src/query/inc/tlosertree.h

@@ -38,7 +38,7 @@ typedef struct SLoserTreeInfo {
   SLoserTreeNode *pNode;
 } SLoserTreeInfo;
 
-uint8_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
+uint32_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
 
 void tLoserTreeInit(SLoserTreeInfo *pTree);
 

src/query/src/queryExecutor.c

@@ -20,7 +20,6 @@
 #include "tlog.h"
 #include "tlosertree.h"
 #include "tscompression.h"
-#include "tstatus.h"
 #include "ttime.h"
 
 #include "qast.h"
@@ -5705,7 +5704,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
                                SArray *pTableIdList) {
   SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
   if (pQInfo == NULL) {
-    goto _clean_memory;
+    goto _clean_pQInfo_memory;
   }
 
   SQuery *pQuery = calloc(1, sizeof(SQuery));
@@ -5834,6 +5833,7 @@ _clean_memory:
   tfree(pExprs);
   tfree(pGroupbyExpr);
 
+_clean_pQInfo_memory:
   tfree(pQInfo);
 
   return NULL;
@@ -6201,4 +6201,4 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c
 //    vnodeDecRefCount(pObj->qhandle);
 //    pObj->qhandle = NULL;
 //  }
-}
+}

src/query/src/tlosertree.c

@@ -40,7 +40,7 @@ void tLoserTreeDisplay(SLoserTreeInfo* pTree) {
   printf("\n");
 }
 
-uint8_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) {
+uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) {
   int32_t totalEntries = numOfEntries << 1;
 
   *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries);

src/rpc/src/rpcMain.c

@@ -331,6 +331,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
   char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead);
   if (contLen == 0 ) {
     free(start); 
+    return NULL;
   }
 
   int size = contLen + RPC_MSG_OVERHEAD;
@@ -725,10 +726,6 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) {
     return TSDB_CODE_INVALID_RESPONSE_TYPE;
   }
 
-  if (pHead->code == TSDB_CODE_NOT_READY) {
-    return TSDB_CODE_ALREADY_PROCESSED;
-  }
-
   taosTmrStopA(&pConn->pTimer);
   pConn->retry = 0;
 
@@ -934,6 +931,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) {
       memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet));
       tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps);
       rpcSendReqToServer(pRpc, pContext);
+    } else if (pHead->code == TSDB_CODE_NOT_READY) {
+      pContext->code = pHead->code;
+      rpcProcessConnError(pContext, NULL);
     } else {
       rpcNotifyClient(pContext, &rpcMsg);
     }
@@ -1078,7 +1078,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
     if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16))
       tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d",
           pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort, 
-          pHead->code, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+          htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
   writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
@@ -1102,7 +1102,7 @@ static void rpcProcessConnError(void *param, void *id) {
   
   tTrace("%s connection error happens", pRpc->label);
 
-  if ( pContext->numOfTry >= pContext->ipSet.numOfIps ) {
+  if (pContext->numOfTry >= pContext->ipSet.numOfIps) {
     rpcMsg.msgType = pContext->msgType+1;
     rpcMsg.handle = pContext->ahandle;
     rpcMsg.code = pContext->code;

src/util/inc/tmodule.h

@@ -28,8 +28,6 @@ enum _module {
   TSDB_MOD_MGMT,
   TSDB_MOD_HTTP,
   TSDB_MOD_MONITOR,
-  TSDB_MOD_DCLUSTER,
-  TSDB_MOD_MSTORAGE,
   TSDB_MOD_MAX
 };
 

src/util/inc/tstatus.h

@@ -1,127 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TSTATUS_H
-#define TDENGINE_TSTATUS_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-#include <stdbool.h>
-#include "taoserror.h"
-
-enum _TSDB_VG_STATUS {
-  TSDB_VG_STATUS_READY               = TSDB_CODE_SUCCESS,
-  TSDB_VG_STATUS_IN_PROGRESS         = 1, //TSDB_CODE_ACTION_IN_PROGRESS,
-  TSDB_VG_STATUS_NO_DISK_PERMISSIONS = 73,//TSDB_CODE_NO_DISK_PERMISSIONS,
-  TSDB_VG_STATUS_SERVER_NO_PACE      = 110, //TSDB_CODE_SERV_NO_DISKSPACE,
-  TSDB_VG_STATUS_SERV_OUT_OF_MEMORY  = 69, //TSDB_CODE_SERV_OUT_OF_MEMORY,
-  TSDB_VG_STATUS_INIT_FAILED         = 74, //TSDB_CODE_VG_INIT_FAILED,
-  TSDB_VG_STATUS_FULL                = 48, //TSDB_CODE_NO_ENOUGH_DNODES,
-};
-
-enum _TSDB_DB_STATUS {
-  TSDB_DB_STATUS_READY,
-  TSDB_DB_STATUS_DROPPING,
-  TSDB_DB_STATUS_DROP_FROM_SDB
-};
-
-typedef enum _TSDB_VN_STATUS {
-  TSDB_VN_STATUS_NOT_READY,
-  TSDB_VN_STATUS_UNSYNCED,
-  TSDB_VN_STATUS_SLAVE,
-  TSDB_VN_STATUS_MASTER,
-  TSDB_VN_STATUS_CREATING,
-  TSDB_VN_STATUS_CLOSING,
-  TSDB_VN_STATUS_DELETING,
-} EVnodeStatus;
-
-enum _TSDB_VN_SYNC_STATUS {
-  TSDB_VN_SYNC_STATUS_INIT,
-  TSDB_VN_SYNC_STATUS_SYNCING,
-  TSDB_VN_SYNC_STATUS_SYNC_CACHE,
-  TSDB_VN_SYNC_STATUS_SYNC_FILE
-};
-
-enum _TSDB_VN_DROP_STATUS {
-  TSDB_VN_DROP_STATUS_READY,
-  TSDB_VN_DROP_STATUS_DROPPING
-};
-
-enum _TSDB_MN_STATUS {
-  TSDB_MN_STATUS_OFFLINE,
-  TSDB_MN_STATUS_UNSYNCED,
-  TSDB_MN_STATUS_SYNCING,
-  TSDB_MN_STATUS_SERVING
-};
-
-enum _TSDB_MN_ROLE {
-  TSDB_MN_ROLE_UNDECIDED,
-  TSDB_MN_ROLE_SLAVE,
-  TSDB_MN_ROLE_MASTER
-};
-
-enum _TSDB_DN_STATUS {
-  TSDB_DN_STATUS_OFFLINE,
-  TSDB_DN_STATUS_READY
-};
-
-enum _TSDB_DN_LB_STATUS {
-  TSDB_DN_LB_STATUS_BALANCED,
-  TSDB_DN_LB_STATUS_BALANCING,
-  TSDB_DN_LB_STATUS_OFFLINE_REMOVING,
-  TSDB_DN_LB_STATE_SHELL_REMOVING
-};
-
-enum _TSDB_VG_LB_STATUS {
-  TSDB_VG_LB_STATUS_READY,
-  TSDB_VG_LB_STATUS_UPDATE
-};
-
-enum _TSDB_VN_STREAM_STATUS {
-  TSDB_VN_STREAM_STATUS_STOP,
-  TSDB_VN_STREAM_STATUS_START
-};
-
-enum TSDB_TABLE_STATUS {
-  TSDB_METER_STATE_READY       = 0x00,
-  TSDB_METER_STATE_INSERTING   = 0x01,
-  TSDB_METER_STATE_IMPORTING   = 0x02,
-  TSDB_METER_STATE_UPDATING    = 0x04,
-  TSDB_METER_STATE_DROPPING    = 0x10,
-  TSDB_METER_STATE_DROPPED     = 0x18,
-};
-
-char* taosGetVgroupStatusStr(int32_t vgroupStatus);
-char* taosGetDbStatusStr(int32_t dbStatus);
-char* taosGetVnodeStatusStr(int32_t vnodeStatus);
-char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus);
-char* taosGetVnodeDropStatusStr(int32_t dropping);
-char* taosGetDnodeStatusStr(int32_t dnodeStatus);
-char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus);
-char* taosGetVgroupLbStatusStr(int32_t vglbStatus);
-char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus);
-char* taosGetTableStatusStr(int32_t tableStatus);
-char *taosGetShowTypeStr(int32_t showType);
-char *taosGetMnodeStatusStr(int32_t mnodeStatus);
-char *taosGetMnodeRoleStr(int32_t mnodeRole);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TSTATUS_H

src/util/src/tglobalcfg.c

@@ -110,7 +110,12 @@ short tsDaysPerFile = 10;
 int   tsDaysToKeep = 3650;
 int   tsReplications = TSDB_REPLICA_MIN_NUM;
 
+#ifdef _MPEER
 int  tsNumOfMPeers = 3;
+#else
+int  tsNumOfMPeers = 1;
+#endif
+
 int  tsMaxShellConns = 2000;
 int  tsMaxTables = 100000;
 
@@ -551,9 +556,11 @@ static void doInitGlobalConfig() {
   tsInitConfigOption(cfg++, "tblocks", &tsNumOfBlocksPerMeter, TSDB_CFG_VTYPE_SHORT,
                      TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW,
                      32, 4096, 0, TSDB_CFG_UTYPE_NONE);
+#ifdef _MPEER                     
   tsInitConfigOption(cfg++, "numOfMPeers", &tsNumOfMPeers, TSDB_CFG_VTYPE_INT,
                      TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER,
                      1, 3, 0, TSDB_CFG_UTYPE_NONE);
+#endif 
   tsInitConfigOption(cfg++, "balanceInterval", &tsBalanceStartInterval, TSDB_CFG_VTYPE_INT,
                      TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER,
                      1, 30000, 0, TSDB_CFG_UTYPE_NONE);

src/util/src/tskiplist.c

@@ -19,7 +19,7 @@
 #include "tutil.h"
 #include "tcompare.h"
 
-static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) {  // record link count in each level
+__attribute__ ((unused)) static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) {  // record link count in each level
 #if SKIP_LIST_RECORD_PERFORMANCE
   for (int32_t i = 0; i < level; ++i) {
     pSkipList->state.nLevelNodeCnt[i]++;
@@ -27,7 +27,7 @@ static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level
 #endif
 }
 
-static FORCE_INLINE void removeNodeEachLevel(SSkipList *pSkipList, int32_t level) {
+__attribute__ ((unused)) static FORCE_INLINE void removeNodeEachLevel(SSkipList *pSkipList, int32_t level) {
 #if SKIP_LIST_RECORD_PERFORMANCE
   for (int32_t i = 0; i < level; ++i) {
     pSkipList->state.nLevelNodeCnt[i]--;

src/util/src/tstatus.c

@@ -1,157 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "taosmsg.h"
-#include "tstatus.h"
-
-char* taosGetVgroupStatusStr(int32_t vgroupStatus) {
-  switch (vgroupStatus) {
-    case TSDB_VG_STATUS_READY:                 return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_IN_PROGRESS:           return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_NO_DISK_PERMISSIONS:   return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_SERVER_NO_PACE:        return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_SERV_OUT_OF_MEMORY:    return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_INIT_FAILED:           return (char*)tstrerror(vgroupStatus);
-    case TSDB_VG_STATUS_FULL:                  return (char*)tstrerror(vgroupStatus);
-    default:                                   return "undefined";
-  }
-}
-
-char* taosGetDbStatusStr(int32_t dbStatus) {
-  switch (dbStatus) {
-    case TSDB_DB_STATUS_READY:         return "ready";
-    case TSDB_DB_STATUS_DROPPING:      return "dropping";
-    case TSDB_DB_STATUS_DROP_FROM_SDB: return "drop_from_sdb";
-    default:                           return "undefined";
-  }
-}
-
-char* taosGetVnodeStatusStr(int32_t vnodeStatus) {
-  switch (vnodeStatus) {
-    case TSDB_VN_STATUS_NOT_READY:return "not_ready";
-    case TSDB_VN_STATUS_UNSYNCED: return "unsynced";
-    case TSDB_VN_STATUS_SLAVE:    return "slave";
-    case TSDB_VN_STATUS_MASTER:   return "master";
-    case TSDB_VN_STATUS_CREATING: return "creating";
-    case TSDB_VN_STATUS_CLOSING:  return "closing";
-    case TSDB_VN_STATUS_DELETING: return "deleting";
-    default:                      return "undefined";
-  }
-}
-
-char* taosGetVnodeSyncStatusStr(int32_t vnodeSyncStatus) {
-  switch (vnodeSyncStatus) {
-    case TSDB_VN_SYNC_STATUS_INIT:       return "ready";
-    case TSDB_VN_SYNC_STATUS_SYNCING:    return "syncing";
-    case TSDB_VN_SYNC_STATUS_SYNC_CACHE: return "sync_cache";
-    case TSDB_VN_SYNC_STATUS_SYNC_FILE:  return "sync_file";
-    default:                             return "undefined";
-  }
-}
-
-char* taosGetVnodeDropStatusStr(int32_t dropping) {
-  switch (dropping) {
-    case TSDB_VN_DROP_STATUS_READY:     return "ready";
-    case TSDB_VN_DROP_STATUS_DROPPING:  return "dropping";
-    default:                            return "undefined";
-  }
-}
-
-char* taosGetDnodeStatusStr(int32_t dnodeStatus) {
-  switch (dnodeStatus) {
-    case TSDB_DN_STATUS_OFFLINE: return "offline";
-    case TSDB_DN_STATUS_READY:   return "ready";
-    default:                     return "undefined";
-  }
-}
-
-char* taosGetDnodeLbStatusStr(int32_t dnodeBalanceStatus) {
-  switch (dnodeBalanceStatus) {
-    case TSDB_DN_LB_STATUS_BALANCED:         return "balanced";
-    case TSDB_DN_LB_STATUS_BALANCING:        return "balancing";
-    case TSDB_DN_LB_STATUS_OFFLINE_REMOVING: return "offline removing";
-    case TSDB_DN_LB_STATE_SHELL_REMOVING:    return "removing";
-    default:                                 return "undefined";
-  }
-}
-
-char* taosGetVgroupLbStatusStr(int32_t vglbStatus) {
-  switch (vglbStatus) {
-    case TSDB_VG_LB_STATUS_READY:   return "ready";
-    case TSDB_VG_LB_STATUS_UPDATE:  return "updating";
-    default:                        return "undefined";
-  }
-}
-
-char* taosGetVnodeStreamStatusStr(int32_t vnodeStreamStatus) {
-  switch (vnodeStreamStatus) {
-    case TSDB_VN_STREAM_STATUS_START: return "start";
-    case TSDB_VN_STREAM_STATUS_STOP:  return "stop";
-    default:                          return "undefined";
-  }
-}
-
-char* taosGetTableStatusStr(int32_t tableStatus) {
-  switch(tableStatus) {
-    case TSDB_METER_STATE_INSERTING:return "inserting";
-    case TSDB_METER_STATE_IMPORTING:return "importing";
-    case TSDB_METER_STATE_UPDATING: return "updating";
-    case TSDB_METER_STATE_DROPPING: return "deleting";
-    case TSDB_METER_STATE_DROPPED:  return "dropped";
-    case TSDB_METER_STATE_READY:    return "ready";
-    default:return "undefined";
-  }
-}
-
-char *taosGetShowTypeStr(int32_t showType) {
-  switch (showType) {
-    case TSDB_MGMT_TABLE_ACCT:    return "show accounts";
-    case TSDB_MGMT_TABLE_USER:    return "show users";
-    case TSDB_MGMT_TABLE_DB:      return "show databases";
-    case TSDB_MGMT_TABLE_TABLE:   return "show tables";
-    case TSDB_MGMT_TABLE_DNODE:   return "show dnodes";
-    case TSDB_MGMT_TABLE_MNODE:   return "show mnodes";
-    case TSDB_MGMT_TABLE_VGROUP:  return "show vgroups";
-    case TSDB_MGMT_TABLE_METRIC:  return "show stables";
-    case TSDB_MGMT_TABLE_MODULE:  return "show modules";
-    case TSDB_MGMT_TABLE_QUERIES: return "show queries";
-    case TSDB_MGMT_TABLE_STREAMS: return "show streams";
-    case TSDB_MGMT_TABLE_CONFIGS: return "show configs";
-    case TSDB_MGMT_TABLE_CONNS:   return "show connections";
-    case TSDB_MGMT_TABLE_SCORES:  return "show scores";
-    case TSDB_MGMT_TABLE_GRANTS:  return "show grants";
-    case TSDB_MGMT_TABLE_VNODES:  return "show vnodes";
-    default:                      return "undefined";
-  }
-}
-
-char *taosGetMnodeStatusStr(int32_t mnodeStatus) {
-  switch (mnodeStatus) {
-    case TSDB_MN_STATUS_OFFLINE:   return "offline";
-    case TSDB_MN_STATUS_UNSYNCED:  return "unsynced";
-    case TSDB_MN_STATUS_SYNCING:   return "syncing";
-    case TSDB_MN_STATUS_SERVING:   return "serving";
-    default:                       return "undefined";
-  }
-}
-
-char *taosGetMnodeRoleStr(int32_t mnodeRole) {
-  switch (mnodeRole) {
-    case TSDB_MN_ROLE_UNDECIDED: return "undicided";
-    case TSDB_MN_ROLE_SLAVE:     return "slave";
-    case TSDB_MN_ROLE_MASTER:    return "master";
-    default:                     return "undefined";
-  }
-}

src/util/src/tstring.c

@@ -110,6 +110,8 @@ char *taosMsg[] = {
     "",
     "",
     "",
+    "",
+    "",
     "",                    //90
 
     "config-table",

src/vnode/main/inc/vnodeInt.h

@@ -20,19 +20,14 @@
 extern "C" {
 #endif
 
-typedef enum _VN_STATUS {
+#include "tsync.h"
-  VN_STATUS_INIT,
+#include "twal.h"
-  VN_STATUS_CREATING,
-  VN_STATUS_READY,
-  VN_STATUS_CLOSING,
-  VN_STATUS_DELETING,
-} EVnStatus;
 
 typedef struct {
   int32_t      vgId;      // global vnode group ID
   int32_t      refCount;  // reference count
-  EVnStatus    status; 
+  int          status; 
-  int          role;   
+  int8_t       role;   
   int64_t      version;
   void        *wqueue;
   void        *rqueue;
@@ -41,12 +36,14 @@ typedef struct {
   void        *sync;
   void        *events;
   void        *cq;  // continuous query
-  int32_t      replicas;
+  STsdbCfg    tsdbCfg;
-  SVnodeDesc   vpeers[TSDB_MAX_MPEERS];
+  SSyncCfg    syncCfg;
+  SWalCfg     walCfg;
 } SVnodeObj;
 
-int  vnodeWriteToQueue(void *param, SWalHead *pHead, int type);
+int  vnodeWriteToQueue(void *param, void *pHead, int type);
 void vnodeInitWriteFp(void);
+void vnodeInitReadFp(void);
 
 #ifdef __cplusplus
 }

src/vnode/main/src/vnodeMain.c

@@ -20,7 +20,6 @@
 #include "taosmsg.h"
 #include "tlog.h"
 #include "trpc.h"
-#include "tstatus.h"
 #include "tsdb.h"
 #include "ttime.h"
 #include "ttimer.h"
@@ -29,18 +28,28 @@
 #include "vnode.h"
 #include "vnodeInt.h"
 
-static void   *tsDnodeVnodesHash;
+static int32_t  tsOpennedVnodes;
-static void    vnodeCleanUp(SVnodeObj *pVnode);
+static void    *tsDnodeVnodesHash;
-static void    vnodeBuildVloadMsg(char *pNode, void * param);
+static void     vnodeCleanUp(SVnodeObj *pVnode);
-static int     vnodeWALCallback(void *arg);
+static void     vnodeBuildVloadMsg(char *pNode, void * param);
-static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
+static int      vnodeWalCallback(void *arg);
-static int32_t vnodeReadCfg(SVnodeObj *pVnode);
+static int32_t  vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg);
+static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
+static int      vnodeWalCallback(void *arg);
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size);
+static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
+static void     vnodeNotifyRole(void *ahandle, int8_t role);
 
-static int   tsOpennedVnodes;
 static pthread_once_t  vnodeModuleInit = PTHREAD_ONCE_INIT;
 
+#ifndef _VPEER
+tsync_h syncStart(SSyncInfo *info) { return NULL; }
+int     syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle) { return 0; }
+#endif
+
 static void vnodeInit() {
   vnodeInitWriteFp();
+  vnodeInitReadFp();
 
   tsDnodeVnodesHash = taosInitIntHash(TSDB_MAX_VNODES, sizeof(SVnodeObj *), taosHashInt);
   if (tsDnodeVnodesHash == NULL) {
@@ -53,22 +62,11 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
   pthread_once(&vnodeModuleInit, vnodeInit);
 
   SVnodeObj *pTemp = (SVnodeObj *)taosGetIntHashData(tsDnodeVnodesHash, pVnodeCfg->cfg.vgId);
-
   if (pTemp != NULL) {
     dPrint("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp);
     return TSDB_CODE_SUCCESS;
   }
 
-  STsdbCfg tsdbCfg = {0};
-  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
-  tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
-  tsdbCfg.maxTables           = pVnodeCfg->cfg.maxSessions;
-  tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
-  tsdbCfg.minRowsPerFileBlock = -1;
-  tsdbCfg.maxRowsPerFileBlock = -1;
-  tsdbCfg.keep                = -1;
-  tsdbCfg.maxCacheSize        = -1;
-
   char rootDir[TSDB_FILENAME_LEN] = {0};
   sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
   if (mkdir(rootDir, 0755) != 0) {
@@ -88,6 +86,16 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
     return code;
   }
 
+  STsdbCfg tsdbCfg = {0};
+  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
+  tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
+  tsdbCfg.maxTables           = pVnodeCfg->cfg.maxSessions;
+  tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
+  tsdbCfg.minRowsPerFileBlock = -1;
+  tsdbCfg.maxRowsPerFileBlock = -1;
+  tsdbCfg.keep                = -1;
+  tsdbCfg.maxCacheSize        = -1;
+
   char tsdbDir[TSDB_FILENAME_LEN] = {0};
   sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
   code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL);
@@ -103,15 +111,15 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
 }
 
 int32_t vnodeDrop(int32_t vgId) {
-
+  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
-  SVnodeObj *pVnode = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  if (ppVnode == NULL || *ppVnode == NULL) {
-  if (pVnode == NULL) {
     dTrace("vgId:%d, failed to drop, vgId not exist", vgId);
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
 
+  SVnodeObj *pVnode = *ppVnode;
   dTrace("pVnode:%p vgId:%d, vnode will be dropped", pVnode, pVnode->vgId);
-  pVnode->status = VN_STATUS_DELETING;
+  pVnode->status = TAOS_VN_STATUS_DELETING;
   vnodeCleanUp(pVnode);
  
   return TSDB_CODE_SUCCESS;
@@ -123,7 +131,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   SVnodeObj *pVnode = calloc(sizeof(SVnodeObj), 1);
   pVnode->vgId     = vnode;
-  pVnode->status   = VN_STATUS_INIT;
+  pVnode->status   = TAOS_VN_STATUS_INIT;
   pVnode->refCount = 1;
   pVnode->version  = 0;  
   taosAddIntHash(tsDnodeVnodesHash, pVnode->vgId, (char *)(&pVnode));
@@ -139,14 +147,27 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pVnode->rqueue = dnodeAllocateRqueue(pVnode);
 
   sprintf(temp, "%s/wal", rootDir);
-  pVnode->wal      = walOpen(temp, 3, tsCommitLog);
+  pVnode->wal      = walOpen(temp, &pVnode->walCfg);
-  pVnode->sync     = NULL;
+
+  SSyncInfo syncInfo;
+  syncInfo.vgId = pVnode->vgId;
+  syncInfo.version = pVnode->version;
+  syncInfo.syncCfg = pVnode->syncCfg;
+  sprintf(syncInfo.path, "%s/tsdb/", rootDir);
+  syncInfo.ahandle = pVnode;
+  syncInfo.getWalInfo = vnodeGetWalInfo;
+  syncInfo.getFileInfo = vnodeGetFileInfo;
+  syncInfo.writeToCache = vnodeWriteToQueue;
+  syncInfo.confirmForward = dnodeSendRpcWriteRsp; 
+  syncInfo.notifyRole = vnodeNotifyRole;
+  pVnode->sync     = syncStart(&syncInfo);;
+
   pVnode->events   = NULL;
   pVnode->cq       = NULL;
 
   STsdbAppH appH = {0};
   appH.appH = (void *)pVnode;
-  appH.walCallBack = vnodeWALCallback;
+  appH.walCallBack = vnodeWalCallback;
 
   sprintf(temp, "%s/tsdb", rootDir);
   void *pTsdb = tsdbOpenRepo(temp, &appH);
@@ -160,20 +181,20 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   walRestore(pVnode->wal, pVnode, vnodeWriteToQueue);
 
-  pVnode->status = VN_STATUS_READY;
+  pVnode->status = TAOS_VN_STATUS_READY;
   dTrace("pVnode:%p vgId:%d, vnode is opened in %s", pVnode, pVnode->vgId, rootDir);
 
-  tsOpennedVnodes++;
+  atomic_add_fetch_32(&tsOpennedVnodes, 1);
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t vnodeClose(int32_t vgId) {
+  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  if (ppVnode == NULL || *ppVnode == NULL) return 0;
 
-  SVnodeObj *pVnode = *(SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  SVnodeObj *pVnode = *ppVnode;
-  if (pVnode == NULL) return 0;
-
   dTrace("pVnode:%p vgId:%d, vnode will be closed", pVnode, pVnode->vgId);
-  pVnode->status = VN_STATUS_CLOSING;
+  pVnode->status = TAOS_VN_STATUS_CLOSING;
   vnodeCleanUp(pVnode);
 
   return 0;
@@ -181,11 +202,13 @@ int32_t vnodeClose(int32_t vgId) {
 
 void vnodeRelease(void *pVnodeRaw) {
   SVnodeObj *pVnode = pVnodeRaw;
+  int32_t    vgId = pVnode->vgId;
 
   int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
+  assert(refCount >= 0);
 
   if (refCount > 0) {
-    dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, pVnode->vgId, refCount);
+    dTrace("pVnode:%p vgId:%d, release vnode, refCount:%d", pVnode, vgId, refCount);
     return;
   }
 
@@ -197,15 +220,15 @@ void vnodeRelease(void *pVnodeRaw) {
   dnodeFreeWqueue(pVnode->wqueue);
   pVnode->wqueue = NULL;
 
-  if (pVnode->status == VN_STATUS_DELETING) {
+  if (pVnode->status == TAOS_VN_STATUS_DELETING) {
     // remove the whole directory
   }
 
   dTrace("pVnode:%p vgId:%d, vnode is released", pVnode, pVnode->vgId);
   free(pVnode);
 
-  tsOpennedVnodes--;
+  int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1);
-  if (tsOpennedVnodes <= 0) {
+  if (count <= 0) {
     taosCleanUpIntHash(tsDnodeVnodesHash);
     vnodeModuleInit = PTHREAD_ONCE_INIT;
     tsDnodeVnodesHash = NULL;
@@ -213,13 +236,13 @@ void vnodeRelease(void *pVnodeRaw) {
 }
 
 void *vnodeGetVnode(int32_t vgId) {
-  SVnodeObj *pVnode = *(SVnodeObj **) taosGetIntHashData(tsDnodeVnodesHash, vgId);
+  SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId);
-  if (pVnode == NULL) {
+  if (ppVnode == NULL || *ppVnode == NULL) {
     terrno = TSDB_CODE_INVALID_VGROUP_ID;
-    return NULL;
+    assert(false);
   }
 
-  return pVnode;
+  return *ppVnode;
 }
 
 void *vnodeAccquireVnode(int32_t vgId) {
@@ -257,15 +280,15 @@ void vnodeBuildStatusMsg(void *param) {
 
 static void vnodeBuildVloadMsg(char *pNode, void * param) {
   SVnodeObj *pVnode = *(SVnodeObj **) pNode;
-  if (pVnode->status == VN_STATUS_DELETING) return;
+  if (pVnode->status == TAOS_VN_STATUS_DELETING) return;
 
   SDMStatusMsg *pStatus = param;
   if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
 
   SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
   pLoad->vgId = htonl(pVnode->vgId);
-  pLoad->vnode = htonl(pVnode->vgId);
   pLoad->status = pVnode->status;
+  pLoad->role = pVnode->role;
 }
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
@@ -280,11 +303,27 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
 }
 
 // TODO: this is a simple implement
-static int vnodeWALCallback(void *arg) {
+static int vnodeWalCallback(void *arg) {
   SVnodeObj *pVnode = arg;
   return walRenew(pVnode->wal);
 }
 
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) {
+  // SVnodeObj *pVnode = ahandle;
+  //tsdbGetFileInfo(pVnode->tsdb, name, index, size);
+  return 0;
+}
+
+static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) {
+  SVnodeObj *pVnode = ahandle;
+  return walGetWalFile(pVnode->wal, name, index);
+}
+
+static void vnodeNotifyRole(void *ahandle, int8_t role) {
+  SVnodeObj *pVnode = ahandle;
+  pVnode->role = role;
+}
+
 static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
   char cfgFile[TSDB_FILENAME_LEN * 2] = {0};
   sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnodeCfg->cfg.vgId);
@@ -292,9 +331,13 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
   FILE *fp = fopen(cfgFile, "w");
   if (!fp) return errno;
 
-  fprintf(fp, "replicas %d\n", pVnodeCfg->cfg.replications);
+  fprintf(fp, "commitLog %d\n", pVnodeCfg->cfg.commitLog);
+  fprintf(fp, "wals %d\n", 3);
+  fprintf(fp, "arbitratorIp %d\n", pVnodeCfg->vpeerDesc[0].ip);
+  fprintf(fp, "quorum %d\n", 1);
+  fprintf(fp, "replica %d\n", pVnodeCfg->cfg.replications);
   for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) {
-    fprintf(fp, "index%d dnode %d ip %u\n", i, pVnodeCfg->vpeerDesc[i].dnodeId, pVnodeCfg->vpeerDesc[i].ip);
+    fprintf(fp, "index%d nodeId %d nodeIp %u name n%d\n", i, pVnodeCfg->vpeerDesc[i].dnodeId, pVnodeCfg->vpeerDesc[i].ip, pVnodeCfg->vpeerDesc[i].dnodeId);
   }
 
   fclose(fp);
@@ -305,37 +348,64 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) {
 
 // TODO: this is a simple implement
 static int32_t vnodeReadCfg(SVnodeObj *pVnode) {
+  char option[5][16] = {0};
   char cfgFile[TSDB_FILENAME_LEN * 2] = {0};
   sprintf(cfgFile, "%s/vnode%d/config", tsVnodeDir, pVnode->vgId);
 
   FILE *fp = fopen(cfgFile, "r");
   if (!fp) return errno;
 
-  char    option[3][32] = {0};
+  int32_t commitLog = -1;
-  int32_t replicas = 0;
+  int32_t num = fscanf(fp, "%s %d", option[0], &commitLog);
-  int32_t num = fscanf(fp, "%s %d", option[0], &replicas);
   if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT;
-  if (strcmp(option[0], "replicas") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (strcmp(option[0], "commitLog") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
-  if (replicas == 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (commitLog == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
-  pVnode->replicas = replicas;
+  pVnode->walCfg.commitLog = (int8_t)commitLog;
 
-  for (int32_t i = 0; i < replicas; ++i) {
+  int32_t wals = -1;
-    int32_t  dnodeId = 0;
+  num = fscanf(fp, "%s %d", option[0], &wals);
-    uint32_t dnodeIp = 0;
+  if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT;
-    num = fscanf(fp, "%s %s %d %s %u", option[0], option[1], &dnodeId, option[2], &dnodeIp);
+  if (strcmp(option[0], "wals") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
-    if (num != 5) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (wals == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
-    if (strcmp(option[1], "dnode") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  pVnode->walCfg.wals = (int8_t)wals;
-    if (strcmp(option[2], "ip") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
-    if (dnodeId == 0) return TSDB_CODE_INVALID_FILE_FORMAT;
-    if (dnodeIp == 0) return TSDB_CODE_INVALID_FILE_FORMAT;
 
-    pVnode->vpeers[i].dnodeId = dnodeId;
+  int32_t arbitratorIp = -1;
-    pVnode->vpeers[i].ip = dnodeIp;
+  num = fscanf(fp, "%s %u", option[0], &arbitratorIp);
-    pVnode->vpeers[i].vgId = pVnode->vgId;
+  if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (strcmp(option[0], "arbitratorIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (arbitratorIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
+  pVnode->syncCfg.arbitratorIp = arbitratorIp;
+
+  int32_t quorum = -1;
+  num = fscanf(fp, "%s %d", option[0], &quorum);
+  if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (strcmp(option[0], "quorum") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (quorum == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
+  pVnode->syncCfg.quorum = (int8_t)quorum;
+
+  int32_t replica = -1;
+  num = fscanf(fp, "%s %d", option[0], &replica);
+  if (num != 2) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (strcmp(option[0], "replica") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+  if (replica == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
+  pVnode->syncCfg.replica = (int8_t)replica;
+
+  for (int32_t i = 0; i < replica; ++i) {
+    int32_t  dnodeId = -1;
+    uint32_t dnodeIp = -1;
+    num = fscanf(fp, "%s %s %d %s %u %s %s", option[0], option[1], &dnodeId, option[2], &dnodeIp, option[3], pVnode->syncCfg.nodeInfo[i].name);
+    if (num != 7) return TSDB_CODE_INVALID_FILE_FORMAT;
+    if (strcmp(option[1], "nodeId") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+    if (strcmp(option[2], "nodeIp") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+    if (strcmp(option[3], "name") != 0) return TSDB_CODE_INVALID_FILE_FORMAT;
+    if (dnodeId == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
+    if (dnodeIp == -1) return TSDB_CODE_INVALID_FILE_FORMAT;
+    pVnode->syncCfg.nodeInfo[i].nodeId = dnodeId;
+    pVnode->syncCfg.nodeInfo[i].nodeIp = dnodeIp;
   }
 
   fclose(fp);
   dTrace("pVnode:%p vgId:%d, read vnode cfg successed", pVnode, pVnode->vgId);
 
   return TSDB_CODE_SUCCESS;
-}
+}

src/vnode/main/src/vnodeRead.c

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "taoserror.h"
+#include "tlog.h"
+#include "tqueue.h"
+#include "trpc.h"
+#include "tsdb.h"
+#include "twal.h"
+#include "dataformat.h"
+#include "vnode.h"
+#include "vnodeInt.h"
+#include "queryExecutor.h"
+
+static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, int32_t contLen, SRspRet *pRet);
+static int32_t  vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet);
+static int32_t  vnodeProcessRetrieveMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet);
+
+void vnodeInitReadFp(void) {
+  vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY]    = vnodeProcessQueryMsg;
+  vnodeProcessReadMsgFp[TSDB_MSG_TYPE_RETRIEVE] = vnodeProcessRetrieveMsg;
+}
+
+int32_t vnodeProcessRead(void *param, int msgType, void *pCont, int32_t contLen, SRspRet *ret) {
+  SVnodeObj *pVnode = (SVnodeObj *)param;
+
+  if (vnodeProcessReadMsgFp[msgType] == NULL) 
+    return TSDB_CODE_MSG_NOT_PROCESSED; 
+
+  if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) 
+    return TSDB_CODE_NOT_ACTIVE_VNODE; 
+
+  return (*vnodeProcessReadMsgFp[msgType])(pVnode, pCont, contLen, ret);
+}
+
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) {
+  SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pCont;
+  memset(pRet, 0, sizeof(SRspRet));
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  
+  SQInfo* pQInfo = NULL;
+  if (contLen != 0) {
+    void* tsdb = vnodeGetTsdb(pVnode);
+    pRet->code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo);
+  
+    SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
+    pRsp->qhandle = htobe64((uint64_t) (pQInfo));
+    pRsp->code = pRet->code;
+     
+    pRet->len = sizeof(SQueryTableRsp);
+    pRet->rsp = pRsp;
+    
+    dTrace("pVnode:%p vgId:%d QInfo:%p, dnode query msg disposed", pVnode, pVnode->vgId, pQInfo);
+  } else {
+    pQInfo = pCont;
+    code = TSDB_CODE_ACTION_IN_PROGRESS;
+  }
+
+  qTableQuery(pQInfo); // do execute query
+  
+  return code;
+}
+
+static int32_t vnodeProcessRetrieveMsg(SVnodeObj *pVnode, void *pCont, int32_t contLen, SRspRet *pRet) {
+  SRetrieveTableMsg *pRetrieve = pCont;
+  void *pQInfo = (void*) htobe64(pRetrieve->qhandle);
+  memset(pRet, 0, sizeof(SRspRet));
+
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  dTrace("pVnode:%p vgId:%d QInfo:%p, retrieve msg is received", pVnode, pVnode->vgId, pQInfo);
+  
+  pRet->code = qRetrieveQueryResultInfo(pQInfo);
+  if (pRet->code != TSDB_CODE_SUCCESS) {
+    //TODO
+    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+  } else {
+    // todo check code and handle error in build result set
+    pRet->code = qDumpRetrieveResult(pQInfo, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len);
+
+    if (qHasMoreResultsToRetrieve(pQInfo)) {
+      pRet->qhandle = pQInfo;
+      code = TSDB_CODE_ACTION_NEED_REPROCESSED;
+    } else {  
+      // no further execution invoked, release the ref to vnode
+      qDestroyQueryInfo(pQInfo);
+      vnodeRelease(pVnode);
+    }
+  }
+  
+  dTrace("pVnode:%p vgId:%d QInfo:%p, retrieve msg is disposed", pVnode, pVnode->vgId, pQInfo);
+  return code;
+}

src/vnode/main/src/vnodeWrite.c

@@ -25,6 +25,7 @@
 #include "dataformat.h"
 #include "vnode.h"
 #include "vnodeInt.h"
+#include "tutil.h"
 
 static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *, SRspRet *);
 static int32_t  vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pMsg, SRspRet *);
@@ -41,21 +42,23 @@ void vnodeInitWriteFp(void) {
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = vnodeProcessDropStableMsg;
 }
 
-int32_t vnodeProcessWrite(void *param, int qtype, SWalHead *pHead, void *item) {
+int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
   int32_t    code = 0;
-  SVnodeObj *pVnode = (SVnodeObj *)param;
+  SVnodeObj *pVnode = (SVnodeObj *)param1;
+  SWalHead  *pHead = param2;
 
   if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) 
     return TSDB_CODE_MSG_NOT_PROCESSED; 
 
-  if (pVnode->status == VN_STATUS_DELETING || pVnode->status == VN_STATUS_CLOSING) 
+  if (pVnode->status == TAOS_VN_STATUS_DELETING || pVnode->status == TAOS_VN_STATUS_CLOSING) 
     return TSDB_CODE_NOT_ACTIVE_VNODE; 
 
   if (pHead->version == 0) { // from client 
-    if (pVnode->status != VN_STATUS_READY) 
+    if (pVnode->status != TAOS_VN_STATUS_READY) 
       return TSDB_CODE_NOT_ACTIVE_VNODE;
 
-    // if (pVnode->replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER)
+    if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER)
+      return TSDB_CODE_NO_MASTER;
 
     // assign version
     pVnode->version++;
@@ -76,11 +79,8 @@ int32_t vnodeProcessWrite(void *param, int qtype, SWalHead *pHead, void *item) {
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item);
   if (code < 0) return code;
 
-/* forward
+  if (pVnode->syncCfg.replica > 1) 
-  if (pVnode->replica > 1 && pVnode->role == TAOS_SYNC_ROLE_MASTER) {
     code = syncForwardToPeer(pVnode->sync, pHead, item);
-  }
-*/
 
   return code;
 }
@@ -110,36 +110,27 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
   int32_t code = 0;
 
   dTrace("pVnode:%p vgId:%d, table:%s, start to create", pVnode, pVnode->vgId, pTable->tableId);
-  pTable->numOfColumns  = htons(pTable->numOfColumns);
+  int16_t numOfColumns  = htons(pTable->numOfColumns);
-  pTable->numOfTags     = htons(pTable->numOfTags);
+  int16_t numOfTags     = htons(pTable->numOfTags);
-  pTable->sid           = htonl(pTable->sid);
+  int32_t sid           = htonl(pTable->sid);
-  pTable->sversion      = htonl(pTable->sversion);
+  uint64_t uid           = htobe64(pTable->uid);
-  pTable->tagDataLen    = htonl(pTable->tagDataLen);
-  pTable->sqlDataLen    = htonl(pTable->sqlDataLen);
-  pTable->uid           = htobe64(pTable->uid);
-  pTable->superTableUid = htobe64(pTable->superTableUid);
-  pTable->createdTime   = htobe64(pTable->createdTime);
   SSchema *pSchema = (SSchema *) pTable->data;
 
-  int totalCols = pTable->numOfColumns + pTable->numOfTags;
+  int32_t totalCols = numOfColumns + numOfTags;
-  for (int i = 0; i < totalCols; i++) {
+  
-    pSchema[i].colId = htons(pSchema[i].colId);
-    pSchema[i].bytes = htons(pSchema[i].bytes);
-  }
-
   STableCfg tCfg;
-  tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid);
+  tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid);
 
-  STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns);
+  STSchema *pDestSchema = tdNewSchema(numOfColumns);
-  for (int i = 0; i < pTable->numOfColumns; i++) {
+  for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes);
+    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
 
-  if (pTable->numOfTags != 0) {
+  if (numOfTags != 0) {
-    STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags);
+    STSchema *pDestTagSchema = tdNewSchema(numOfTags);
-    for (int i = pTable->numOfColumns; i < totalCols; i++) {
+    for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes);
+      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
     tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
 
@@ -147,9 +138,9 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
     int accumBytes = 0;
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
-    for (int i = 0; i < pTable->numOfTags; i++) {
+    for (int i = 0; i < numOfTags; i++) {
       tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
-      accumBytes += pSchema[i + pTable->numOfColumns].bytes;
+      accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);
   }
@@ -157,6 +148,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
   void *pTsdb = vnodeGetTsdb(pVnode);
   code = tsdbCreateTable(pTsdb, &tCfg);
 
+  tfree(pDestSchema);
+
   dTrace("pVnode:%p vgId:%d, table:%s is created, result:%x", pVnode, pVnode->vgId, pTable->tableId, code);
   return code; 
 }
@@ -181,51 +174,46 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
   int32_t code = 0;
 
   dTrace("pVnode:%p vgId:%d, table:%s, start to alter", pVnode, pVnode->vgId, pTable->tableId);
-  pTable->numOfColumns  = htons(pTable->numOfColumns);
+  int16_t numOfColumns  = htons(pTable->numOfColumns);
-  pTable->numOfTags     = htons(pTable->numOfTags);
+  int16_t numOfTags     = htons(pTable->numOfTags);
-  pTable->sid           = htonl(pTable->sid);
+  int32_t sid           = htonl(pTable->sid);
-  pTable->sversion      = htonl(pTable->sversion);
+  uint64_t uid           = htobe64(pTable->uid);
-  pTable->tagDataLen    = htonl(pTable->tagDataLen);
-  pTable->sqlDataLen    = htonl(pTable->sqlDataLen);
-  pTable->uid           = htobe64(pTable->uid);
-  pTable->superTableUid = htobe64(pTable->superTableUid);
-  pTable->createdTime   = htobe64(pTable->createdTime);
   SSchema *pSchema = (SSchema *) pTable->data;
 
-  int totalCols = pTable->numOfColumns + pTable->numOfTags;
+  int32_t totalCols = numOfColumns + numOfTags;
-  for (int i = 0; i < totalCols; i++) {
+  
-    pSchema[i].colId = htons(pSchema[i].colId);
-    pSchema[i].bytes = htons(pSchema[i].bytes);
-  }
-
   STableCfg tCfg;
-  tsdbInitTableCfg(&tCfg, pTable->tableType, pTable->uid, pTable->sid);
+  tsdbInitTableCfg(&tCfg, pTable->tableType, uid, sid);
 
-  STSchema *pDestSchema = tdNewSchema(pTable->numOfColumns);
+  STSchema *pDestSchema = tdNewSchema(numOfColumns);
-  for (int i = 0; i < pTable->numOfColumns; i++) {
+  for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes);
+    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
 
-  if (pTable->numOfTags != 0) {
+  if (numOfTags != 0) {
-    STSchema *pDestTagSchema = tdNewSchema(pTable->numOfTags);
+    STSchema *pDestTagSchema = tdNewSchema(numOfTags);
-    for (int i = pTable->numOfColumns; i < totalCols; i++) {
+    for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, pSchema[i].colId, pSchema[i].bytes);
+      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
-    tsdbTableSetSchema(&tCfg, pDestTagSchema, false);
+    tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
 
     char *pTagData = pTable->data + totalCols * sizeof(SSchema);
     int accumBytes = 0;
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
-    for (int i = 0; i < pTable->numOfTags; i++) {
+    for (int i = 0; i < numOfTags; i++) {
       tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
-      accumBytes += pSchema[i + pTable->numOfColumns].bytes;
+      accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);
   }
 
-  code = tsdbAlterTable(pVnode->tsdb, &tCfg);
+  void *pTsdb = vnodeGetTsdb(pVnode);
+  code = tsdbAlterTable(pTsdb, &tCfg);
+
+  tfree(pDestSchema);
+
   dTrace("pVnode:%p vgId:%d, table:%s, alter table result:%d", pVnode, pVnode->vgId, pTable->tableId, code);
 
   return code;
@@ -236,7 +224,7 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
   int32_t code = 0;
 
   dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId);
-  pTable->uid = htobe64(pTable->uid);
+  // int64_t uid = htobe64(pTable->uid);
 
   // TODO: drop stable in vvnode
   //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode);
@@ -248,8 +236,9 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
   return code;
 }
 
-int vnodeWriteToQueue(void *param, SWalHead *pHead, int type) {
+int vnodeWriteToQueue(void *param, void *data, int type) {
   SVnodeObj *pVnode = param;
+  SWalHead *pHead = data;
 
   int size = sizeof(SWalHead) + pHead->len;
   SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);

src/vnode/tsdb/src/tsdbFile.c

@@ -62,6 +62,7 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) {
       // TODO
     }
   }
+  closedir(dir);
 
   return pFileH;
 }

src/vnode/tsdb/src/tsdbMain.c

@@ -269,6 +269,9 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
 
   tsdbFreeCache(pRepo->tsdbCache);
 
+  tfree(pRepo->rootDir);
+  tfree(pRepo);
+
   return 0;
 }
 
@@ -849,6 +852,7 @@ static void *tsdbCommitData(void *arg) {
 
   tsdbLockRepo(arg);
   tdListMove(pCache->imem->list, pCache->pool.memPool);
+  tdListFree(pCache->imem->list);
   free(pCache->imem);
   pCache->imem = NULL;
   pRepo->commit = 0;
@@ -1127,11 +1131,11 @@ static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsTo
     *len += pCompCol->len;
   }
 
-  if (pCompData == NULL) free((void *)pCompData);
+  tfree(pCompData);
   return 0;
 
 _err:
-  if (pCompData == NULL) free((void *)pCompData);
+  tfree(pCompData);
   return -1;
 }
 

src/vnode/tsdb/src/tsdbMeta.c

@@ -317,6 +317,14 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
 //   return 0;
 // }
 
+static void tsdbFreeMemTable(SMemTable *pMemTable) {
+  if (pMemTable) {
+    tSkipListDestroy(pMemTable->pData);
+  }
+
+  free(pMemTable);
+}
+
 static int tsdbFreeTable(STable *pTable) {
   // TODO: finish this function
   if (pTable->type == TSDB_CHILD_TABLE) {
@@ -328,7 +336,10 @@ static int tsdbFreeTable(STable *pTable) {
   // Free content
   if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) {
     tSkipListDestroy(pTable->pIndex);
-  } 
+  }
+
+  tsdbFreeMemTable(pTable->mem);
+  tsdbFreeMemTable(pTable->imem);
 
   free(pTable);
   return 0;

src/vnode/tsdb/src/tsdbMetaFile.c

@@ -177,6 +177,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) {
   close(mfh->fd);
 
   taosHashCleanup(mfh->map);
+  tfree(mfh);
 }
 
 static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {

src/vnode/wal/src/walMain.c

@@ -49,18 +49,18 @@ int wDebugFlag = 135;
 
 static uint32_t walSignature = 0xFAFBFDFE;
 static int walHandleExistingFiles(char *path);
-static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int));
+static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int));
 static int walRemoveWalFiles(char *path);
 
-void *walOpen(char *path, int max, int level) {
+void *walOpen(char *path, SWalCfg *pCfg) {
   SWal *pWal = calloc(sizeof(SWal), 1);
   if (pWal == NULL) return NULL;
 
   pWal->fd = -1;
-  pWal->max = max;
+  pWal->max = pCfg->wals;
   pWal->id = 0;
   pWal->num = 0;
-  pWal->level = level;
+  pWal->level = pCfg->commitLog;
   strcpy(pWal->path, path);
   pthread_mutex_init(&pWal->mutex, NULL);
 
@@ -80,7 +80,8 @@ void *walOpen(char *path, int max, int level) {
 }
 
 void walClose(void *handle) {
- 
+  if (handle == NULL) return;
+  
   SWal *pWal = (SWal *)handle;
   
   close(pWal->fd);
@@ -169,7 +170,7 @@ void walFsync(void *handle) {
     fsync(pWal->fd);
 }
 
-int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) {
+int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) {
   SWal    *pWal = (SWal *)handle;
   int      code = 0;
   struct   dirent *ent;
@@ -246,7 +247,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
   return code;
 }  
 
-static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, SWalHead *, int)) {
+static int walRestoreWalFile(char *name, void *pVnode, int (*writeFp)(void *, void *, int)) {
   int code = 0;
 
   char *buffer = malloc(1024000);  // size for one record

src/vnode/wal/test/waltest.c

@@ -21,8 +21,10 @@
 int64_t  ver = 0;
 void    *pWal = NULL;
 
-int writeToQueue(void *pVnode, SWalHead *pHead, int type) {
+int writeToQueue(void *pVnode, void *data, int type) {
   // do nothing
+  SWalHead *pHead = data;
+
   if (pHead->version > ver)
     ver = pHead->version;
 
@@ -74,7 +76,11 @@ int main(int argc, char *argv[]) {
 
   taosInitLog("wal.log", 100000, 10);
 
-  pWal = walOpen(path, max, level);
+  SWalCfg walCfg;
+  walCfg.commitLog = level;
+  walCfg.wals = max;
+
+  pWal = walOpen(path, &walCfg);
   if (pWal == NULL) {
     printf("failed to open wal\n");
     exit(-1);

tests/CMakeLists.txt

@@ -11,3 +11,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON)
 
 ADD_SUBDIRECTORY(examples/c)
 ADD_SUBDIRECTORY(tsim)
+ADD_SUBDIRECTORY(test/c)

tests/script/basicSuite.sim

@@ -1,8 +1,8 @@
 ################################# 
 
-run general/user/basic1.sim
+run general/table/basic1.sim
-
+run general/table/basic2.sim
-run general/show/dnodes.sim
+run general/table/basic3.sim
 
 run general/db/basic1.sim
 run general/db/basic2.sim
@@ -10,8 +10,4 @@ run general/db/basic3.sim
 run general/db/basic4.sim
 run general/db/basic5.sim
 
-run general/table/basic1.sim
-run general/table/basic2.sim
-run general/table/basic3.sim
-
 ################################## 

tests/script/general/db/basic4.sim

@@ -36,7 +36,7 @@ sql show d1.vgroups
 if $rows != 1 then 
   return -1
 endi
-if $data00 != 1 then 
+if $data00 != 2 then 
   return -1
 endi
 if $data01 != 4 then 
@@ -75,7 +75,7 @@ sql show d1.vgroups
 if $rows != 1 then 
   return -1
 endi
-if $data00 != 1 then 
+if $data00 != 2 then 
   return -1
 endi
 if $data01 != 3 then 

tests/script/general/db/basic5.sim

@@ -36,7 +36,7 @@ sql show d1.vgroups
 if $rows != 1 then 
   return -1
 endi
-if $data00 != 1 then 
+if $data00 != 2 then 
   return -1
 endi
 if $data01 != 4 then 
@@ -46,7 +46,7 @@ if $data02 != ready then
   return -1
 endi
 
-print =============== drop table 
+print =============== drop database
 sql drop database d1
 
 sql show databases

tests/script/general/table/basic1.sim

@@ -43,9 +43,9 @@ print $data10 $data11 $data22
 print $data20 $data11 $data22
 
 print =============== insert data
-sql insert into d1.n1 values(now, 1)
+sql insert into d1.n1 values(now+1s, 1)
-sql insert into d1.n1 values(now, 2)
+sql insert into d1.n1 values(now+2s, 2)
-sql insert into d1.n1 values(now, 3)
+sql insert into d1.n1 values(now+3s, 3)
 
 print =============== query data
 sql select * from d1.n1
@@ -69,7 +69,3 @@ if $data21 != 3 then
   return -1
 endi
 
-sql drop database d1
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-

tests/script/general/table/basic2.sim

@@ -29,7 +29,7 @@ if $data03 != 1 then
 endi
 
 sql show d1.vgroups
-if $data00 != 1 then 
+if $data00 != 2 then 
   return -1
 endi
 
@@ -43,11 +43,12 @@ sql_error insert into d1.n2 values(now, 1)
 
 print =============== insert data2
 sql insert into d1.n3 values(now, 1)
-sql insert into d1.n3 values(now, 2)
+sql insert into d1.n3 values(now+1s, 2)
-sql insert into d1.n3 values(now, 3)
+sql insert into d1.n3 values(now+2s, 3)
 
 print =============== query data
 sql select * from d1.n3
+print $rows
 if $rows != 3 then 
   return -1
 endi
@@ -64,6 +65,3 @@ if $data21 != 3 then
   return -1
 endi
 
-sql drop database d1
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/table/basic3.sim

@@ -43,9 +43,9 @@ print $data10 $data11 $data22
 print $data20 $data11 $data22
 
 print =============== insert data
-sql insert into db.n1 values(now, 1)
+sql insert into db.n1 values(now+1s, 1)
-sql insert into db.n1 values(now, 2)
+sql insert into db.n1 values(now+2s, 2)
-sql insert into db.n1 values(now, 3)
+sql insert into db.n1 values(now+3s, 3)
 
 print =============== query data
 sql select * from db.n1

tests/script/sh/stop_dnodes.sh

@@ -6,8 +6,10 @@ if [ -n "$PID" ]; then
 	sudo systemctl stop taosd
 fi
   
-PID=`ps -ef|grep taosd | grep -v grep | awk '{print $2}'`
+PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
-if [ -n "$PID" ]; then 
+while [ -n "$PID" ]; do
 	echo sudo kill -9 $PID 
-	sudo kill -9 $PID
+	sudo pkill -9 taosd
-fi 
+  sudo fuser -k -n tcp 6030
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+done

tests/script/tmp/prepare.sim

@@ -1,3 +1,4 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
-system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2
+system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2
+system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3

tests/test/c/CMakeLists.txt

@@ -0,0 +1,11 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  add_executable(benchmarkPerTable benchmarkPerTable.c)
+  target_link_libraries(benchmarkPerTable taos_static pthread)
+ENDIF()

tests/test/c/benchmarkPerTable.c

@@ -0,0 +1,319 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tlog.h"
+#include "ttimer.h"
+#include "tutil.h"
+
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+typedef struct {
+  int64_t   rowsPerTable;
+  int64_t   pointsPerTable;
+  int64_t   tableBeginIndex;
+  int64_t   tableEndIndex;
+  int       threadIndex;
+  char      dbName[32];
+  char      stableName[64];
+  pthread_t thread;
+} SInfo;
+
+void *syncTest(void *param);
+void  generateRandomPoints();
+void  shellParseArgument(int argc, char *argv[]);
+void  createDbAndTable();
+void  insertData();
+
+int32_t randomData[MAX_RANDOM_POINTS];
+int64_t rowsPerTable = 10000;
+int64_t pointsPerTable = 1;
+int64_t numOfThreads = 1;
+int64_t numOfTablesPerThread = 1;
+char    dbName[32] = "db";
+char    stableName[64] = "st";
+int32_t cache = 16384;
+int32_t tables = 1000;
+
+int main(int argc, char *argv[]) {
+  shellParseArgument(argc, argv);
+  generateRandomPoints();
+  taos_init();
+  createDbAndTable();
+  insertData();
+}
+
+void createDbAndTable() {
+  dPrint("start to create table");
+
+  TAOS *         con;
+  struct timeval systemTime;
+  int64_t        st, et;
+  char           qstr[64000];
+
+  con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
+  if (con == NULL) {
+    dError("failed to connect to DB, reason:%s", taos_errstr(con));
+    exit(1);
+  }
+
+  sprintf(qstr, "create database if not exists %s cache %d tables %d", dbName, cache, tables);
+  if (taos_query(con, qstr)) {
+    dError("failed to create database:%s, code:%d reason:%s", dbName, taos_errno(con), taos_errstr(con));
+    exit(0);
+  }
+
+  sprintf(qstr, "use %s", dbName);
+  if (taos_query(con, qstr)) {
+    dError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+    exit(0);
+  }
+
+  gettimeofday(&systemTime, NULL);
+  st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+  int64_t totalTables = numOfTablesPerThread * numOfThreads;
+    
+  if (strcmp(stableName, "no") != 0) {
+    int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
+    for (int64_t f = 0; f < pointsPerTable; ++f) {
+      len += sprintf(qstr + len, ", f%ld double", f);
+    }
+    sprintf(qstr + len, ") tags(t int)");
+
+    if (taos_query(con, qstr)) {
+      dError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+      exit(0);
+    }
+    
+    for (int64_t t = 0; t < totalTables; ++t) {
+      sprintf(qstr, "create table if not exists %s%ld using %s tags(%ld)", stableName, t, stableName, t);
+      if (taos_query(con, qstr)) {
+        dError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
+        exit(0);
+      }
+    }
+  } else {
+    for (int64_t t = 0; t < totalTables; ++t) {
+      int len = sprintf(qstr, "create table if not exists %s%ld(ts timestamp", stableName, t);
+      for (int64_t f = 0; f < pointsPerTable; ++f) {
+        len += sprintf(qstr + len, ", f%ld double", f);
+      }
+      sprintf(qstr + len, ")");
+
+      if (taos_query(con, qstr)) {
+        dError("failed to create table %s%ld, reason:%s", stableName, t, taos_errstr(con));
+        exit(0);
+      }
+    }
+  }
+
+  gettimeofday(&systemTime, NULL);
+  et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+  dPrint("%.1f seconds to create %ld tables", (et - st) / 1000.0 / 1000.0, totalTables);
+}
+
+void insertData() {
+  struct timeval systemTime;
+  int64_t        st, et;
+
+  gettimeofday(&systemTime, NULL);
+  st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+
+  dPrint("%d threads are spawned to insert data", numOfThreads);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads);
+
+  // Start threads to write
+  for (int i = 0; i < numOfThreads; ++i) {
+    pInfo[i].rowsPerTable = rowsPerTable;
+    pInfo[i].pointsPerTable = pointsPerTable;
+    pInfo[i].tableBeginIndex = i * numOfTablesPerThread;
+    pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread;
+    pInfo[i].threadIndex = i;
+    strcpy(pInfo[i].dbName, dbName);
+    strcpy(pInfo[i].stableName, stableName);
+    pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i));
+  }
+
+  taosMsleep(300);
+  for (int i = 0; i < numOfThreads; i++) {
+    pthread_join(pInfo[i].thread, NULL);
+  }
+
+  gettimeofday(&systemTime, NULL);
+  et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+  double seconds = (et - st) / 1000.0 / 1000.0;
+
+  int64_t totalTables = numOfTablesPerThread * numOfThreads;
+  int64_t totalRows = totalTables * rowsPerTable;
+  int64_t totalPoints = totalTables * rowsPerTable * pointsPerTable;
+  double  speedOfRows = totalRows / seconds;
+  double  speedOfPoints = totalPoints / seconds;
+
+  dPrint(
+      "%sall threads:%ld finished, use %.1lf seconds, tables:%.ld rows:%ld points:%ld, speed RowsPerSecond:%.1lf "
+      "PointsPerSecond:%.1lf%s",
+      GREEN, numOfThreads, seconds, totalTables, totalRows, totalPoints, speedOfRows, speedOfPoints, NC);
+
+  dPrint("threads exit");
+
+  pthread_attr_destroy(&thattr);
+  free(pInfo);
+}
+
+void *syncTest(void *param) {
+  TAOS *         con;
+  SInfo *        pInfo = (SInfo *)param;
+  struct timeval systemTime;
+  int64_t        st, et;
+  char           qstr[65000];
+  int            maxBytes = 60000;
+
+  dPrint("thread:%d, start to run", pInfo->threadIndex);
+
+  con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0);
+  if (con == NULL) {
+    dError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
+    exit(1);
+  }
+
+  sprintf(qstr, "use %s", pInfo->dbName);
+  taos_query(con, qstr);
+
+  gettimeofday(&systemTime, NULL);
+  st = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+
+  int64_t start = 1430000000000;
+  int64_t interval = 1000;  // 1000 ms
+
+  char *sql = qstr;
+  char  inserStr[] = "insert into";
+  int   len = sprintf(sql, "%s", inserStr);
+
+  for (int64_t table = pInfo->tableBeginIndex; table < pInfo->tableEndIndex; ++table) {
+    len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
+    for (int64_t row = 0; row < pInfo->rowsPerTable; row++) {
+      len += sprintf(sql + len, "(%ld", start + row * interval);
+      for (int64_t point = 0; point < pInfo->pointsPerTable; ++point) {
+        len += sprintf(sql + len, ",%d", randomData[(123 * table + 456 * row + 789 * point) % MAX_RANDOM_POINTS]);
+        // len += sprintf(sql + len, ",%ld", row);
+      }
+      len += sprintf(sql + len, ")");
+      if (len > maxBytes) {
+        if (taos_query(con, qstr)) {
+          dError("thread:%d, failed to insert table:%s%ld row:%ld, reason:%s", pInfo->threadIndex, pInfo->stableName,
+                 table, row, taos_errstr(con));
+        }
+
+        // "insert into"
+        len = sprintf(sql, "%s", inserStr);
+
+        // "insert into st1 values"
+        if (row != pInfo->rowsPerTable - 1) {
+          len += sprintf(sql + len, " %s%ld values", pInfo->stableName, table);
+        }
+      }
+    }
+  }
+
+  if (len != strlen(inserStr)) {
+    taos_query(con, qstr);
+  }
+
+  gettimeofday(&systemTime, NULL);
+  et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
+  int64_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
+  int64_t totalRows = totalTables * pInfo->rowsPerTable;
+  int64_t totalPoints = totalRows * pInfo->pointsPerTable;
+  dPrint("thread:%d, insert finished, use %.2f seconds, tables:%ld rows:%ld points:%ld", pInfo->threadIndex,
+         (et - st) / 1000.0 / 1000.0, totalTables, totalRows, totalPoints);
+
+  return NULL;
+}
+
+void generateRandomPoints() {
+  for (int r = 0; r < MAX_RANDOM_POINTS; ++r) {
+    randomData[r] = rand() % 1000;
+  }
+}
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance of TDengine, the insert method is table-by-table\n");
+
+  printf("%s%s\n", indent, "-d");
+  printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
+  printf("%s%s\n", indent, "-s");
+  printf("%s%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName, ", if 'no' then create normal table");
+  printf("%s%s\n", indent, "-c");
+  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
+  printf("%s%s\n", indent, "-r");
+  printf("%s%s%s%ld\n", indent, indent, "Number of records to write to each table, default is ", rowsPerTable);
+  printf("%s%s\n", indent, "-p");
+  printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of columns per table, default is ", pointsPerTable);
+  printf("%s%s\n", indent, "-t");
+  printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads);
+  printf("%s%s\n", indent, "-n");
+  printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread);
+  printf("%s%s\n", indent, "-tables");
+  printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables);
+  printf("%s%s\n", indent, "-cache");
+  printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache);
+
+  exit(EXIT_SUCCESS);
+}
+
+void shellParseArgument(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-d") == 0) {
+      strcpy(dbName, argv[++i]);
+    } else if (strcmp(argv[i], "-s") == 0) {
+      strcpy(stableName, argv[++i]);
+    } else if (strcmp(argv[i], "-r") == 0) {
+      rowsPerTable = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-p") == 0) {
+      pointsPerTable = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t") == 0) {
+      numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      numOfTablesPerThread = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-tables") == 0) {
+      tables = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-cache") == 0) {
+      cache = atoi(argv[++i]);
+    } else {
+    }
+  }
+
+  dPrint("%srowsPerTable:%" PRId64 "%s", GREEN, rowsPerTable, NC);
+  dPrint("%spointsPerTable:%" PRId64 "%s", GREEN, pointsPerTable, NC);
+  dPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC);
+  dPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC);
+  dPrint("%scache:%" PRId64 "%s", GREEN, cache, NC);
+  dPrint("%stables:%" PRId64 "%s", GREEN, tables, NC);
+  dPrint("%sdbName:%s%s", GREEN, dbName, NC);
+  dPrint("%stableName:%s%s", GREEN, stableName, NC);
+  dPrint("%sstart to run%s", GREEN, NC);
+}

tests/tsim/src/simExe.c

@@ -576,6 +576,7 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) {
 bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
   simCloseTaosdConnect(script);
   void *taos = NULL;
+  taosMsleep(2000);
   for (int attempt = 0; attempt < 10; ++attempt) {
     taos = taos_connect(NULL, user, pass, NULL, tsMnodeShellPort);
     if (taos == NULL) {