Merge branch '2.0' into liaohj_2

src/common/inc/dataformat.h

@@ -51,44 +51,53 @@ void      tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes);
 
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
-  int32_t  numOfCols;
+  int      numOfCols;  // Number of columns appended
-  int32_t  padding;  // TODO: replace the padding for useful variable
+  int      totalCols;  // Total columns allocated
   STColumn columns[];
 } STSchema;
 
 #define schemaNCols(s) ((s)->numOfCols)
+#define schemaTCols(s) ((s)->totalCols)
 #define schemaColAt(s, i) ((s)->columns + i)
 
 STSchema *tdNewSchema(int32_t nCols);
+int       tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int16_t bytes);
 STSchema *tdDupSchema(STSchema *pSchema);
-void tdFreeSchema(STSchema *pSchema);
+void      tdFreeSchema(STSchema *pSchema);
-void tdUpdateSchema(STSchema *pSchema);
+void      tdUpdateSchema(STSchema *pSchema);
 
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * +---------+---------------------------------+
+ * +----------+---------+---------------------------------+---------------------------------+
- * | int32_t |                                 |
+ * | int32_t  | int32_t |                                 |                                 |
- * +---------+---------------------------------+
+ * +----------+---------+---------------------------------+---------------------------------+
- * |   len   |                row              |
+ * |   len    |  flen   |           First part            |             Second part         |
- * +---------+---------------------------------+
+ * +----------+---------+---------------------------------+---------------------------------+
+ * plen: first part length
  * len: the length including sizeof(row) + sizeof(len)
  * row: actual row data encoding
  */
 typedef void *SDataRow;
 
+#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
+
 #define dataRowLen(r) (*(int32_t *)(r))
-#define dataRowTuple(r) ((char *)(r) + sizeof(int32_t))
+#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
+#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
+#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
 #define dataRowIdx(r, i) ((char *)(r) + i)
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
+#define dataRowAt(r, idx) ((char *)(r) + (idx))
 
-SDataRow tdNewDataRow(int32_t bytes);
+void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-// SDataRow tdNewDdataFromSchema(SSchema *pSchema);
+int      tdMaxRowBytesFromSchema(STSchema *pSchema);
+SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema);
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
-// int32_t  tdAppendColVal(SDataRow row, void *value, SColumn *pCol, int32_t suffixOffset);
+int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
-void     tdDataRowCpy(void *dst, SDataRow row);
+void     tdDataRowReset(SDataRow row, STSchema *pSchema);
-void     tdDataRowReset(SDataRow row);
 SDataRow tdDataRowDup(SDataRow row);
 
 /* Data rows definition, the format of it is like below:

src/common/src/dataformat.c

@@ -14,6 +14,8 @@
  */
 #include "dataformat.h"
 
+static int tdFLenFromSchema(STSchema *pSchema);
+
 /**
  * Create a new STColumn object
  * ASSUMPTIONS: VALID PARAMETERS
@@ -89,13 +91,40 @@ void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
 STSchema *tdNewSchema(int32_t nCols) {
   int32_t  size = sizeof(STSchema) + sizeof(STColumn) * nCols;
 
-  STSchema *pSchema = (STSchema *)calloc(1, size);
+  STSchema *pSchema = (STSchema *)malloc(size);
   if (pSchema == NULL) return NULL;
-  pSchema->numOfCols = nCols;
+  pSchema->numOfCols = 0;
+  pSchema->totalCols = nCols;
 
   return pSchema;
 }
 
+/**
+ * Append a column to the schema
+ */
+int tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int16_t bytes) {
+  if (pSchema->numOfCols >= pSchema->totalCols) return -1;
+  if (!isValidDataType(type, 0)) return -1;
+
+  STColumn *pCol = schemaColAt(pSchema, schemaNCols(pSchema));
+  colSetType(pCol, type);
+  colSetColId(pCol, colId);
+  colSetOffset(pCol, -1);
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      colSetBytes(pCol, bytes);
+      break;
+    default:
+      colSetBytes(pCol, TYPE_BYTES[type]);
+      break;
+  }
+
+  pSchema->numOfCols++;
+
+  return 0;
+}
+
 /**
  * Duplicate the schema and return a new object
  */
@@ -130,6 +159,14 @@ void tdUpdateSchema(STSchema *pSchema) {
   }
 }
 
+/**
+ * Initialize a data row
+ */
+void tdInitDataRow(SDataRow row, STSchema *pSchema) {
+  dataRowSetFLen(row, TD_DATA_ROW_HEAD_SIZE);
+  dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + tdFLenFromSchema(pSchema));
+}
+
 /**
  * Create a data row with maximum row length bytes.
  *
@@ -140,21 +177,37 @@ void tdUpdateSchema(STSchema *pSchema) {
  * @return SDataRow object for success
  *         NULL for failure
  */
-SDataRow tdNewDataRow(int32_t bytes) {
+SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema) {
   int32_t size = sizeof(int32_t) + bytes;
 
   SDataRow row = malloc(size);
   if (row == NULL) return NULL;
 
-  dataRowSetLen(row, sizeof(int32_t));
+  tdInitDataRow(row, pSchema);
 
   return row;
 }
 
-// SDataRow tdNewDdataFromSchema(SSchema *pSchema) {
+/**
-//   int32_t bytes = tdMaxRowDataBytes(pSchema);
+ * Get maximum bytes a data row from a schema
-//   return tdNewDataRow(bytes);
+ * ASSUMPTIONS: VALID PARAMETER
-// }
+ */
+int tdMaxRowBytesFromSchema(STSchema *pSchema) {
+  // TODO
+  int bytes = TD_DATA_ROW_HEAD_SIZE;
+  for (int i = 0; i < schemaNCols(pSchema); i++) {
+    STColumn *pCol = schemaColAt(pSchema, i);
+    bytes += TYPE_BYTES[pCol->type];
+
+    if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
+      bytes += pCol->bytes;
+    }
+  }
+
+  return bytes;
+}
+
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { return tdNewDataRow(tdMaxRowBytesFromSchema(pSchema), pSchema); }
 
 /**
  * Free the SDataRow object
@@ -164,62 +217,37 @@ void tdFreeDataRow(SDataRow row) {
 }
 
 /**
- * Append a column value to a SDataRow object.
+ * Append a column value to the data row
- * NOTE: THE APPLICATION SHOULD MAKE SURE VALID PARAMETERS. THE FUNCTION ASSUMES
- * THE ROW OBJECT HAS ENOUGH SPACE TO HOLD THE VALUE.
- *
- * @param row the row to append value to
- * @param value value pointer to append
- * @param pSchema schema
- * @param colIdx column index
- * 
- * @return 0 for success and -1 for failure
  */
-// int32_t tdAppendColVal(SDataRow row, void *value, SColumn *pCol, int32_t suffixOffset) {
+int tdAppendColVal(SDataRow row, void *value, STColumn *pCol) {
-//   int32_t offset;
+  switch (colType(pCol))
+  {
+  case TSDB_DATA_TYPE_BINARY:
+  case TSDB_DATA_TYPE_NCHAR:
+    *(int32_t *)dataRowAt(row, dataRowFLen(row)) = dataRowLen(row);
+    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+    memcpy((void *)dataRowAt(row, dataRowLen(row)), value, strlen(value));
+    dataRowLen(row) += strlen(value);
+    break;
+  default:
+    memcpy(dataRowAt(row, dataRowFLen(row)), value, TYPE_BYTES[colType(pCol)]);
+    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+    break;
+  }
+}
 
-//   switch (pCol->type) {
+void tdDataRowReset(SDataRow row, STSchema *pSchema) { tdInitDataRow(row, pSchema); }
-//     case TD_DATATYPE_BOOL:
-//     case TD_DATATYPE_TINYINT:
-//     case TD_DATATYPE_SMALLINT:
-//     case TD_DATATYPE_INT:
-//     case TD_DATATYPE_BIGINT:
-//     case TD_DATATYPE_FLOAT:
-//     case TD_DATATYPE_DOUBLE:
-//     case TD_DATATYPE_TIMESTAMP:
-//       memcpy(dataRowIdx(row, pCol->offset + sizeof(int32_t)), value, rowDataLen[pCol->type]);
-//       if (dataRowLen(row) < suffixOffset + sizeof(int32_t))
-//         dataRowSetLen(row, dataRowLen(row) + rowDataLen[pCol->type]);
-//       break;
-//     case TD_DATATYPE_VARCHAR:
-//       offset = dataRowLen(row) > suffixOffset ? dataRowLen(row) : suffixOffset;
-//       memcpy(dataRowIdx(row, pCol->offset+sizeof(int32_t)), (void *)(&offset), sizeof(offset));
-//     case TD_DATATYPE_NCHAR:
-//     case TD_DATATYPE_BINARY:
-//       break;
-//     default:
-//       return -1;
-//   }
 
-//   return 0;
-// }
-
-/**
- * Copy a data row to a destination
- * ASSUMPTIONS: dst has enough room for a copy of row
- */
-void tdDataRowCpy(void *dst, SDataRow row) { memcpy(dst, row, dataRowLen(row)); }
-void tdDataRowReset(SDataRow row) { dataRowSetLen(row, sizeof(int32_t)); }
 SDataRow tdDataRowDup(SDataRow row) {
-  SDataRow trow = tdNewDataRow(dataRowLen(row));
+  SDataRow trow = malloc(dataRowLen(row));
   if (trow == NULL) return NULL;
 
   dataRowCpy(trow, row);
-  return row;
+  return trow;
 }
 
 void tdDataRowsAppendRow(SDataRows rows, SDataRow row) {
-  tdDataRowCpy((void *)((char *)rows + dataRowsLen(rows)), row);
+  dataRowCpy((void *)((char *)rows + dataRowsLen(rows)), row);
   dataRowsSetLen(rows, dataRowsLen(rows) + dataRowLen(row));
 }
 
@@ -250,4 +278,17 @@ SDataRow tdDataRowsNext(SDataRowsIter *pIter) {
   }
 
   return row;
+}
+
+/**
+ * Return the first part length of a data row for a schema
+ */
+static int tdFLenFromSchema(STSchema *pSchema) {
+  int ret = 0;
+  for (int i = 0; i < schemaNCols(pSchema); i++) {
+    STColumn *pCol = schemaColAt(pSchema, i);
+    ret += TYPE_BYTES[pCol->type];
+  }
+
+  return ret;
 }

src/common/src/ttypes.c

@@ -26,9 +26,9 @@ const int32_t TYPE_BYTES[11] = {
     sizeof(int64_t),  // TSDB_DATA_TYPE_BIGINT
     sizeof(float),    // TSDB_DATA_TYPE_FLOAT
     sizeof(double),   // TSDB_DATA_TYPE_DOUBLE
-    -1,               // TSDB_DATA_TYPE_BINARY
+    sizeof(int32_t),  // TSDB_DATA_TYPE_BINARY
     sizeof(TSKEY),    // TSDB_DATA_TYPE_TIMESTAMP
-    -1                // TSDB_DATA_TYPE_NCHAR
+    sizeof(int32_t)   // TSDB_DATA_TYPE_NCHAR
 };
 
 tDataTypeDescriptor tDataTypeDesc[11] = {

src/dnode/src/dnodeMClient.c

@@ -57,11 +57,10 @@ void dnodeCleanupMClient() {
 }
 
 static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) {
-
   if (dnodeProcessMgmtRspFp[pMsg->msgType]) {
     (*dnodeProcessMgmtRspFp[pMsg->msgType])(pMsg);
   } else {
-    dError("%s is not processed", taosMsg[pMsg->msgType]);
+    dError("%s is not processed in mclient", taosMsg[pMsg->msgType]);
   }
 
   rpcFreeCont(pMsg->pCont);

src/dnode/src/dnodeMain.c

@@ -226,20 +226,18 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 
 static int32_t dnodeInitStorage() {
   struct stat dirstat;
-  strcpy(tsDirectory, dataDir);
   if (stat(dataDir, &dirstat) < 0) {
     mkdir(dataDir, 0755);
   }
 
-  char fileName[128];
+  sprintf(tsMnodeDir, "%s/mnode", dataDir);
-  sprintf(fileName, "%s/tsdb", tsDirectory);
+  sprintf(tsVnodeDir, "%s/vnode", dataDir);
-  mkdir(fileName, 0755);
+  sprintf(tsDnodeDir, "%s/dnode", dataDir);
-  sprintf(fileName, "%s/data", tsDirectory);
+  mkdir(tsMnodeDir, 0755);
-  mkdir(fileName, 0755);
+  mkdir(tsVnodeDir, 0755);
-  sprintf(tsMgmtDirectory, "%s/mgmt", tsDirectory);
+  mkdir(tsDnodeDir, 0755);
-  sprintf(tsDirectory, "%s/tsdb", dataDir);
 
-  dnodeCheckDataDirOpenned(dataDir);
+  dnodeCheckDataDirOpenned(tsDnodeDir);
 
   dPrint("storage directory is initialized");
   return 0;

src/dnode/src/dnodeMgmt.c

@@ -29,6 +29,7 @@
 
 typedef struct {
   int32_t vgId;     // global vnode group ID
+  int32_t vnode;
   int32_t status;   // status: master, slave, notready, deleting
   int32_t refCount; // reference count
   int64_t version;
@@ -43,7 +44,7 @@ typedef struct {
 
 static int32_t  dnodeOpenVnodes();
 static void     dnodeCleanupVnodes();
-static int32_t  dnodeOpenVnode(int32_t vgId);
+static int32_t  dnodeOpenVnode(int32_t vnode, char *rootDir);
 static void     dnodeCleanupVnode(SVnodeObj *pVnode);
 static int32_t  dnodeCreateVnode(SMDCreateVnodeMsg *cfg);
 static void     dnodeDropVnode(SVnodeObj *pVnode);
@@ -79,7 +80,25 @@ int32_t dnodeInitMgmt() {
   }
   taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
 
-  return dnodeOpenVnodes();
+  SMDCreateVnodeMsg cfg;
+  cfg.cfg.vgId = 1;
+  cfg.cfg.precision = 0;
+  cfg.vnode = 1;
+  cfg.cfg.maxSessions = 1000;
+  cfg.cfg.daysPerFile = 10;
+
+  dnodeCreateVnode(&cfg);
+  SVnodeObj *pVnode = dnodeGetVnode(cfg.cfg.vgId);
+  dnodeDropVnode(pVnode);
+
+  // dnodeCreateVnode(&cfg);
+  // SVnodeObj *pVnode = dnodeGetVnode(cfg.cfg.vgId);
+  // dnodeCleanupVnodes();
+
+  dnodeOpenVnodes();
+  dnodeCleanupVnodes();
+
+  //return dnodeOpenVnodes();
 }
 
 void dnodeCleanupMgmt() {
@@ -98,14 +117,13 @@ void dnodeMgmt(SRpcMsg *pMsg) {
   if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
     (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
   } else {
-    terrno = TSDB_CODE_MSG_NOT_PROCESSED;  
+    SRpcMsg rsp;
+    rsp.handle = pMsg->handle;
+    rsp.code   = TSDB_CODE_MSG_NOT_PROCESSED;
+    rsp.pCont  = NULL;
+    rpcSendResponse(&rsp);
   }
 
-  SRpcMsg rsp;
-  rsp.handle = pMsg->handle;
-  rsp.code   = terrno;
-  rsp.pCont  = NULL;
-  rpcSendResponse(&rsp);
   rpcFreeCont(pMsg->pCont);  // free the received message
 }
 
@@ -150,28 +168,55 @@ void dnodeReleaseVnode(void *pVnode) {
 }
 
 static int32_t dnodeOpenVnodes() {
-  dPrint("open all vnodes");
+  DIR *dir = opendir(tsVnodeDir);
+  if (dir == NULL) {
+    return TSDB_CODE_NO_WRITE_ACCESS;
+  }
+
+  int32_t numOfVnodes = 0;
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+    if (de->d_type & DT_DIR) {
+      if (strncmp("vnode", de->d_name, 5) != 0) continue;
+      int32_t vnode = atoi(de->d_name + 5);
+      if (vnode == 0) continue;
+
+      char tsdbDir[TSDB_FILENAME_LEN];
+      sprintf(tsdbDir, "%s/%s", tsVnodeDir, de->d_name);
+      int32_t code = dnodeOpenVnode(vnode, tsdbDir);
+      if (code == 0) {
+        numOfVnodes++;
+      }
+    }
+  }
+  closedir(dir);
+
+  dPrint("all vnodes is opened, num:%d", numOfVnodes);
   return TSDB_CODE_SUCCESS;
 }
 
 static void dnodeCleanupVnodes() {
-  dPrint("clean all vnodes");
+  int32_t num = taosGetIntHashSize(tsDnodeVnodesHash);
+  taosCleanUpIntHashWithFp(tsDnodeVnodesHash, dnodeCleanupVnode);
+  dPrint("all vnodes is opened, num:%d", num);
 }
 
-static int32_t dnodeOpenVnode(int32_t vgId) {
+static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) {
-  char rootDir[TSDB_FILENAME_LEN] = {0};
-  sprintf(rootDir, "%s/vnode%d", tsDirectory, vgId);
-
   void *pTsdb = tsdbOpenRepo(rootDir);
-  if (pTsdb != NULL) {
+  if (pTsdb == NULL) {
+    dError("failed to open vnode:%d in dir:%s, reason:%s", vnode, rootDir, tstrerror(terrno));
     return terrno;
   }
 
+  //STsdbRepoInfo *tsdbInfo = tsdbGetStatus(pTsdb);
+
   SVnodeObj vnodeObj;
-  vnodeObj.vgId     = vgId;
+  vnodeObj.vgId     = vnode; //tsdbInfo->tsdbCfg.vgId;
+  vnodeObj.vnode    = vnode; //tsdbInfo->tsdbCfg.tsdbId;
   vnodeObj.status   = TSDB_VN_STATUS_NOT_READY;
   vnodeObj.refCount = 1;
-  vnodeObj.version  = 0;
+  vnodeObj.version  = version;
   vnodeObj.wworker  = dnodeAllocateWriteWorker();
   vnodeObj.rworker  = dnodeAllocateReadWorker();
   vnodeObj.wal      = NULL;
@@ -182,6 +227,7 @@ static int32_t dnodeOpenVnode(int32_t vgId) {
 
   taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj));
 
+  dTrace("open vnode:%d in %s", vnodeObj.vnode, rootDir);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -210,11 +256,12 @@ static void dnodeCleanupVnode(SVnodeObj *pVnode) {
     pVnode->tsdb = NULL;
   }
 
-  taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
+  dTrace("cleanup vnode:%d", pVnode->vnode);
 }
 
 static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) {
   STsdbCfg tsdbCfg;
+  tsdbCfg.vgId                = pVnodeCfg->cfg.vgId;
   tsdbCfg.precision           = pVnodeCfg->cfg.precision;
   tsdbCfg.tsdbId              = pVnodeCfg->vnode;
   tsdbCfg.maxTables           = pVnodeCfg->cfg.maxSessions;
@@ -225,15 +272,16 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) {
   tsdbCfg.maxCacheSize        = -1;
 
   char rootDir[TSDB_FILENAME_LEN] = {0};
-  sprintf(rootDir, "%s/vnode%d", tsDirectory, pVnodeCfg->cfg.vgId);
+  sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
 
   void *pTsdb = tsdbCreateRepo(rootDir, &tsdbCfg, NULL);
-  if (pTsdb != NULL) {
+  if (pTsdb == NULL) {
     return terrno;
   }
 
   SVnodeObj vnodeObj;
   vnodeObj.vgId     = pVnodeCfg->cfg.vgId;
+  vnodeObj.vnode    = pVnodeCfg->vnode;
   vnodeObj.status   = TSDB_VN_STATUS_NOT_READY;
   vnodeObj.refCount = 1;
   vnodeObj.version  = 0;
@@ -264,6 +312,7 @@ static void dnodeDropVnode(SVnodeObj *pVnode) {
   }
 
   dnodeCleanupVnode(pVnode);
+  taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
 }
 
 static void dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
@@ -281,9 +330,8 @@ static void dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
   } else {
     rpcRsp.code = dnodeCreateVnode(pCreate);
   }
-
+  rpcRsp.code = TSDB_CODE_SUCCESS;
   rpcSendResponse(&rpcRsp);
-  rpcFreeCont(rpcMsg->pCont);
 }
 
 static void dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
@@ -301,7 +349,6 @@ static void dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
   }
 
   rpcSendResponse(&rpcRsp);
-  rpcFreeCont(rpcMsg->pCont);
 }
 
 static void dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
@@ -321,7 +368,6 @@ static void dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
   }
 
   rpcSendResponse(&rpcRsp);
-  rpcFreeCont(rpcMsg->pCont);
 }
 
 static void dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
@@ -342,7 +388,6 @@ static void dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
 //  dnodeSendRspToMnode(pConn, msgType + 1, code, NULL, 0);
 }
 
-
 static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
   if (tsStatusTimer == NULL) {

src/dnode/src/dnodeMnode.c

@@ -81,7 +81,7 @@ static void dnodeProcessMsgFromMnode(SRpcMsg *pMsg) {
   if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
     (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
   } else {
-    dError("%s is not processed", taosMsg[pMsg->msgType]);
+    dError("%s is not processed in mserver", taosMsg[pMsg->msgType]);
     rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);

src/dnode/src/dnodeRead.c

@@ -60,11 +60,13 @@ int32_t dnodeInitRead() {
   maxThreads = tsNumOfCores*tsNumOfThreadsPerCore;
   if (maxThreads <= minThreads*2) maxThreads = 2*minThreads;
 
+  dPrint("dnode read is opened");
   return 0;
 }
 
 void dnodeCleanupRead() {
   taosCloseQset(readQset);
+  dPrint("dnode read is closed");
 }
 
 void dnodeRead(SRpcMsg *pMsg) {

src/dnode/src/dnodeWrite.c

@@ -76,35 +76,43 @@ int32_t dnodeInitWrite() {
     wWorkerPool.writeWorker[i].workerId = i;
   }
 
+  dPrint("dnode write is opened");
   return 0;
 }
 
 void dnodeCleanupWrite() {
   free(wWorkerPool.writeWorker);
+  dPrint("dnode write is closed");
 }
 
 void dnodeWrite(SRpcMsg *pMsg) {
+  int32_t     queuedMsgNum = 0;
   int32_t     leftLen      = pMsg->contLen;
   char        *pCont       = (char *) pMsg->pCont;
-  int32_t     contLen      = 0;
-  int32_t     numOfVnodes  = 0;
-  int32_t     vgId         = 0;
   SRpcContext *pRpcContext = NULL;
 
-  // parse head, get number of vnodes;
+  int32_t numOfVnodes = 0;
+  if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
+    // TODO parse head, get number of vnodes;
+    numOfVnodes = 1;
+  } else {
+    numOfVnodes = 1;
+  }
 
-  if ( numOfVnodes > 1) {
+  if (numOfVnodes > 1) {
     pRpcContext = calloc(sizeof(SRpcContext), 1);
     pRpcContext->numOfVnodes = numOfVnodes;
   }
 
   while (leftLen > 0) {
-    // todo: parse head, get vgId, contLen
+    SWriteMsgHead *pHead = (SWriteMsgHead *) pCont;
+    int32_t vgId    = htonl(pHead->vgId);
+    int32_t contLen = htonl(pHead->contLen);
 
-    // get pVnode from vgId
     void *pVnode = dnodeGetVnode(vgId);
     if (pVnode == NULL) {
-
+      leftLen -= contLen;
+      pCont -= contLen;
       continue;
     }
    
@@ -118,20 +126,37 @@ void dnodeWrite(SRpcMsg *pMsg) {
  
     taos_queue queue = dnodeGetVnodeWworker(pVnode);
     taosWriteQitem(queue, &writeMsg);
- 
+
     // next vnode 
     leftLen -= contLen;
-    pCont -= contLen; 
+    pCont -= contLen;
+    queuedMsgNum++;
+  }
+
+  if (queuedMsgNum == 0) {
+    SRpcMsg rpcRsp = {
+      .handle  = pMsg->handle,
+      .pCont   = NULL,
+      .contLen = 0,
+      .code    = TSDB_CODE_INVALID_VGROUP_ID,
+      .msgType = 0
+    };
+    rpcSendResponse(&rpcRsp);
   }
 }
 
 void *dnodeAllocateWriteWorker() {
   SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
+  taos_queue *queue = taosOpenQueue(sizeof(SWriteMsg));
+  if (queue != NULL) return queue;
 
   if (pWorker->qset == NULL) {
     pWorker->qset = taosOpenQset();
     if (pWorker->qset == NULL) return NULL;
 
+    taosAddIntoQset(pWorker->qset, queue);
+    wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
+
     pthread_attr_t thAttr;
     pthread_attr_init(&thAttr);
     pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
@@ -140,14 +165,11 @@ void *dnodeAllocateWriteWorker() {
       dError("failed to create thread to process read queue, reason:%s", strerror(errno));
       taosCloseQset(pWorker->qset);
     }
-  }
+  } else {
-
-  taos_queue *queue = taosOpenQueue(sizeof(SWriteMsg));
-  if (queue) {
     taosAddIntoQset(pWorker->qset, queue);
     wWorkerPool.nextId = (wWorkerPool.nextId + 1) % wWorkerPool.max;
   }
- 
+
   return queue;
 }
 

src/inc/mnode.h

@@ -39,12 +39,7 @@ extern "C" {
 #include "ttimer.h"
 #include "tutil.h"
 
-// internal globals
+  typedef struct {
-extern char  version[];
-extern void *tsMgmtTmr;
-extern char  tsMgmtDirectory[];
-
-typedef struct {
   uint32_t   privateIp;
   int32_t    sid;
   uint32_t   moduleStatus;
@@ -87,11 +82,6 @@ typedef struct {
   int32_t  vnode;
 } SVnodeGid;
 
-typedef struct {
-  int32_t sid;
-  int32_t vgId;  // vnode group ID
-} STableGid;
-
 typedef struct {
   char     tableId[TSDB_TABLE_ID_LEN + 1];
   int8_t   type;
@@ -248,16 +238,32 @@ typedef struct {
   int16_t  offset[TSDB_MAX_COLUMNS];
   int16_t  bytes[TSDB_MAX_COLUMNS];
   void *   signature;
-  uint16_t payloadLen; /* length of payload*/
+  uint16_t payloadLen;
-  char     payload[];  /* payload for wildcard match in show tables */
+  char     payload[];
 } SShowObj;
 
-//mgmtSystem
+typedef struct {
+  uint8_t  msgType;
+  int8_t   expected;
+  int8_t   received;
+  int8_t   successed;
+  int32_t  contLen;
+  int32_t  code;
+  void     *ahandle;
+  void     *thandle;
+  void     *pCont;
+  SDbObj   *pDb;
+  SUserObj *pUser;
+} SQueuedMsg;
+
 int32_t mgmtInitSystem();
 int32_t mgmtStartSystem();
 void    mgmtCleanUpSystem();
 void    mgmtStopSystem();
 
+extern char  version[];
+extern void *tsMgmtTmr;
+extern char  tsMnodeDir[];
 
 #ifdef __cplusplus
 }

src/inc/taosmsg.h

@@ -238,11 +238,20 @@ typedef struct SSchema {
 } SSchema;
 
 typedef struct {
+  int32_t  vgId;
   int32_t  vnode;  //the index of vnode
   uint32_t ip;
 } SVnodeDesc;
 
 typedef struct {
+  int32_t contLen;
+  int32_t vgId;
+} SWriteMsgHead;
+
+typedef struct {
+  int32_t    contLen;
+  int32_t    vgId;
+
   int8_t     tableType;
   int16_t    numOfColumns;
   int16_t    numOfTags;
@@ -250,7 +259,6 @@ typedef struct {
   int32_t    sversion;
   int32_t    tagDataLen;
   int32_t    sqlDataLen;
-  int32_t    contLen;
   int32_t    numOfVPeers;
   uint64_t   uid;
   uint64_t   superTableUid;
@@ -336,6 +344,7 @@ typedef struct {
 } SMgmtHead;
 
 typedef struct {
+  int32_t    vgId;
   int32_t    sid;
   int32_t    numOfVPeers;
   uint64_t   uid;

src/mnode/inc/mgmtProfile.h

@@ -28,22 +28,6 @@ bool mgmtCheckQhandle(uint64_t qhandle);
 void mgmtSaveQhandle(void *qhandle);
 void mgmtFreeQhandle(void *qhandle);
 
-enum {
-  TSDB_PROCESS_CREATE_VGROUP,
-  TSDB_PROCESS_CREATE_VGROUP_GET_META,
-  TSDB_PROCESS_CREATE_TABLE,
-  TSDB_PROCESS_CREATE_TABLE_GET_META,
-};
-
-typedef struct {
-  void    *thandle;  // come from uplayer
-  void    *ahandle;  // object to process
-  void    *cont;     // additional information of object to process
-  int32_t type;      // the type of sync process
-  int32_t received;  // num of received, such as numOfVnodes
-  int32_t contLen;   // the length of additional information
-} SProcessInfo;
-
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtShell.h

@@ -23,13 +23,16 @@ extern "C" {
 
 int32_t mgmtInitShell();
 void mgmtCleanUpShell();
-void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg));
 
 typedef int32_t (*SShowMetaFp)(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp);
 void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
 
+void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
+void mgmtSendSimpleResp(void *thandle, int32_t code);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtTable.h

@@ -33,7 +33,6 @@ STableInfo* mgmtGetTableByPos(uint32_t dnodeIp, int32_t vnode, int32_t sid);
 int32_t     mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMeta *pMeta, bool usePublicIp);
 
 int32_t  mgmtRetrieveMetricMeta(void *pConn, char **pStart, SSuperTableMetaMsg *pInfo);
-int32_t  mgmtCreateTable(SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta);
 int32_t  mgmtDropTable(SDbObj *pDb, char *tableId, int32_t ignore);
 int32_t  mgmtAlterTable(SDbObj *pDb, SCMAlterTableMsg *pAlter);
 
@@ -44,10 +43,6 @@ void    mgmtSetTableDirty(STableInfo *pTable, bool isDirty);
 SMDDropTableMsg *mgmtBuildRemoveTableMsg(STableInfo *pTable);
 SMDDropSTableMsg *mgmtBuildRemoveSuperTableMsg(STableInfo *pTable);
 
-void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle);
-void mgmtProcessCreateTable(SVgObj *pVgroup, SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta);
-void mgmtProcessCreateVgroup(SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta);
-
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtVgroup.h

@@ -29,7 +29,7 @@ void    mgmtCleanUpVgroups();
 SVgObj *mgmtGetVgroup(int32_t vgId);
 SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode);
 
-SVgObj *mgmtCreateVgroup(SDbObj *pDb);
+void    mgmtCreateVgroup(SQueuedMsg *pMsg);
 int32_t mgmtDropVgroup(SDbObj *pDb, SVgObj *pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);
 

src/mnode/src/mgmtBalance.c

@@ -56,10 +56,10 @@ int32_t mgmtAllocVnodes(SVgObj *pVgroup) {
   }
 
   if (selectedVnode == -1) {
-    mError("vgroup:%d alloc vnode failed, free vnodes:%d", pVgroup->vgId, pDnode->numOfFreeVnodes);
+    mError("alloc vnode failed, free vnodes:%d", pDnode->numOfFreeVnodes);
     return -1;
   } else {
-    mTrace("vgroup:%d allocate vnode:%d, last allocated vnode:%d", pVgroup->vgId, selectedVnode, lastAllocVode);
+    mTrace("allocate vnode:%d, last allocated vnode:%d", selectedVnode, lastAllocVode);
     pVgroup->vnodeGid[0].vnode = selectedVnode;
     pDnode->lastAllocVnode     = selectedVnode + 1;
     if (pDnode->lastAllocVnode >= pDnode->numOfVnodes) pDnode->lastAllocVnode = 0;

src/mnode/src/mgmtChildTable.c

@@ -198,7 +198,7 @@ int32_t mgmtInitChildTables() {
   tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
 
   tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize,
-                                 "ctables", SDB_KEYTYPE_STRING, tsMgmtDirectory, mgmtChildTableAction);
+                                 "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction);
   if (tsChildTableSdb == NULL) {
     mError("failed to init child table data");
     return -1;
@@ -359,7 +359,7 @@ int32_t mgmtCreateChildTable(SCMCreateTableMsg *pCreate, int32_t contLen, SVgObj
 
   *pTableOut = (STableInfo *) pTable;
 
-  mTrace("table:%s, create table in vgroup, vgroup:%d sid:%d vnode:%d uid:%" PRIu64 ,
+  mTrace("table:%s, create ctable in vgroup, vgroup:%d sid:%d vnode:%d uid:%" PRIu64 ,
          pTable->tableId, pVgroup->vgId, sid, pVgroup->vnodeGid[0].vnode, pTable->uid);
 
   return TSDB_CODE_SUCCESS;

src/mnode/src/mgmtDClient.c

@@ -78,45 +78,12 @@ static void mgmtProcessRspFromDnode(SRpcMsg *rpcMsg) {
   if (mgmtProcessDnodeRspFp[rpcMsg->msgType]) {
     (*mgmtProcessDnodeRspFp[rpcMsg->msgType])(rpcMsg);
   } else {
-    dError("%s is not processed", taosMsg[rpcMsg->msgType]);
+    mError("%s is not processed in dclient", taosMsg[rpcMsg->msgType]);
   }
 
   rpcFreeCont(rpcMsg->pCont);
 }
 
-
-//static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) {
-//  mTrace("create table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
-//  if (rpcMsg->handle == NULL) return;
-//
-//  SProcessInfo *info = rpcMsg->handle;
-//  assert(info->type == TSDB_PROCESS_CREATE_TABLE || info->type == TSDB_PROCESS_CREATE_TABLE_GET_META);
-//
-//  STableInfo *pTable = info->ahandle;
-//  if (rpcMsg->code != TSDB_CODE_SUCCESS) {
-//    mError("table:%s, failed to create in dnode, code:%d, set it dirty", pTable->tableId, rpcMsg->code);
-//    mgmtSetTableDirty(pTable, true);
-//  } else {
-//    mTrace("table:%s, created in dnode", pTable->tableId);
-//    mgmtSetTableDirty(pTable, false);
-//  }
-//
-//  if (rpcMsg->code != TSDB_CODE_SUCCESS) {
-//    SRpcMsg rpcRsp = {.handle = info->thandle, .pCont = NULL, .contLen = 0, .code = rpcMsg->code, .msgType = 0};
-//    rpcSendResponse(&rpcMsg);
-//  } else {
-//    if (info->type == TSDB_PROCESS_CREATE_TABLE_GET_META) {
-//      mTrace("table:%s, start to process get meta", pTable->tableId);
-//      mgmtProcessGetTableMeta(pTable, rpcMsg->handle);
-//    } else {
-//      SRpcMsg rpcRsp = {.handle = info->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-//      rpcSendResponse(&rpcMsg);
-//    }
-//  }
-//
-//  free(info);
-//}
-//
 //static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) {
 //  mTrace("drop table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
 //}
@@ -125,27 +92,6 @@ static void mgmtProcessRspFromDnode(SRpcMsg *rpcMsg) {
 //  mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
 //}
 //
-//static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
-//  mTrace("create vnode rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);
-//  if (rpcMsg->handle == NULL) return;
-//
-//  SProcessInfo *info = rpcMsg->handle;
-//  assert(info->type == TSDB_PROCESS_CREATE_VGROUP || info->type == TSDB_PROCESS_CREATE_VGROUP_GET_META);
-//
-//  info->received++;
-//  SVgObj *pVgroup = info->ahandle;
-//
-//  bool isGetMeta = false;
-//  if (info->type == TSDB_PROCESS_CREATE_VGROUP_GET_META) {
-//    isGetMeta = true;
-//  }
-//
-//  mTrace("vgroup:%d, received:%d numOfVnodes:%d", pVgroup->vgId, info->received, pVgroup->numOfVnodes);
-//  if (info->received == pVgroup->numOfVnodes) {
-//    mgmtProcessCreateTable(pVgroup, info->cont, info->contLen, info->thandle, isGetMeta);
-//    free(info);
-//  }
-//}
 //
 //static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
 //  mTrace("drop vnode rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code);

src/mnode/src/mgmtDServer.c

@@ -76,7 +76,7 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
   if (mgmtProcessDnodeMsgFp[rpcMsg->msgType]) {
     (*mgmtProcessDnodeMsgFp[rpcMsg->msgType])(rpcMsg);
   } else {
-    mError("%s is not processed", taosMsg[rpcMsg->msgType]);
+    mError("%s is not processed in dserver", taosMsg[rpcMsg->msgType]);
   }
 
   rpcFreeCont(rpcMsg->pCont);
@@ -210,22 +210,6 @@ static int mgmtDServerRetrieveAuth(char *user, char *spi, char *encrypt, char *s
 //  }
 //}
 //
-//void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-//  mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
-//  for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
-//    SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].ip);
-//    mgmtSendCreateVnodeMsg(pVgroup, pVgroup->vnodeGid[i].vnode, &ipSet, ahandle);
-//  }
-//}
-//
-//void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode, SRpcIpSet *ipSet, void *ahandle) {
-//  mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, vnode, ahandle);
-//  SMDCreateVnodeMsg *pVpeer = mgmtBuildCreateVnodeMsg(pVgroup, vnode);
-//  if (pVpeer != NULL) {
-//    mgmtSendMsgToDnode(ipSet, TSDB_MSG_TYPE_MD_CREATE_VNODE, pVpeer, sizeof(SMDCreateVnodeMsg), ahandle);
-//  }
-//}
-//
 //void mgmtProcessMsgFromDnode(char msgType, void *pCont, int32_t contLen, void *pConn, int32_t code) {
 //  if (msgType < 0 || msgType >= TSDB_MSG_TYPE_MAX) {
 //    mError("invalid msg type:%d", msgType);

src/mnode/src/mgmtDb.c

@@ -41,9 +41,9 @@ static int32_t mgmtDropDb(SDbObj *pDb);
 
 static int32_t mgmtGetDbMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static void    mgmtProcessCreateDbMsg(SRpcMsg *rpcMsg);
+static void    mgmtProcessCreateDbMsg(SQueuedMsg *pMsg);
-static void    mgmtProcessAlterDbMsg(SRpcMsg *rpcMsg);
+static void    mgmtProcessAlterDbMsg(SQueuedMsg *pMsg);
-static void    mgmtProcessDropDbMsg(SRpcMsg *rpcMsg);
+static void    mgmtProcessDropDbMsg(SQueuedMsg *pMsg);
 
 static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
 static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
@@ -81,7 +81,7 @@ int32_t mgmtInitDbs() {
   SDbObj tObj;
   tsDbUpdateSize = tObj.updateEnd - (char *)&tObj;
 
-  tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMgmtDirectory, mgmtDbAction);
+  tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction);
   if (tsDbSdb == NULL) {
     mError("failed to init db data");
     return -1;
@@ -383,6 +383,7 @@ static void mgmtDropDbFromSdb(SDbObj *pDb) {
 }
 
 static int32_t mgmtDropDb(SDbObj *pDb) {
+
   if (pDb->dropStatus == TSDB_DB_STATUS_DROPPING) {
     bool finished = mgmtCheckDropDbFinished(pDb);
     if (!finished) {
@@ -405,6 +406,7 @@ static int32_t mgmtDropDb(SDbObj *pDb) {
   }
 }
 
+UNUSED_FUNC
 static int32_t mgmtDropDbByName(SAcctObj *pAcct, char *name, short ignoreNotExists) {
   SDbObj *pDb = (SDbObj *)sdbGetRow(tsDbSdb, name);
   if (pDb == NULL) {
@@ -904,19 +906,10 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) {
   atomic_add_fetch_32(&pDb->numOfTables, -1);
 }
 
-static void mgmtProcessCreateDbMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
-
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
-  if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
-
-  SCMCreateDbMsg *pCreate = (SCMCreateDbMsg *) rpcMsg->pCont;
 
+  SCMCreateDbMsg *pCreate = pMsg->pCont;
   pCreate->maxSessions     = htonl(pCreate->maxSessions);
   pCreate->cacheBlockSize  = htonl(pCreate->cacheBlockSize);
   pCreate->daysPerFile     = htonl(pCreate->daysPerFile);
@@ -928,69 +921,58 @@ static void mgmtProcessCreateDbMsg(SRpcMsg *rpcMsg) {
   pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock);
   // pCreate->cacheNumOfBlocks = htonl(pCreate->cacheNumOfBlocks);
 
+  int32_t code;
   if (mgmtCheckExpired()) {
-    rpcRsp.code = TSDB_CODE_GRANT_EXPIRED;
+    code = TSDB_CODE_GRANT_EXPIRED;
-  } else if (!pUser->writeAuth) {
+  } else if (!pMsg->pUser->writeAuth) {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    code = TSDB_CODE_NO_RIGHTS;
   } else {
-    rpcRsp.code = mgmtCreateDb(pUser->pAcct, pCreate);
+    code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate);
-    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("DB:%s is created by %s", pCreate->db, pUser->user);
+      mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user);
     }
   }
 
-  rpcSendResponse(&rpcRsp);
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }
 
-static void mgmtProcessAlterDbMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SCMAlterDbMsg *pAlter = pMsg->pCont;
-  if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
-
-  SCMAlterDbMsg *pAlter = (SCMAlterDbMsg *) rpcMsg->pCont;
   pAlter->daysPerFile = htonl(pAlter->daysPerFile);
   pAlter->daysToKeep  = htonl(pAlter->daysToKeep);
   pAlter->maxSessions = htonl(pAlter->maxSessions) + 1;
 
-  if (!pUser->writeAuth) {
+  int32_t code;
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+  if (!pMsg->pUser->writeAuth) {
+    code = TSDB_CODE_NO_RIGHTS;
   } else {
-    rpcRsp.code = mgmtAlterDb(pUser->pAcct, pAlter);
+    code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter);
-    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+    if (code == TSDB_CODE_SUCCESS) {
-      mLPrint("DB:%s is altered by %s", pAlter->db, pUser->user);
+      mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user);
     }
   }
 
-  rpcSendResponse(&rpcRsp);
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }
 
+static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-static void mgmtProcessDropDbMsg(SRpcMsg *rpcMsg) {
+  int32_t code;
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (pMsg->pUser->superAuth) {
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
+    code = TSDB_CODE_OPS_NOT_SUPPORT;
-
+    //SCMDropDbMsg *pDrop = rpcMsg->pCont;
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+    //rpcRsp.code = mgmtDropDbByName(pUser->pAcct, pDrop->db, pDrop->ignoreNotExists);
-  if (pUser == NULL) {
+    //if (rpcRsp.code == TSDB_CODE_SUCCESS) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    //  mLPrint("DB:%s is dropped by %s", pDrop->db, pUser->user);
-    rpcSendResponse(&rpcRsp);
+    //}
-    return ;
-  }
-
-  if (pUser->superAuth) {
-    SCMDropDbMsg *pDrop = rpcMsg->pCont;
-    rpcRsp.code = mgmtDropDbByName(pUser->pAcct, pDrop->db, pDrop->ignoreNotExists);
-    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
-      mLPrint("DB:%s is dropped by %s", pDrop->db, pUser->user);
-    }
   } else {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    code = TSDB_CODE_NO_RIGHTS;
   }
 
-  rpcSendResponse(&rpcRsp);
+  if (code != TSDB_CODE_SUCCESS) {
+    mgmtSendSimpleResp(pMsg->thandle, code);
+  }
 }

src/mnode/src/mgmtDnode.c

@@ -43,7 +43,7 @@ static int32_t mgmtGetConfigMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn
 static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-static void    mgmtProcessCfgDnodeMsg(SRpcMsg *rpcMsg);
+static void    mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg);
 
 void mgmtSetDnodeMaxVnodes(SDnodeObj *pDnode) {
   int32_t maxVnodes = pDnode->numOfCores * tsNumOfVnodesPerCore;
@@ -93,7 +93,7 @@ void mgmtSetDnodeVgid(SVnodeGid vnodeGid[], int32_t numOfVnodes, int32_t vgId) {
       memset(pVload, 0, sizeof(SVnodeLoad));
       pVload->vnode = vnodeGid[i].vnode;
       pVload->vgId = vgId;
-      mTrace("dnode:%s, vnode:%d add to vgroup:%d", taosIpStr(vnodeGid[i].ip), vnodeGid[i].vnode, pVload->vgId);
+      mTrace("dnode:%s, vnode:%d add to vgroup:%d", taosIpStr(pDnode->privateIp), vnodeGid[i].vnode, pVload->vgId);
       mgmtCalcNumOfFreeVnodes(pDnode);
     } else {
       mError("dnode:%s, not in dnode DB!!!", taosIpStr(vnodeGid[i].ip));
@@ -527,21 +527,14 @@ bool mgmtCheckDnodeInOfflineState(SDnodeObj *pDnode) {
   return pDnode->status == TSDB_DN_STATUS_OFFLINE;
 }
 
-void mgmtProcessCfgDnodeMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont;
-  if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
-
-  SCMCfgDnodeMsg *pCmCfgDnode = (SCMCfgDnodeMsg *) rpcMsg->pCont;
   uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip);
 
-  if (strcmp(pUser->pAcct->user, "root") != 0) {
+  if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) {
     rpcRsp.code = TSDB_CODE_NO_RIGHTS;
   } else {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp);
@@ -560,7 +553,7 @@ void mgmtProcessCfgDnodeMsg(SRpcMsg *rpcMsg) {
   }
 
   if (rpcRsp.code == TSDB_CODE_SUCCESS) {
-    mTrace("dnode:%s is configured by %s", pCmCfgDnode->ip, pUser->user);
+    mTrace("dnode:%s is configured by %s", pCmCfgDnode->ip, pMsg->pUser->user);
   }
 
   rpcSendResponse(&rpcRsp);

src/mnode/src/mgmtMain.c

@@ -31,7 +31,6 @@
 #include "mgmtShell.h"
 
 static int32_t mgmtCheckMgmtRunning();
-char tsMgmtDirectory[128] = {0};
 void *tsMgmtTmr = NULL;
 
 int32_t mgmtInitSystem() {
@@ -41,7 +40,7 @@ int32_t mgmtInitSystem() {
   }
 
   struct stat dirstat;
-  bool fileExist  = (stat(tsMgmtDirectory, &dirstat) == 0);
+  bool fileExist  = (stat(tsMnodeDir, &dirstat) == 0);
   bool asMaster = (strcmp(tsMasterIp, tsPrivateIp) == 0);
 
   if (asMaster || fileExist) {
@@ -57,8 +56,8 @@ int32_t mgmtStartSystem() {
   mPrint("starting to initialize TDengine mgmt ...");
 
   struct stat dirstat;
-  if (stat(tsMgmtDirectory, &dirstat) < 0) {
+  if (stat(tsMnodeDir, &dirstat) < 0) {
-    mkdir(tsMgmtDirectory, 0755);
+    mkdir(tsMnodeDir, 0755);
   }
 
   if (mgmtCheckMgmtRunning() != 0) {
@@ -110,7 +109,7 @@ int32_t mgmtStartSystem() {
     return -1;
   }
 
-  if (sdbInitPeers(tsMgmtDirectory) < 0) {
+  if (sdbInitPeers(tsMnodeDir) < 0) {
     mError("failed to init peers");
     return -1;
   }
@@ -132,7 +131,7 @@ void mgmtStopSystem() {
   }
 
   mgmtCleanUpSystem();
-  remove(tsMgmtDirectory);
+  remove(tsMnodeDir);
 }
 
 void mgmtCleanUpSystem() {

src/mnode/src/mgmtNormalTable.c

@@ -224,7 +224,7 @@ int32_t mgmtInitNormalTables() {
   tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
 
   tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,
-                                 "ntables", SDB_KEYTYPE_STRING, tsMgmtDirectory, mgmtNormalTableAction);
+                                 "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction);
   if (tsNormalTableSdb == NULL) {
     mError("failed to init ntables data");
     return -1;
@@ -393,7 +393,7 @@ int32_t mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, int32_t contLen, SVgOb
 
   *pTableOut = (STableInfo *) pTable;
 
-  mTrace("table:%s, create table in vgroup, vgroup:%d sid:%d vnode:%d uid:%" PRIu64 ,
+  mTrace("table:%s, create ntable in vgroup, vgroup:%d sid:%d vnode:%d uid:%" PRIu64 ,
          pTable->tableId, pVgroup->vgId, sid, pVgroup->vnodeGid[0].vnode, pTable->uid);
 
   return TSDB_CODE_SUCCESS;

src/mnode/src/mgmtProfile.c

@@ -558,9 +558,11 @@ bool mgmtCheckQhandle(uint64_t qhandle) {
 }
 
 void mgmtSaveQhandle(void *qhandle) {
+  mTrace("qhandle:%p is allocated", qhandle);
 }
 
 void mgmtFreeQhandle(void *qhandle) {
+  mTrace("qhandle:%p is freed", qhandle);
 }
 
 int mgmtGetConns(SShowObj *pShow, void *pConn) {
@@ -673,72 +675,72 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
   return numOfRows;
 }
 
-void mgmtProcessKillQueryMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
     return;
   }
 
-  SCMKillQueryMsg *pKill = (SCMKillQueryMsg *) rpcMsg->pCont;
+  SCMKillQueryMsg *pKill = pMsg->pCont;
   int32_t code;
 
   if (!pUser->writeAuth) {
     code = TSDB_CODE_NO_RIGHTS;
   } else {
-    code = mgmtKillQuery(pKill->queryId, rpcMsg->handle);
+    code = mgmtKillQuery(pKill->queryId, pMsg->thandle);
   }
 
   rpcRsp.code = code;
   rpcSendResponse(&rpcRsp);
 }
 
-void mgmtProcessKillStreamMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
     return;
   }
 
-  SCMKillStreamMsg *pKill = (SCMKillStreamMsg *) rpcMsg->pCont;
+  SCMKillStreamMsg *pKill = pMsg->pCont;
   int32_t code;
 
   if (!pUser->writeAuth) {
     code = TSDB_CODE_NO_RIGHTS;
   } else {
-    code = mgmtKillStream(pKill->queryId, rpcMsg->handle);
+    code = mgmtKillStream(pKill->queryId, pMsg->thandle);
   }
 
   rpcRsp.code = code;
   rpcSendResponse(&rpcRsp);
 }
 
-void mgmtProcessKillConnectionMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
     return;
   }
 
-  SCMKillConnMsg *pKill = (SCMKillConnMsg *) rpcMsg->pCont;
+  SCMKillConnMsg *pKill = pMsg->pCont;
   int32_t code;
 
   if (!pUser->writeAuth) {
     code = TSDB_CODE_NO_RIGHTS;
   } else {
-    code = mgmtKillConnection(pKill->queryId, rpcMsg->handle);
+    code = mgmtKillConnection(pKill->queryId, pMsg->thandle);
   }
 
   rpcRsp.code = code;

src/mnode/src/mgmtShell.c

@@ -41,24 +41,27 @@
 typedef int32_t (*SShowMetaFp)(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-static void mgmtProcessMsgFromShell(SRpcMsg *pMsg);
-static void mgmtProcessShowMsg(SRpcMsg *rpcMsg);
-static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg);
-static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
 static int  mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 static bool mgmtCheckMsgReadOnly(int8_t type, void *pCont);
-static void mgmtProcessHeartBeatMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessMsgFromShell(SRpcMsg *pMsg);
-static void mgmtProcessConnectMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg);
+static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg);
+static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg);
+static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
+static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 
 static void *tsMgmtShellRpc = NULL;
 static void *tsMgmtTranQhandle = NULL;
-static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *) = {0};
+static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
 static SShowMetaFp     tsMgmtShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMgmtShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
 
 int32_t mgmtInitShell() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mgmtProcessShowMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_RETRIEVE, mgmtProcessRetrieveMsg);
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg);
+  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
 
   tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT");
 
@@ -84,9 +87,6 @@ int32_t mgmtInitShell() {
     return -1;
   }
 
-  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg);
-  mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
-
   mPrint("server connection to shell is opened");
   return 0;
 }
@@ -104,7 +104,7 @@ void mgmtCleanUpShell() {
   }
 }
 
-void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SRpcMsg *rpcMsg)) {
+void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SQueuedMsg *queuedMsg)) {
   tsMgmtProcessShellMsgFp[showType] = fp;
 }
 
@@ -117,107 +117,117 @@ void mgmtAddShellShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) {
 }
 
 void mgmtProcessTranRequest(SSchedMsg *sched) {
-  SRpcMsg *rpcMsg = sched->msg;
+  SQueuedMsg *queuedMsg = sched->msg;
-  (*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(rpcMsg);
+  (*tsMgmtProcessShellMsgFp[queuedMsg->msgType])(queuedMsg);
-  rpcFreeCont(rpcMsg->pCont);
+  rpcFreeCont(queuedMsg->pCont);
-  free(rpcMsg);
+  free(queuedMsg);
 }
 
-void mgmtAddToTranRequest(SRpcMsg *rpcMsg) {
+void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) {
-  SRpcMsg *queuedRpcMsg = malloc(sizeof(SRpcMsg));
-  memcpy(queuedRpcMsg, rpcMsg, sizeof(SRpcMsg));
-
   SSchedMsg schedMsg;
-  schedMsg.msg = queuedRpcMsg;
+  schedMsg.msg = queuedMsg;
   schedMsg.fp  = mgmtProcessTranRequest;
   taosScheduleTask(tsMgmtTranQhandle, &schedMsg);
 }
 
 static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) {
   if (sdbGetRunStatus() != SDB_STATUS_SERVING) {
-    mTrace("shell msg is ignored since SDB is not ready");
+    mgmtProcessMsgWhileNotReady(rpcMsg);
-    SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = TSDB_CODE_NOT_READY, .msgType = 0};
-    rpcSendResponse(&rpcRsp);
     rpcFreeCont(rpcMsg->pCont);
     return;
   }
 
-  mTrace("%s is received", taosMsg[rpcMsg->msgType]);
+  if (tsMgmtProcessShellMsgFp[rpcMsg->msgType] == NULL) {
-  if (tsMgmtProcessShellMsgFp[rpcMsg->msgType]) {
-    if (mgmtCheckMsgReadOnly(rpcMsg->msgType, rpcMsg->pCont)) {
-      (*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(rpcMsg);
-      rpcFreeCont(rpcMsg->pCont);
-    } else {
-      mgmtAddToTranRequest(rpcMsg);
-    }
-  } else {
-    mError("%s is not processed", taosMsg[rpcMsg->msgType]);
     mgmtProcessUnSupportMsg(rpcMsg);
     rpcFreeCont(rpcMsg->pCont);
+    return;
+  }
+
+  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  if (pUser == NULL) {
+    mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_USER);
+    rpcFreeCont(rpcMsg->pCont);
+    return;
+  }
+
+  if (mgmtCheckMsgReadOnly(rpcMsg->msgType, rpcMsg->pCont)) {
+    SQueuedMsg queuedMsg = {0};
+    queuedMsg.thandle = rpcMsg->handle;
+    queuedMsg.msgType = rpcMsg->msgType;
+    queuedMsg.contLen = rpcMsg->contLen;
+    queuedMsg.pCont   = rpcMsg->pCont;
+    queuedMsg.pUser   = pUser;
+    (*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(&queuedMsg);
+    rpcFreeCont(rpcMsg->pCont);
+  } else {
+    SQueuedMsg *queuedMsg = calloc(1, sizeof(SQueuedMsg));
+    queuedMsg->thandle = rpcMsg->handle;
+    queuedMsg->msgType = rpcMsg->msgType;
+    queuedMsg->contLen = rpcMsg->contLen;
+    queuedMsg->pCont   = rpcMsg->pCont;
+    queuedMsg->pUser   = pUser;
+    mgmtAddToShellQueue(queuedMsg);
   }
 }
 
-static void mgmtProcessShowMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SCMShowMsg *pShowMsg = pMsg->pCont;
-
-  SCMShowMsg *pShowMsg = rpcMsg->pCont;
   if (pShowMsg->type == TSDB_MGMT_TABLE_DNODE || TSDB_MGMT_TABLE_GRANTS || TSDB_MGMT_TABLE_SCORES) {
-    if (mgmtCheckRedirect(rpcMsg->handle) != TSDB_CODE_SUCCESS) {
+    if (mgmtCheckRedirect(pMsg->thandle)) {
       return;
     }
   }
 
-  int32_t  size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
+  if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
-  SCMShowRsp *pShowRsp = rpcMallocCont(size);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_MSG_TYPE);
-  if (pShowRsp == NULL) {
-    rpcRsp.code = TSDB_CODE_SERV_OUT_OF_MEMORY;
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
-  int32_t code;
+  if (!tsMgmtShowMetaFp[pShowMsg->type]) {
-  if (pShowMsg->type >= TSDB_MGMT_TABLE_MAX) {
+    mError("show type:%d %s is not support", pShowMsg->type, taosMsg[pShowMsg->type]);
-    code = TSDB_CODE_INVALID_MSG_TYPE;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OPS_NOT_SUPPORT);
-  } else {
+    return;
-    SShowObj *pShow = (SShowObj *) calloc(1, sizeof(SShowObj) + htons(pShowMsg->payloadLen));
-    pShow->signature = pShow;
-    pShow->type      = pShowMsg->type;
-    strcpy(pShow->db, pShowMsg->db);
-    mTrace("pShow:%p is allocated", pShow);
-
-    // set the table name query condition
-    pShow->payloadLen = htons(pShowMsg->payloadLen);
-    memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
-
-    mgmtSaveQhandle(pShow);
-    pShowRsp->qhandle = htobe64((uint64_t) pShow);
-    if (tsMgmtShowMetaFp[pShowMsg->type]) {
-      code = (*tsMgmtShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, rpcMsg->handle);
-      if (code == 0) {
-        size = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
-      } else {
-        mError("pShow:%p, type:%d %s, failed to get Meta, code:%d", pShow, pShowMsg->type,
-               taosMsg[(uint8_t) pShowMsg->type], code);
-        free(pShow);
-      }
-    } else {
-      code = TSDB_CODE_OPS_NOT_SUPPORT;
-    }
   }
 
-  rpcRsp.code = code;
+  int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
-  rpcRsp.pCont = pShowRsp;
+  SCMShowRsp *pShowRsp = rpcMallocCont(size);
-  rpcRsp.contLen = size;
+  if (pShowRsp == NULL) {
-  rpcSendResponse(&rpcRsp);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
+    return;
+  }
+
+  SShowObj *pShow = (SShowObj *) calloc(1, sizeof(SShowObj) + htons(pShowMsg->payloadLen));
+  pShow->signature  = pShow;
+  pShow->type       = pShowMsg->type;
+  pShow->payloadLen = htons(pShowMsg->payloadLen);
+  strcpy(pShow->db, pShowMsg->db);
+  memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
+
+  mgmtSaveQhandle(pShow);
+  pShowRsp->qhandle = htobe64((uint64_t) pShow);
+
+  int32_t code = (*tsMgmtShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->thandle);
+  if (code == 0) {
+    SRpcMsg rpcRsp = {
+      .handle  = pMsg->thandle,
+      .pCont   = pShowRsp,
+      .contLen = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns,
+      .code    = code,
+      .msgType = 0
+    };
+    rpcSendResponse(&rpcRsp);
+  } else {
+    mError("pShow:%p, type:%d %s, failed to get Meta, code:%d", pShow, pShowMsg->type, taosMsg[pShowMsg->type], code);
+    mgmtFreeQhandle(pShow);
+    rpcFreeCont(pShowRsp);
+  }
 }
 
-static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-
   int32_t rowsToRead = 0;
   int32_t size = 0;
   int32_t rowsRead = 0;
-  SRetrieveTableMsg *pRetrieve = (SRetrieveTableMsg *) rpcMsg->pCont;
+  SRetrieveTableMsg *pRetrieve = pMsg->pCont;
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
   /*
@@ -226,16 +236,14 @@ static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg) {
    */
   if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
     mError("retrieve:%p, qhandle:%p is invalid", pRetrieve, pRetrieve->qhandle);
-    rpcRsp.code = TSDB_CODE_INVALID_QHANDLE;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_QHANDLE);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
-  if (pShow->signature != (void *)pShow) {
+  if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
-    mError("pShow:%p, signature:%p, query memory is corrupted", pShow, pShow->signature);
+    mError("pShow:%p, query memory is corrupted", pShow);
-    rpcRsp.code = TSDB_CODE_MEMORY_CORRUPTED;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MEMORY_CORRUPTED);
-    rpcSendResponse(&rpcRsp);
     return;
   } else {
     if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
@@ -258,10 +266,9 @@ static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg) {
 
   // if free flag is set, client wants to clean the resources
   if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE)
-    rowsRead = (*tsMgmtShowRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, rpcMsg->handle);
+    rowsRead = (*tsMgmtShowRetrieveFp[pShow->type])(pShow, pRsp->data, rowsToRead, pMsg->thandle);
 
-  if (rowsRead < 0) {
+  if (rowsRead < 0) {  // TSDB_CODE_ACTION_IN_PROGRESS;
-    rowsRead = 0;  // TSDB_CODE_ACTION_IN_PROGRESS;
     rpcFreeCont(pRsp);
     return;
   }
@@ -269,8 +276,13 @@ static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg) {
   pRsp->numOfRows = htonl(rowsRead);
   pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI);  // millisecond time precision
 
-  rpcRsp.pCont = pRsp;
+  SRpcMsg rpcRsp = {
-  rpcRsp.contLen = size;
+      .handle  = pMsg->thandle,
+      .pCont   = pRsp,
+      .contLen = size,
+      .code    = 0,
+      .msgType = 0
+  };
   rpcSendResponse(&rpcRsp);
 
   if (rowsToRead == 0) {
@@ -278,21 +290,19 @@ static void mgmtProcessRetrieveMsg(SRpcMsg *rpcMsg) {
   }
 }
 
-static void mgmtProcessHeartBeatMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   //SCMHeartBeatMsg *pHBMsg = (SCMHeartBeatMsg *) rpcMsg->pCont;
   //mgmtSaveQueryStreamList(pHBMsg);
 
   SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp));
   if (pHBRsp == NULL) {
-    rpcRsp.code = TSDB_CODE_SERV_OUT_OF_MEMORY;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   SRpcConnInfo connInfo;
-  if (rpcGetConnInfo(rpcMsg->handle, &connInfo) != 0) {
+  if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) {
-    mError("conn:%p is already released while process heart beat msg", rpcMsg->handle);
+    mError("conn:%p is already released while process heart beat msg", pMsg->thandle);
     return;
   }
 
@@ -320,8 +330,13 @@ static void mgmtProcessHeartBeatMsg(SRpcMsg *rpcMsg) {
   pHBRsp->streamId = 0;
   pHBRsp->killConnection = 0;
 
-  rpcRsp.pCont = pHBRsp;
+  SRpcMsg rpcRsp = {
-  rpcRsp.contLen = sizeof(SCMHeartBeatRsp);
+      .handle  = pMsg->thandle,
+      .pCont   = pHBRsp,
+      .contLen = sizeof(SCMHeartBeatRsp),
+      .code    = 0,
+      .msgType = 0
+  };
   rpcSendResponse(&rpcRsp);
 }
 
@@ -340,13 +355,13 @@ static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secr
   }
 }
 
-static void mgmtProcessConnectMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-  SCMConnectMsg *pConnectMsg = (SCMConnectMsg *) rpcMsg->pCont;
+  SCMConnectMsg *pConnectMsg = pMsg->pCont;
 
   SRpcConnInfo connInfo;
-  if (rpcGetConnInfo(rpcMsg->handle, &connInfo) != 0) {
+  if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) {
-    mError("conn:%p is already released while process connect msg", rpcMsg->handle);
+    mError("thandle:%p is already released while process connect msg", pMsg->thandle);
     return;
   }
 
@@ -450,6 +465,7 @@ static bool mgmtCheckMsgReadOnly(int8_t type, void *pCont) {
 }
 
 static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) {
+  mError("%s is not processed in shell", taosMsg[rpcMsg->msgType]);
   SRpcMsg rpcRsp = {
     .msgType = 0,
     .pCont   = 0,
@@ -459,3 +475,26 @@ static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) {
   };
   rpcSendResponse(&rpcRsp);
 }
+
+static void mgmtProcessMsgWhileNotReady(SRpcMsg *rpcMsg) {
+  mTrace("%s is ignored since SDB is not ready", taosMsg[rpcMsg->msgType]);
+  SRpcMsg rpcRsp = {
+    .msgType = 0,
+    .pCont   = 0,
+    .contLen = 0,
+    .code    = TSDB_CODE_NOT_READY,
+    .handle  = rpcMsg->handle
+  };
+  rpcSendResponse(&rpcRsp);
+}
+
+void mgmtSendSimpleResp(void *thandle, int32_t code) {
+  SRpcMsg rpcRsp = {
+      .msgType = 0,
+      .pCont   = 0,
+      .contLen = 0,
+      .code    = code,
+      .handle  = thandle
+  };
+  rpcSendResponse(&rpcRsp);
+}

src/mnode/src/mgmtSuperTable.c

@@ -165,7 +165,7 @@ int32_t mgmtInitSuperTables() {
   mgmtSuperTableActionInit();
 
   tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS,
-                          "stables", SDB_KEYTYPE_STRING, tsMgmtDirectory, mgmtSuperTableAction);
+                          "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction);
   if (tsSuperTableSdb == NULL) {
     mError("failed to init stables data");
     return -1;

src/mnode/src/mgmtTable.c

@@ -44,14 +44,16 @@
 extern void *tsNormalTableSdb;
 extern void *tsChildTableSdb;
 
-static void mgmtProcessCreateTableMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessCreateTableMsg(SQueuedMsg *queueMsg);
-static void mgmtProcessDropTableMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessDropTableMsg(SQueuedMsg *queueMsg);
-static void mgmtProcessAlterTableMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg);
-static void mgmtProcessTableMetaMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg);
-static void mgmtProcessMultiTableMetaMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg);
-static void mgmtProcessSuperTableMetaMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessSuperTableMetaMsg(SQueuedMsg *queueMsg);
+static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg);
 static int32_t mgmtGetShowTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle);
 
 int32_t mgmtInitTables() {
   int32_t code = mgmtInitSuperTables();
@@ -79,6 +81,7 @@ int32_t mgmtInitTables() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_META, mgmtProcessSuperTableMetaMsg);
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables);
+  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -131,170 +134,58 @@ int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMeta *pMeta, boo
   return TSDB_CODE_SUCCESS;
 }
 
+static void mgmtCreateTable(SVgObj *pVgroup, SQueuedMsg *pMsg) {
+  SCMCreateTableMsg *pCreate = pMsg->pCont;
 
+  pCreate->numOfColumns = htons(pCreate->numOfColumns);
+  pCreate->numOfTags    = htons(pCreate->numOfTags);
+  pCreate->sqlLen       = htons(pCreate->sqlLen);
 
-void mgmtProcessCreateVgroup(SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta) {
+  SSchema *pSchema = (SSchema*) pCreate->schema;
-  SRpcMsg rpcRsp = {.handle = thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  for (int32_t i = 0; i < pCreate->numOfColumns + pCreate->numOfTags; ++i) {
-  SDbObj *pDb = mgmtGetDb(pCreate->db);
+    pSchema->bytes = htons(pSchema->bytes);
-  if (pDb == NULL) {
+    pSchema->colId = i;
-    mError("table:%s, failed to create vgroup, db not found", pCreate->tableId);
+    pSchema++;
-    rpcRsp.code = TSDB_CODE_INVALID_DB;
-    rpcSendResponse(&rpcRsp);
-    return;
   }
 
-  SVgObj *pVgroup = mgmtCreateVgroup(pDb);
-  if (pVgroup == NULL) {
-    mError("table:%s, failed to alloc vnode to vgroup", pCreate->tableId);
-    rpcRsp.code = TSDB_CODE_NO_ENOUGH_DNODES;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
-
-  void *cont = rpcMallocCont(contLen);
-  if (cont == NULL) {
-    mError("table:%s, failed to create table, can not alloc memory", pCreate->tableId);
-    rpcRsp.code = TSDB_CODE_SERV_OUT_OF_MEMORY;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
-
-  memcpy(cont, pCreate, contLen);
-
-  SProcessInfo *info = calloc(1, sizeof(SProcessInfo));
-  info->type    = TSDB_PROCESS_CREATE_VGROUP;
-  info->thandle = thandle;
-  info->ahandle = pVgroup;
-  info->cont    = cont;
-  info->contLen = contLen;
-
-  if (isGetMeta) {
-    info->type = TSDB_PROCESS_CREATE_VGROUP_GET_META;
-  }
-
-  mgmtSendCreateVgroupMsg(pVgroup, info);
-}
-
-//void mgmtSendCreateTableMsg(SMDCreateTableMsg *pCreate, SRpcIpSet *ipSet, void *ahandle) {
-//  mTrace("table:%s, send create table msg, ahandle:%p", pCreate->tableId, ahandle);
-//  SRpcMsg rpcMsg = {
-//    .handle  = ahandle,
-//    .pCont   = pCreate,
-//    .contLen = htonl(pCreate->contLen),
-//    .code    = 0,
-//    .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
-//  };
-//  rpcSendRequest(tsMgmtDClientRpc, ipSet, &rpcMsg);
-//}
-//
-
-
-void mgmtProcessCreateTable(SVgObj *pVgroup, SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta) {
-  assert(pVgroup != NULL);
-  SRpcMsg rpcRsp = {.handle = thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
-
   int32_t sid = taosAllocateId(pVgroup->idPool);
   if (sid < 0) {
-    mTrace("table:%s, no enough sid in vgroup:%d, start to create a new vgroup", pCreate->tableId, pVgroup->vgId);
+    mTrace("thandle:%p, no enough sid in vgroup:%d, start to create a new one", pMsg->thandle, pVgroup->vgId);
-    mgmtProcessCreateVgroup(pCreate, contLen, thandle, isGetMeta);
+    mgmtCreateVgroup(pMsg);
     return;
   }
 
+  int32_t code;
   STableInfo *pTable;
-  SMDCreateTableMsg *pDCreate = NULL;
+  SMDCreateTableMsg *pMDCreate = NULL;
 
   if (pCreate->numOfColumns == 0) {
-    mTrace("table:%s, start to create child table, vgroup:%d sid:%d", pCreate->tableId, pVgroup->vgId, sid);
+    mTrace("thandle:%p, create ctable:%s, vgroup:%d sid:%d ahandle:%p", pMsg->thandle, pCreate->tableId, pVgroup->vgId, sid, pMsg);
-    rpcRsp.code = mgmtCreateChildTable(pCreate, contLen, pVgroup, sid, &pDCreate, &pTable);
+    code = mgmtCreateChildTable(pCreate, pMsg->contLen, pVgroup, sid, &pMDCreate, &pTable);
   } else {
-    mTrace("table:%s, start to create normal table, vgroup:%d sid:%d", pCreate->tableId, pVgroup->vgId, sid);
+    mTrace("thandle:%p, create ntable:%s, vgroup:%d sid:%d ahandle:%p", pMsg->thandle, pCreate->tableId, pVgroup->vgId, sid, pMsg);
-    rpcRsp.code = mgmtCreateNormalTable(pCreate, contLen, pVgroup, sid, &pDCreate, &pTable);
+    code = mgmtCreateNormalTable(pCreate, pMsg->contLen, pVgroup, sid, &pMDCreate, &pTable);
   }
 
-  if (rpcRsp.code != TSDB_CODE_SUCCESS) {
+  if (code != TSDB_CODE_SUCCESS) {
-    mTrace("table:%s, failed to create table in vgroup:%d sid:%d ", pCreate->tableId, pVgroup->vgId, sid);
+    mTrace("thandle:%p, failed to create table:%s in vgroup:%d", pMsg->thandle, pCreate->tableId, pVgroup->vgId);
-    rpcSendResponse(&rpcRsp);
+    mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  assert(pDCreate != NULL);
-  assert(pTable != NULL);
-
-  SProcessInfo *info = calloc(1, sizeof(SProcessInfo));
-  info->type    = TSDB_PROCESS_CREATE_TABLE;
-  info->thandle = thandle;
-  info->ahandle = pTable;
   SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
-  if (isGetMeta) {
-    info->type = TSDB_PROCESS_CREATE_TABLE_GET_META;
-  }
-
   SRpcMsg rpcMsg = {
-    .handle  = info,
+      .handle  = pMsg,
-    .pCont   = pCreate,
+      .pCont   = pCreate,
-    .contLen = htonl(pDCreate->contLen),
+      .contLen = htonl(pMDCreate->contLen),
-    .code    = 0,
+      .code    = 0,
-    .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
+      .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
   };
+
+  pMsg->ahandle = pTable;
   mgmtSendMsgToDnode(&ipSet, &rpcMsg);
 }
 
-int32_t mgmtCreateTable(SCMCreateTableMsg *pCreate, int32_t contLen, void *thandle, bool isGetMeta) {
-  SDbObj *pDb = mgmtGetDb(pCreate->db);
-  if (pDb == NULL) {
-    mError("table:%s, failed to create table, db not selected", pCreate->tableId);
-    return TSDB_CODE_DB_NOT_SELECTED;
-  }
-
-  STableInfo *pTable = mgmtGetTable(pCreate->tableId);
-  if (pTable != NULL) {
-    if (pCreate->igExists) {
-      mTrace("table:%s, table is already exist, think it success", pCreate->tableId);
-      return TSDB_CODE_SUCCESS;
-    } else {
-      mError("table:%s, failed to create table, table already exist", pCreate->tableId);
-      return TSDB_CODE_TABLE_ALREADY_EXIST;
-    }
-  }
-
-  SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct);
-  assert(pAcct != NULL);
-
-  int32_t code = mgmtCheckTableLimit(pAcct, pCreate->numOfColumns);
-  if (code != TSDB_CODE_SUCCESS) {
-    mError("table:%s, failed to create table, table num exceed the limit", pCreate->tableId);
-    return code;
-  }
-
-  if (mgmtCheckExpired()) {
-    mError("table:%s, failed to create table, grant expired", pCreate->tableId);
-    return TSDB_CODE_GRANT_EXPIRED;
-  }
-
-  if (pCreate->numOfTags != 0) {
-    mTrace("table:%s, start to create super table, tags:%d columns:%d",
-        pCreate->tableId, pCreate->numOfTags, pCreate->numOfColumns);
-    return mgmtCreateSuperTable(pDb, pCreate);
-  }
-
-  code = mgmtCheckTimeSeries(pCreate->numOfColumns);
-  if (code != TSDB_CODE_SUCCESS) {
-    mError("table:%s, failed to create table, timeseries exceed the limit", pCreate->tableId);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SVgObj *pVgroup = mgmtGetAvailableVgroup(pDb);
-  if (pVgroup == NULL) {
-    mTrace("table:%s, no avaliable vgroup, start to create a new one", pCreate->tableId);
-    mgmtProcessCreateVgroup(pCreate, contLen, thandle, isGetMeta);
-  } else {
-    mTrace("table:%s, try to create table in vgroup:%d", pCreate->tableId, pVgroup->vgId);
-    mgmtProcessCreateTable(pVgroup, pCreate, contLen, thandle, isGetMeta);
-  }
-
-  return TSDB_CODE_ACTION_IN_PROGRESS;
-}
-
 int32_t mgmtDropTable(SDbObj *pDb, char *tableId, int32_t ignore) {
   STableInfo *pTable = mgmtGetTable(tableId);
   if (pTable == NULL) {
@@ -547,114 +438,142 @@ SMDDropTableMsg *mgmtBuildRemoveTableMsg(STableInfo *pTable) {
 }
 
 void mgmtSetTableDirty(STableInfo *pTable, bool isDirty) {
+  // TODO: if dirty, delete from sdb
   pTable->dirty = isDirty;
 }
 
-void mgmtProcessCreateTableMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
 
-  SCMCreateTableMsg *pCreate = (SCMCreateTableMsg *) rpcMsg->pCont;
+  SCMCreateTableMsg *pCreate = pMsg->pCont;
-  pCreate->numOfColumns = htons(pCreate->numOfColumns);
+  mTrace("thandle:%p, start to create table:%s", pMsg->thandle, pCreate->tableId);
-  pCreate->numOfTags    = htons(pCreate->numOfTags);
-  pCreate->sqlLen       = htons(pCreate->sqlLen);
 
-  SSchema *pSchema = (SSchema*) pCreate->schema;
+  if (mgmtCheckExpired()) {
-  for (int32_t i = 0; i < pCreate->numOfColumns + pCreate->numOfTags; ++i) {
+    mError("thandle:%p, failed to create table:%s, grant expired", pCreate->tableId);
-    pSchema->bytes = htons(pSchema->bytes);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED);
-    pSchema->colId = i;
-    pSchema++;
-  }
-
-  if (mgmtCheckRedirect(rpcMsg->handle) != TSDB_CODE_SUCCESS) {
-    mError("table:%s, failed to create table, need redirect message", pCreate->tableId);
     return;
   }
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  if (!pMsg->pUser->writeAuth) {
-  if (pUser == NULL) {
+    mError("thandle:%p, failed to create table:%s, no rights", pMsg->thandle, pCreate->tableId);
-    mError("table:%s, failed to create table, invalid user", pCreate->tableId);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
-  if (!pUser->writeAuth) {
+  SAcctObj *pAcct = pMsg->pUser->pAcct;
-    mError("table:%s, failed to create table, no rights", pCreate->tableId);
+  int32_t code = mgmtCheckTableLimit(pAcct, htons(pCreate->numOfColumns));
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+  if (code != TSDB_CODE_SUCCESS) {
-    rpcSendResponse(&rpcRsp);
+    mError("thandle:%p, failed to create table:%s, exceed the limit", pMsg->thandle, pCreate->tableId);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
     return;
   }
 
-  int32_t code = mgmtCreateTable(pCreate, rpcMsg->contLen, rpcMsg->handle, false);
+  pMsg->pDb = mgmtGetDb(pCreate->db);
-  if (code != TSDB_CODE_ACTION_IN_PROGRESS) {
+  if (pMsg->pDb == NULL) {
-    rpcRsp.code = code;
+    mError("thandle:%p, failed to create table:%s, db not selected", pMsg->thandle, pCreate->tableId);
-    rpcSendResponse(&rpcRsp);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
+    return;
+  }
+
+  STableInfo *pTable = mgmtGetTable(pCreate->tableId);
+  if (pTable != NULL) {
+    if (pCreate->igExists) {
+      mTrace("thandle:%p, table:%s is already exist", pMsg->thandle, pCreate->tableId);
+      mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
+      return;
+    } else {
+      mError("thandle:%p, failed to create table:%s, table already exist", pMsg->thandle, pCreate->tableId);
+      mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_TABLE_ALREADY_EXIST);
+      return;
+    }
+  }
+
+  if (pCreate->numOfTags != 0) {
+    mTrace("thandle:%p, start to create super table:%s, tags:%d columns:%d",
+           pMsg->thandle, pCreate->tableId, pCreate->numOfTags, pCreate->numOfColumns);
+    code = mgmtCreateSuperTable(pMsg->pDb, pCreate);
+    mgmtSendSimpleResp(pMsg->thandle, code);
+    return;
+  }
+
+  code = mgmtCheckTimeSeries(pCreate->numOfColumns);
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("thandle:%p, failed to create table:%s, timeseries exceed the limit", pMsg->thandle, pCreate->tableId);
+    mgmtSendSimpleResp(pMsg->thandle, code);
+    return;
+  }
+
+  SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg));
+  memcpy(newMsg, pMsg, sizeof(SQueuedMsg));
+  pMsg->pCont = NULL;
+
+  SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb);
+  if (pVgroup == NULL) {
+    mTrace("thandle:%p, table:%s start to create a new vgroup", newMsg->thandle, pCreate->tableId);
+    mgmtCreateVgroup(newMsg);
+  } else {
+    mTrace("thandle:%p, create table:%s in vgroup:%d", newMsg->thandle, pCreate->tableId, pVgroup->vgId);
+    mgmtCreateTable(pVgroup, newMsg);
   }
 }
 
-void mgmtProcessDropTableMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SCMDropTableMsg *pDrop = pMsg->pCont;
-  SCMDropTableMsg *pDrop = (SCMDropTableMsg *) rpcMsg->pCont;
 
-  if (mgmtCheckRedirect(rpcMsg->handle) != TSDB_CODE_SUCCESS) {
+  if (mgmtCheckRedirect(pMsg->thandle)) {
-    mError("table:%s, failed to drop table, need redirect message", pDrop->tableId);
+    mError("thandle:%p, failed to drop table:%s, need redirect message", pMsg->thandle, pDrop->tableId);
     return;
   }
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     mError("table:%s, failed to drop table, invalid user", pDrop->tableId);
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   if (!pUser->writeAuth) {
     mError("table:%s, failed to drop table, no rights", pDrop->tableId);
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   SDbObj *pDb = mgmtGetDbByTableId(pDrop->tableId);
   if (pDb == NULL) {
     mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
-    rpcRsp.code = TSDB_CODE_DB_NOT_SELECTED;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   int32_t code = mgmtDropTable(pDb, pDrop->tableId, pDrop->igNotExists);
   if (code != TSDB_CODE_ACTION_IN_PROGRESS) {
-    rpcRsp.code = code;
+    mgmtSendSimpleResp(pMsg->thandle, code);
-    rpcSendResponse(&rpcRsp);
   }
 }
 
-void mgmtProcessAlterTableMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) {
-  if (mgmtCheckRedirect(rpcMsg->handle) != TSDB_CODE_SUCCESS) {
     return;
   }
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
-  SCMAlterTableMsg *pAlter = (SCMAlterTableMsg *) rpcMsg->pCont;
+  SCMAlterTableMsg *pAlter = pMsg->pCont;
 
+  int32_t code;
   if (!pUser->writeAuth) {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    code = TSDB_CODE_NO_RIGHTS;
   } else {
     pAlter->type      = htons(pAlter->type);
     pAlter->numOfCols = htons(pAlter->numOfCols);
 
     if (pAlter->numOfCols > 2) {
       mError("table:%s error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols);
-      rpcRsp.code = TSDB_CODE_APP_ERROR;
+      code = TSDB_CODE_APP_ERROR;
     } else {
       SDbObj *pDb = mgmtGetDb(pAlter->db);
       if (pDb) {
@@ -662,17 +581,17 @@ void mgmtProcessAlterTableMsg(SRpcMsg *rpcMsg) {
           pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes);
         }
 
-        rpcRsp.code = mgmtAlterTable(pDb, pAlter);
+        code = mgmtAlterTable(pDb, pAlter);
-        if (rpcRsp.code == 0) {
+        if (code == 0) {
           mLPrint("table:%s is altered by %s", pAlter->tableId, pUser->user);
         }
       } else {
-        rpcRsp.code = TSDB_CODE_DB_NOT_SELECTED;
+        code = TSDB_CODE_DB_NOT_SELECTED;
       }
     }
   }
 
-  rpcSendResponse(&rpcRsp);
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }
 
 void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) {
@@ -707,21 +626,14 @@ void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) {
   rpcSendResponse(&rpcRsp);
 }
 
-
+void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
-void mgmtProcessTableMetaMsg(SRpcMsg *rpcMsg) {
+  SCMTableInfoMsg *pInfo = pMsg->pCont;
-  SRpcMsg rpcRsp;
-  rpcRsp.handle  = rpcMsg->handle;
-  rpcRsp.pCont   = NULL;
-  rpcRsp.contLen = 0;
-
-  SCMTableInfoMsg *pInfo = rpcMsg->pCont;
   pInfo->createFlag = htons(pInfo->createFlag);
 
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     mError("table:%s, failed to get table meta, invalid user", pInfo->tableId);
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
@@ -729,12 +641,11 @@ void mgmtProcessTableMetaMsg(SRpcMsg *rpcMsg) {
   if (pTable == NULL) {
     if (pInfo->createFlag != 1) {
       mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
-      rpcRsp.code = TSDB_CODE_INVALID_TABLE;
+      mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
-      rpcSendResponse(&rpcRsp);
       return;
     } else {
       // on demand create table from super table if table does not exists
-      if (mgmtCheckRedirect(rpcMsg->handle) != TSDB_CODE_SUCCESS) {
+      if (mgmtCheckRedirect(pMsg->thandle)) {
         mError("table:%s, failed to create table while get meta info, need redirect message", pInfo->tableId);
         return;
       }
@@ -743,8 +654,7 @@ void mgmtProcessTableMetaMsg(SRpcMsg *rpcMsg) {
       SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
       if (pCreateMsg == NULL) {
         mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId);
-        rpcRsp.code = TSDB_CODE_SERV_OUT_OF_MEMORY;
+        mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
-        rpcSendResponse(&rpcRsp);
         return;
       }
 
@@ -752,41 +662,34 @@ void mgmtProcessTableMetaMsg(SRpcMsg *rpcMsg) {
       strcpy(pCreateMsg->tableId, pInfo->tableId);
 
       mError("table:%s, start to create table while get meta info", pInfo->tableId);
-      mgmtCreateTable(pCreateMsg, contLen, rpcMsg->handle, true);
+//      mgmtCreateTable(pCreateMsg, contLen, pMsg->thandle, true);
     }
   } else {
-    mgmtProcessGetTableMeta(pTable, rpcMsg->handle);
+    mgmtProcessGetTableMeta(pTable, pMsg->thandle);
   }
 }
 
-void mgmtProcessMultiTableMetaMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp;
-  rpcRsp.handle  = rpcMsg->handle;
-  rpcRsp.pCont   = NULL;
-  rpcRsp.contLen = 0;
-
   SRpcConnInfo connInfo;
-  if (rpcGetConnInfo(rpcMsg->handle, &connInfo) != 0) {
+  if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) {
-    mError("conn:%p is already released while get mulit table meta", rpcMsg->handle);
+    mError("conn:%p is already released while get mulit table meta", pMsg->thandle);
     return;
   }
 
   bool usePublicIp = (connInfo.serverIp == tsPublicIpInt);
   SUserObj *pUser = mgmtGetUser(connInfo.user);
   if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
-  SCMMultiTableInfoMsg *pInfo = rpcMsg->pCont;
+  SCMMultiTableInfoMsg *pInfo = pMsg->pCont;
   pInfo->numOfTables = htonl(pInfo->numOfTables);
 
   int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice
   SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen);
   if (pMultiMeta == NULL) {
-    rpcRsp.code = TSDB_CODE_SERV_OUT_OF_MEMORY;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
@@ -823,29 +726,69 @@ void mgmtProcessMultiTableMetaMsg(SRpcMsg *rpcMsg) {
     }
   }
 
+  SRpcMsg rpcRsp = {0};
+  rpcRsp.handle = pMsg->thandle;
   rpcRsp.pCont = pMultiMeta;
   rpcRsp.contLen = pMultiMeta->contLen;
   rpcSendResponse(&rpcRsp);
 }
 
-void mgmtProcessSuperTableMetaMsg(SRpcMsg *rpcMsg) {
+void mgmtProcessSuperTableMetaMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  SCMSuperTableInfoMsg *pInfo = pMsg->pCont;
-  SCMSuperTableInfoMsg *pInfo = rpcMsg->pCont;
   STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId);
   if (pTable == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_TABLE;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable);
   if (pRsp != NULL) {
     int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t);
+    SRpcMsg rpcRsp = {0};
+    rpcRsp.handle = pMsg->thandle;
     rpcRsp.pCont = pRsp;
     rpcRsp.contLen = msgLen;
     rpcSendResponse(&rpcRsp);
   } else {
-    rpcRsp.code = TSDB_CODE_INVALID_TABLE;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
-    rpcSendResponse(&rpcRsp);
   }
-}
+}
+
+static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) {
+  if (rpcMsg->handle == NULL) return;
+
+  SQueuedMsg *queueMsg = rpcMsg->handle;
+  queueMsg->received++;
+
+  STableInfo *pTable = queueMsg->ahandle;
+  mTrace("thandle:%p, create table:%s rsp received, ahandle:%p code:%d received:%d",
+         queueMsg->thandle, pTable->tableId, rpcMsg->handle, rpcMsg->code, queueMsg->received);
+
+  if (rpcMsg->code != TSDB_CODE_SUCCESS) {
+    mgmtSetTableDirty(pTable, true);
+    //sdbDeleteRow(tsVgroupSdb, pVgroup);
+    mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
+    mError("table:%s, failed to create in dnode, reason:%s, set it dirty", pTable->tableId, tstrerror(rpcMsg->code));
+    mgmtSetTableDirty(pTable, true);
+  } else {
+    mTrace("table:%s, created in dnode", pTable->tableId);
+    mgmtSetTableDirty(pTable, false);
+
+    if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) {
+      SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg));
+      newMsg->msgType = queueMsg->msgType;
+      newMsg->thandle = queueMsg->thandle;
+      newMsg->pDb     = queueMsg->pDb;
+      newMsg->pUser   = queueMsg->pUser;
+      newMsg->contLen = queueMsg->contLen;
+      newMsg->pCont   = rpcMallocCont(newMsg->contLen);
+      memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen);
+      mTrace("table:%s, start to process get meta", pTable->tableId);
+      mgmtAddToShellQueue(newMsg);
+    } else {
+      mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
+    }
+  }
+
+  free(queueMsg);
+}

src/mnode/src/mgmtUser.c

@@ -33,9 +33,9 @@ static int32_t   mgmtUpdateUser(SUserObj *pUser);
 static int32_t   mgmtGetUserMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 static int32_t   mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
-static void mgmtProcessCreateUserMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg);
-static void mgmtProcessAlterUserMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg);
-static void mgmtProcessDropUserMsg(SRpcMsg *rpcMsg);
+static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg);
 
 static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize);
 static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize);
@@ -59,7 +59,7 @@ int32_t mgmtInitUsers() {
   SUserObj tObj;
   tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj;
 
-  tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMgmtDirectory, mgmtUserAction);
+  tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction);
   if (tsUserSdb == NULL) {
     mError("failed to init user data");
     return -1;
@@ -337,52 +337,40 @@ SUserObj *mgmtGetUserFromConn(void *pConn) {
   return NULL;
 }
 
-static void mgmtProcessCreateUserMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
-
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
-  if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
-    return;
-  }
 
+  int32_t code;
+  SUserObj *pUser = pMsg->pUser;
+  
   if (pUser->superAuth) {
-    SCMCreateUserMsg *pCreate = rpcMsg->pCont;
+    SCMCreateUserMsg *pCreate = pMsg->pCont;
-    rpcRsp.code = mgmtCreateUser(pUser->pAcct, pCreate->user, pCreate->pass);
+    code = mgmtCreateUser(pUser->pAcct, pCreate->user, pCreate->pass);
-    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+    if (code == TSDB_CODE_SUCCESS) {
       mLPrint("user:%s is created by %s", pCreate->user, pUser->user);
     }
   } else {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    code = TSDB_CODE_NO_RIGHTS;
   }
 
-  rpcSendResponse(&rpcRsp);
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }
 
-static void mgmtProcessAlterUserMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
 
-  SUserObj *pOperUser = mgmtGetUserFromConn(rpcMsg->handle);
+  int32_t code;
-  if (pOperUser == NULL) {
+  SUserObj *pOperUser = pMsg->pUser;
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+  
-    rpcSendResponse(&rpcRsp);
+  SCMAlterUserMsg *pAlter = pMsg->pCont;
-    return;
-  }
-
-  SCMAlterUserMsg *pAlter = rpcMsg->pCont;
   SUserObj *pUser = mgmtGetUser(pAlter->user);
   if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
   if (strcmp(pUser->user, "monitor") == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    rpcSendResponse(&rpcRsp);
     return;
   }
 
@@ -405,13 +393,13 @@ static void mgmtProcessAlterUserMsg(SRpcMsg *rpcMsg) {
     if (hasRight) {
       memset(pUser->pass, 0, sizeof(pUser->pass));
       taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass);
-      rpcRsp.code = mgmtUpdateUser(pUser);
+      code = mgmtUpdateUser(pUser);
-      mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, rpcRsp.code);
+      mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code);
     } else {
-      rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+      code = TSDB_CODE_NO_RIGHTS;
     }
 
-    rpcSendResponse(&rpcRsp);
+    mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
@@ -454,42 +442,34 @@ static void mgmtProcessAlterUserMsg(SRpcMsg *rpcMsg) {
         pUser->writeAuth = 1;
       }
 
-      rpcRsp.code = mgmtUpdateUser(pUser);
+      code = mgmtUpdateUser(pUser);
-      mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, rpcRsp.code);
+      mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code);
     } else {
-      rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+      code = TSDB_CODE_NO_RIGHTS;
     }
 
-    rpcSendResponse(&rpcRsp);
+    mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+  mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-  rpcSendResponse(&rpcRsp);
 }
 
-static void mgmtProcessDropUserMsg(SRpcMsg *rpcMsg) {
+static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) {
-  SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
+  if (mgmtCheckRedirect(pMsg->thandle)) return;
-  if (mgmtCheckRedirect(rpcMsg->handle)) return;
 
-  SUserObj *pOperUser = mgmtGetUserFromConn(rpcMsg->handle);
+  int32_t code;
-  if (pOperUser == NULL) {
+  SUserObj *pOperUser = pMsg->pUser;
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
-    rpcSendResponse(&rpcRsp);
-    return ;
-  }
 
-  SCMDropUserMsg *pDrop = rpcMsg->pCont;
+  SCMDropUserMsg *pDrop = pMsg->pCont;
   SUserObj *pUser = mgmtGetUser(pDrop->user);
   if (pUser == NULL) {
-    rpcRsp.code = TSDB_CODE_INVALID_USER;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER);
-    rpcSendResponse(&rpcRsp);
     return ;
   }
 
   if (strcmp(pUser->user, "monitor") == 0 || (strcmp(pUser->user + 1, pUser->acct) == 0 && pUser->user[0] == '_')) {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS);
-    rpcSendResponse(&rpcRsp);
     return ;
   }
 
@@ -511,13 +491,13 @@ static void mgmtProcessDropUserMsg(SRpcMsg *rpcMsg) {
   }
 
   if (hasRight) {
-    rpcRsp.code = mgmtDropUser(pUser->pAcct, pDrop->user);
+    code = mgmtDropUser(pUser->pAcct, pDrop->user);
-    if (rpcRsp.code == TSDB_CODE_SUCCESS) {
+    if (code == TSDB_CODE_SUCCESS) {
       mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user);
     }
   } else {
-    rpcRsp.code = TSDB_CODE_NO_RIGHTS;
+    code = TSDB_CODE_NO_RIGHTS;
   }
 
-  rpcSendResponse(&rpcRsp);
+  mgmtSendSimpleResp(pMsg->thandle, code);
 }

src/mnode/src/mgmtVgroup.c

@@ -24,6 +24,7 @@
 #include "mgmtDb.h"
 #include "mgmtDClient.h"
 #include "mgmtDnode.h"
+#include "mgmtProfile.h"
 #include "mgmtShell.h"
 #include "mgmtTable.h"
 #include "mgmtVgroup.h"
@@ -42,6 +43,9 @@ static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t
 
 static int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void    mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg);
+
+void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
 
 static void mgmtVgroupActionInit() {
   SVgObj tObj;
@@ -69,7 +73,7 @@ int32_t mgmtInitVgroups() {
 
   mgmtVgroupActionInit();
 
-  tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMgmtDirectory, mgmtVgroupAction);
+  tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction);
   if (tsVgroupSdb == NULL) {
     mError("failed to init vgroups data");
     return -1;
@@ -114,6 +118,7 @@ int32_t mgmtInitVgroups() {
 
   mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta);
   mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups);
+  mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp);
 
   mTrace("vgroup is initialized");
   return 0;
@@ -123,19 +128,6 @@ SVgObj *mgmtGetVgroup(int32_t vgId) {
   return (SVgObj *)sdbGetRow(tsVgroupSdb, &vgId);
 }
 
-int32_t mgmtAllocateSid(SDbObj *pDb, SVgObj *pVgroup) {
-  int32_t sid = taosAllocateId(pVgroup->idPool);
-  if (sid < 0) {
-    mWarn("table:%s, vgroup:%d run out of ID, num:%d", pDb->name, pVgroup->vgId, taosIdPoolNumOfUsed(pVgroup->idPool));
-    pDb->vgStatus = TSDB_VG_STATUS_IN_PROGRESS;
-    mgmtCreateVgroup(pDb);
-    terrno = TSDB_CODE_ACTION_IN_PROGRESS;
-  }
-
-  terrno = 0;
-  return sid;
-}
-
 /*
  * TODO: check if there is enough sids
  */
@@ -155,21 +147,25 @@ void mgmtProcessVgTimer(void *handle, void *tmrId) {
   pDb->vgTimer = NULL;
 }
 
-SVgObj *mgmtCreateVgroup(SDbObj *pDb) {
+void mgmtCreateVgroup(SQueuedMsg *pMsg) {
+  SDbObj *pDb = pMsg->pDb;
+  if (pDb == NULL) {
+    mError("thandle:%p, failed to create vgroup, db not found", pMsg->thandle);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
+    return;
+  }
+
   SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1);
   strcpy(pVgroup->dbName, pDb->name);
   pVgroup->numOfVnodes = pDb->cfg.replications;
-  pVgroup->createdTime = taosGetTimestampMs();
-
-  // based on load balance, create a new one
   if (mgmtAllocVnodes(pVgroup) != 0) {
-    mError("db:%s, no enough free dnode to alloc %d vnodes", pDb->name, pVgroup->numOfVnodes);
+    mError("thandle:%p, db:%s no enough dnode to alloc %d vnodes", pMsg->thandle, pDb->name, pVgroup->numOfVnodes);
     free(pVgroup);
-    pDb->vgStatus = TSDB_VG_STATUS_FULL;
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES);
-    taosTmrReset(mgmtProcessVgTimer, 5000, pDb, tsMgmtTmr, &pDb->vgTimer);
+    return;
-    return NULL;
   }
 
+  pVgroup->createdTime = taosGetTimestampMs();
   pVgroup->tableList   = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions);
   pVgroup->numOfTables = 0;
   pVgroup->idPool      = taosInitIdPool(pDb->cfg.maxSessions);
@@ -179,11 +175,16 @@ SVgObj *mgmtCreateVgroup(SDbObj *pDb) {
 
   sdbInsertRow(tsVgroupSdb, pVgroup, 0);
 
-  mTrace("vgroup:%d, vgroup is created, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+  mPrint("thandle:%p, vgroup:%d is created in mnode, db:%s replica:%d", pMsg->thandle, pVgroup->vgId, pDb->name,
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i)
+         pVgroup->numOfVnodes);
-    mTrace("vgroup:%d, dnode:%s vnode:%d is created", pVgroup->vgId, taosIpStr(pVgroup->vnodeGid[i].ip), pVgroup->vnodeGid[i].vnode);
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    mPrint("thandle:%p, vgroup:%d, dnode:%s vnode:%d", pMsg->thandle, pVgroup->vgId,
+           taosIpStr(pVgroup->vnodeGid[i].ip), pVgroup->vnodeGid[i].vnode);
+  }
 
-  return pVgroup;
+  pMsg->ahandle = pVgroup;
+  pMsg->expected = pVgroup->numOfVnodes;
+  mgmtSendCreateVgroupMsg(pVgroup, pMsg);
 }
 
 int32_t mgmtDropVgroup(SDbObj *pDb, SVgObj *pVgroup) {
@@ -514,13 +515,13 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode) {
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) return NULL;
 
-  SMDCreateVnodeMsg *pVPeers = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
+  SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg));
-  if (pVPeers == NULL) return NULL;
+  if (pVnode == NULL) return NULL;
 
-  pVPeers->vnode = htonl(vnode);
+  pVnode->vnode = htonl(vnode);
-  pVPeers->cfg   = pDb->cfg;
+  pVnode->cfg   = pDb->cfg;
 
-  SVnodeCfg *pCfg = &pVPeers->cfg;
+  SVnodeCfg *pCfg = &pVnode->cfg;
   pCfg->vgId                         = htonl(pVgroup->vgId);
   pCfg->maxSessions                  = htonl(pCfg->maxSessions);
   pCfg->cacheBlockSize               = htonl(pCfg->cacheBlockSize);
@@ -534,13 +535,14 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode) {
   pCfg->replications                 = (char) pVgroup->numOfVnodes;
   pCfg->rowsInFileBlock              = htonl(pCfg->rowsInFileBlock);
 
-  SVnodeDesc *vpeerDesc = pVPeers->vpeerDesc;
+  SVnodeDesc *vpeerDesc = pVnode->vpeerDesc;
   for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {
-    vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].ip);
+    vpeerDesc[j].vgId  = htonl(pVgroup->vgId);
+    vpeerDesc[j].ip    = htonl(pVgroup->vnodeGid[j].ip);
     vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode);
   }
 
-  return pVPeers;
+  return pVnode;
 }
 
 SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode) {
@@ -558,7 +560,11 @@ SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode) {
 }
 
 SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
-  SRpcIpSet ipSet = {.numOfIps = pVgroup->numOfVnodes, .inUse = 0, .port = tsMnodeDnodePort + 1};
+  SRpcIpSet ipSet = {
+    .numOfIps = pVgroup->numOfVnodes,
+    .inUse = 0,
+    .port = tsDnodeMnodePort
+  };
   for (int i = 0; i < pVgroup->numOfVnodes; ++i) {
     ipSet.ip[i] = pVgroup->vnodeGid[i].ip;
   }
@@ -566,7 +572,12 @@ SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) {
 }
 
 SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip) {
-  SRpcIpSet ipSet = {.ip[0] = ip, .numOfIps = 1, .inUse = 0, .port = tsMnodeDnodePort + 1};
+  SRpcIpSet ipSet = {
+    .ip[0]    = ip,
+    .numOfIps = 1,
+    .inUse    = 0,
+    .port     = tsDnodeMnodePort
+  };
   return ipSet;
 }
 
@@ -574,19 +585,54 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, int32_t vnode, SRpcIpSet *ipSet, vo
   mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, vnode, ahandle);
   SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup, vnode);
   SRpcMsg rpcMsg = {
-      .handle  = ahandle,
+    .handle  = ahandle,
-      .pCont   = pCreate,
+    .pCont   = pCreate,
-      .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0,
+    .contLen = pCreate ? sizeof(SMDCreateVnodeMsg) : 0,
-      .code    = 0,
+    .code    = 0,
-      .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
+    .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE
   };
   mgmtSendMsgToDnode(ipSet, &rpcMsg);
 }
 
 void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgroup:%d, send create all vnodes msg, handle:%p", pVgroup->vgId, ahandle);
+  mTrace("send create vgroup:%d msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].ip);
     mgmtSendCreateVnodeMsg(pVgroup, pVgroup->vnodeGid[i].vnode, &ipSet, ahandle);
   }
+}
+
+static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
+  if (rpcMsg->handle == NULL) return;
+
+  SQueuedMsg *queueMsg = rpcMsg->handle;
+  queueMsg->received++;
+  if (rpcMsg->code == TSDB_CODE_SUCCESS) {
+    queueMsg->code = rpcMsg->code;
+    queueMsg->successed++;
+  }
+
+  SVgObj *pVgroup = queueMsg->ahandle;
+  mTrace("thandle:%p, vgroup:%d create vnode rsp received, ahandle:%p code:%d received:%d successed:%d expected:%d",
+         queueMsg->thandle, pVgroup->vgId, rpcMsg->handle, rpcMsg->code, queueMsg->received, queueMsg->successed,
+         queueMsg->expected);
+
+  if (queueMsg->received != queueMsg->expected) return;
+
+  if (queueMsg->received == queueMsg->successed) {
+    SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg));
+    newMsg->msgType = queueMsg->msgType;
+    newMsg->thandle = queueMsg->thandle;
+    newMsg->pDb     = queueMsg->pDb;
+    newMsg->pUser   = queueMsg->pUser;
+    newMsg->contLen = queueMsg->contLen;
+    newMsg->pCont   = rpcMallocCont(newMsg->contLen);
+    memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen);
+    mgmtAddToShellQueue(newMsg);
+  } else {
+    sdbDeleteRow(tsVgroupSdb, pVgroup);
+    mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
+  }
+
+  free(queueMsg);
 }

src/os/darwin/src/tdarwin.c

@@ -34,7 +34,7 @@
 #include "tutil.h"
 
 char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
-char tsDirectory[TSDB_FILENAME_LEN] = "/var/lib/taos";
+char tsVnodeDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
 char dataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
 char logDir[TSDB_FILENAME_LEN] = "~/TDengineLog";
 char scriptDir[TSDB_FILENAME_LEN] = "/etc/taos";

src/os/linux/src/tlinux.c

@@ -35,7 +35,9 @@
 #include "ttimer.h"
 
 char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
-char tsDirectory[TSDB_FILENAME_LEN] = "/var/lib/taos";
+char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
+char tsDnodeDir[TSDB_FILENAME_LEN] = {0};
+char tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char dataDir[TSDB_FILENAME_LEN] = "/var/lib/taos";
 char logDir[TSDB_FILENAME_LEN] = "/var/log/taos";
 char scriptDir[TSDB_FILENAME_LEN] = "/etc/taos";

src/os/windows/src/twindows.c

@@ -33,7 +33,7 @@
 #include <Winsock2.h>
 
 char configDir[TSDB_FILENAME_LEN] = "C:/TDengine/cfg";
-char tsDirectory[TSDB_FILENAME_LEN] = "C:/TDengine/data";
+char tsVnodeDir[TSDB_FILENAME_LEN] = "C:/TDengine/data";
 char logDir[TSDB_FILENAME_LEN] = "C:/TDengine/log";
 char dataDir[TSDB_FILENAME_LEN] = "C:/TDengine/data";
 char scriptDir[TSDB_FILENAME_LEN] = "C:/TDengine/script";

src/util/inc/ihash.h

@@ -34,6 +34,12 @@ char *taosAddIntHash(void *handle, uint64_t key, char *pData);
 
 int32_t taosHashInt(void *handle, uint64_t key);
 
+void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *));
+
+char *taosVisitIntHashWithFp(void *handle, int (*fp)(char *));
+
+int32_t taosGetIntHashSize(void *handle);
+
 #ifdef __cplusplus
 }
 #endif

src/util/inc/tglobalcfg.h

@@ -50,7 +50,9 @@ extern int tscEmbedded;
 extern int64_t tsMsPerDay[2];
 
 extern char configDir[];
-extern char tsDirectory[];
+extern char tsVnodeDir[];
+extern char tsDnodeDir[];
+extern char tsMnodeDir[];
 extern char dataDir[];
 extern char logDir[];
 extern char scriptDir[];
@@ -263,9 +265,6 @@ SGlobalConfig *tsGetConfigOption(const char *option);
 
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-extern char          tsMgmtDirectory[];
-
-
 #ifdef __cplusplus
 }
 #endif

src/util/src/ihash.c

@@ -26,7 +26,7 @@ typedef struct {
   IHashNode **hashList;
   int32_t     maxSessions;
   int32_t     dataSize;
-  int32_t (*hashFp)(void *, uint64_t key);
+  int32_t   (*hashFp)(void *, uint64_t key);
   pthread_mutex_t mutex;
 } IHashObj;
 
@@ -186,3 +186,93 @@ void taosCleanUpIntHash(void *handle) {
   memset(pObj, 0, sizeof(IHashObj));
   free(pObj);
 }
+
+
+void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) {
+  IHashObj * pObj;
+  IHashNode *pNode, *pNext;
+
+  pObj = (IHashObj *)handle;
+  if (pObj == NULL || pObj->maxSessions <= 0) return;
+
+  pthread_mutex_lock(&pObj->mutex);
+
+  if (pObj->hashList) {
+    for (int i = 0; i < pObj->maxSessions; ++i) {
+      pNode = pObj->hashList[i];
+      while (pNode) {
+        pNext = pNode->next;
+        if (fp != NULL) fp(pNode->data);
+        free(pNode);
+        pNode = pNext;
+      }
+    }
+
+    free(pObj->hashList);
+  }
+
+  pthread_mutex_unlock(&pObj->mutex);
+
+  pthread_mutex_destroy(&pObj->mutex);
+
+  memset(pObj, 0, sizeof(IHashObj));
+  free(pObj);
+}
+
+char *taosVisitIntHashWithFp(void *handle, int (*fp)(char *)) {
+  IHashObj * pObj;
+  IHashNode *pNode, *pNext;
+  char *     pData = NULL;
+
+  pObj = (IHashObj *)handle;
+  if (pObj == NULL || pObj->maxSessions <= 0) return NULL;
+
+  pthread_mutex_lock(&pObj->mutex);
+
+  if (pObj->hashList) {
+    for (int i = 0; i < pObj->maxSessions; ++i) {
+      pNode = pObj->hashList[i];
+      while (pNode) {
+        pNext = pNode->next;
+        int flag = fp(pNode->data);
+        if (flag) {
+          pData = pNode->data;
+          goto VisitEnd;
+        }
+
+        pNode = pNext;
+      }
+    }
+  }
+
+VisitEnd:
+
+  pthread_mutex_unlock(&pObj->mutex);
+  return pData;
+}
+
+int32_t taosGetIntHashSize(void *handle) {
+  IHashObj * pObj;
+  IHashNode *pNode, *pNext;
+  char *     pData = NULL;
+  int32_t    num = 0;
+
+  pObj = (IHashObj *)handle;
+  if (pObj == NULL || pObj->maxSessions <= 0) return NULL;
+
+  pthread_mutex_lock(&pObj->mutex);
+
+  if (pObj->hashList) {
+    for (int i = 0; i < pObj->maxSessions; ++i) {
+      pNode = pObj->hashList[i];
+      while (pNode) {
+        pNext = pNode->next;
+        num++;
+        pNode = pNext;
+      }
+    }
+  }
+
+  pthread_mutex_unlock(&pObj->mutex);
+  return num;
+}

src/util/src/shash.c

@@ -33,7 +33,7 @@ typedef struct {
   SHashNode **hashList;
   uint32_t    maxSessions;
   uint32_t    dataSize;
-  uint32_t (*hashFp)(void *, char *string);
+  uint32_t  (*hashFp)(void *, char *string);
   pthread_mutex_t mutex;
 } SHashObj;
 

src/vnode/tsdb/CMakeLists.txt

@@ -15,5 +15,5 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   TARGET_LINK_LIBRARIES(tsdb common tutil)
 
   # Someone has no gtest directory, so comment it
-  #ADD_SUBDIRECTORY(tests)
+  ADD_SUBDIRECTORY(tests)
 ENDIF ()

src/vnode/tsdb/inc/tsdb.h

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#if !defined(_TD_TSDB_H_)
+#ifndef _TD_TSDB_H_
 #define _TD_TSDB_H_
 
 #include <pthread.h>
@@ -30,34 +30,19 @@ extern "C" {
 #define TSDB_VERSION_MAJOR 1
 #define TSDB_VERSION_MINOR 0
 
-typedef void tsdb_repo_t;  // use void to hide implementation details from outside
+#define TSDB_INVALID_SUPER_TABLE_ID -1
-
-typedef struct {
-  int64_t uid;  // the unique table ID
-  int32_t tid;  // the table ID in the repository.
-} STableId;
-
-// Submit message for this TSDB
-typedef struct {
-  int32_t numOfTables;
-  int32_t compressed;
-  char    data[];
-} SSubmitMsg;
-
-// Submit message for one table
-typedef struct {
-  STableId tableId;
-  int32_t  padding;    // TODO just for padding here
-  int32_t  sversion;   // data schema version
-  int32_t  len;        // message length
-  char     data[];
-} SSubmitBlock;
 
+// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
 enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };
+typedef enum {
+  TSDB_SUPER_TABLE,  // super table
+  TSDB_NTABLE,       // table not created from super table
+  TSDB_STABLE        // table created from super table
+} TSDB_TABLE_TYPE;
 
-// the TSDB repository configuration
 typedef struct {
   int8_t  precision;
+  int32_t vgId;
   int32_t tsdbId;
   int32_t maxTables;            // maximum number of tables this repository can have
   int32_t daysPerFile;          // day per file sharding policy
@@ -67,6 +52,83 @@ typedef struct {
   int64_t maxCacheSize;         // maximum cache size this TSDB can use
 } STsdbCfg;
 
+void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
+STsdbCfg *tsdbCreateDefaultCfg();
+void      tsdbFreeCfg(STsdbCfg *pCfg);
+
+// --------- TSDB REPOSITORY DEFINITION
+typedef void tsdb_repo_t;  // use void to hide implementation details from outside
+
+tsdb_repo_t *  tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
+int32_t        tsdbDropRepo(tsdb_repo_t *repo);
+tsdb_repo_t *  tsdbOpenRepo(char *tsdbDir);
+int32_t        tsdbCloseRepo(tsdb_repo_t *repo);
+int32_t        tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
+
+// --------- TSDB TABLE DEFINITION
+typedef struct {
+  int64_t uid;  // the unique table ID
+  int32_t tid;  // the table ID in the repository.
+} STableId;
+
+// --------- TSDB TABLE configuration
+typedef struct {
+  TSDB_TABLE_TYPE type;
+  STableId        tableId;
+  int64_t         superUid;
+  STSchema *      schema;
+  STSchema *      tagSchema;
+  SDataRow        tagValues;
+} STableCfg;
+
+int  tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid);
+int  tsdbTableSetSuperUid(STableCfg *config, int64_t uid);
+int  tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
+int  tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
+int  tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup);
+void tsdbClearTableCfg(STableCfg *config);
+
+int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
+int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
+int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
+
+// Submit message for one table
+typedef struct {
+  STableId tableId;
+  int32_t  padding;    // TODO just for padding here
+  int32_t  sversion;   // data schema version
+  int32_t  len;        // data part length, not including the SSubmitBlk head
+  char     data[];
+} SSubmitBlk;
+
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SDataRow row;
+} SSubmitBlkIter;
+
+int      tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+
+// Submit message for this TSDB
+typedef struct {
+  int32_t    length;
+  int32_t    compressed;
+  SSubmitBlk blocks[];
+} SSubmitMsg;
+
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+// SSubmitMsg Iterator
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  SSubmitBlk *pBlock;
+} SSubmitMsgIter;
+
+int         tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter);
+
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
   STsdbCfg tsdbCfg;
@@ -75,22 +137,7 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-
+STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
-// the meter configuration
-typedef struct {
-  STableId tableId;
-
-  int64_t stableUid;
-  int64_t createdTime;
-
-  int32_t  numOfCols;  // number of columns. For table form super table, not includes the tag schema
-  STSchema *schema;     // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL
-                       // If numOfCols < schema->numOfCols, it is a table created from super table
-                       // assert(numOfCols <= schema->numOfCols);
-
-  SDataRow tagValues;  // NULL if it is normal table
-                       // otherwise, it contains the tag values.
-} STableCfg;
 
 // the meter information report structure
 typedef struct {
@@ -99,70 +146,7 @@ typedef struct {
   int64_t   tableTotalDataSize;  // In bytes
   int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;
-
+STableInfo *   tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
-/**
- * Create a configuration for TSDB default
- * @return a pointer to a configuration. the configuration must call tsdbFreeCfg to free memory after usage
- */
-STsdbCfg *tsdbCreateDefaultCfg();
-
-/**
- * Free
- */
-void tsdbFreeCfg(STsdbCfg *pCfg);
-
-/**
- * Create a new TSDB repository
- * @param rootDir the TSDB repository root directory
- * @param pCfg the TSDB repository configuration, upper layer to free the pointer
- *
- * @return a TSDB repository handle on success, NULL for failure and the error number is set
- */
-tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
-
-/**
- * Close and free all resources taken by the repository
- * @param repo the TSDB repository handle. The interface will free the handle too, so upper
- *              layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbDropRepo(tsdb_repo_t *repo);
-
-/**
- * Open an existing TSDB storage repository
- * @param tsdbDir the existing TSDB root directory
- *
- * @return a TSDB repository handle on success, NULL for failure and the error number is set
- */
-tsdb_repo_t *tsdbOpenRepo(char *tsdbDir);
-
-/**
- * Close a TSDB repository. Only free memory resources, and keep the files.
- * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
- *              layer do NOT need to free the repo handle again.
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbCloseRepo(tsdb_repo_t *repo);
-
-/**
- * Change the configuration of a repository
- * @param pCfg the repository configuration, the upper layer should free the pointer
- *
- * @return 0 for success, -1 for failure and the error number is set
- */
-int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
-
-/**
- * Get the TSDB repository information, including some statistics
- * @param pRepo the TSDB repository handle
- * @param error the error number to set when failure occurs
- *
- * @return a info struct handle on success, NULL for failure and the error number is set. The upper
- *         layers should free the info handle themselves or memory leak will occur
- */
-STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
 
 // -- For table manipulation
 
@@ -173,8 +157,6 @@ STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
-int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
 
 /**
  * Drop a table in a repository and free all the resources it takes
@@ -184,7 +166,6 @@ int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
  *
  * @return 0 for success, -1 for failure and the error number is set
  */
-int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
 
 /**
  * Get the information of a table in the repository
@@ -194,7 +175,6 @@ int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
  *
  * @return a table information handle for success, NULL for failure and the error number is set
  */
-STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
 
 // -- FOR INSERT DATA
 /**

src/vnode/tsdb/inc/tsdbCache.h

@@ -53,7 +53,7 @@ typedef struct STSDBCache {
 #define TSDB_NEXT_CACHE_BLOCK(pBlock) ((pBlock)->next)
 #define TSDB_PREV_CACHE_BLOCK(pBlock) ((pBlock)->prev)
 
-STsdbCache *tsdbCreateCache(int32_t numOfBlocks);
+STsdbCache *tsdbInitCache(int64_t maxSize);
 int32_t     tsdbFreeCache(STsdbCache *pCache);
 void *      tsdbAllocFromCache(STsdbCache *pCache, int64_t bytes);
 

src/vnode/tsdb/inc/tsdbFile.h

@@ -16,7 +16,8 @@
 #define _TD_TSDB_FILE_H_
 
 #include <stdint.h>
-// #include "tstring.h"
+
+#include "taosdef.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,7 +35,8 @@ typedef enum {
 extern const char *tsdbFileSuffix[];
 
 typedef struct {
-  int64_t fileSize;
+  int64_t size;
+  int64_t tombSize;
 } SFileInfo;
 
 typedef struct {

src/vnode/tsdb/inc/tsdbMeta.h

@@ -20,6 +20,7 @@
 #include "tsdb.h"
 #include "dataformat.h"
 #include "tskiplist.h"
+#include "tsdbMetaFile.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,62 +31,47 @@ extern "C" {
 // Initially, there are 4 tables
 #define TSDB_INIT_NUMBER_OF_SUPER_TABLE 4
 
-typedef enum {
-  TSDB_SUPER_TABLE,  // super table
-  TSDB_NTABLE,       // table not created from super table
-  TSDB_STABLE        // table created from super table
-} TSDB_TABLE_TYPE;
-
 #define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)
 
+// ---------- TSDB TABLE DEFINITION
 typedef struct STable {
-  STableId        tableId;
   TSDB_TABLE_TYPE type;
-
+  STableId        tableId;
-  int64_t createdTime;
+  int32_t         superUid;  // Super table UID
-
+  STSchema *      schema;
-  // super table UID -1 for normal table
+  STSchema *      tagSchema;
-  int32_t stableUid;
+  SDataRow        tagVal;
-
-  int32_t numOfCols;
-
-  // Schema for this table
-  // For TSDB_SUPER_TABLE, it is the schema including tags
-  // For TSDB_NTABLE, it is only the schema, not including tags
-  // For TSDB_STABLE, it is NULL
-  STSchema *pSchema;
-
-  // Tag value for this table
-  // For TSDB_SUPER_TABLE and TSDB_NTABLE, it is NULL
-  // For TSDB_STABLE, it is the tag value string
-  SDataRow pTagVal;
-
-  // Object content;
-  // For TSDB_SUPER_TABLE, it is the index of tables created from it
-  // For TSDB_STABLE and TSDB_NTABLE, it is the cache data
   union {
-    void *pData;
+    void *pData;   // For TSDB_NTABLE and TSDB_STABLE, it is the skiplist for cache data
-    void *pIndex;
+    void *pIndex;  // For TSDB_SUPER_TABLE, it is the skiplist index
   } content;
-
+  void *         eventHandler;   // TODO
-  // A handle to deal with event
+  void *         streamHandler;  // TODO
-  void *eventHandler;
+  struct STable *next; // TODO: remove the next
-
-  // A handle to deal with stream
-  void *streamHandler;
-
-  struct STable *next;
-
 } STable;
 
+void *  tsdbEncodeTable(STable *pTable, int *contLen);
+STable *tsdbDecodeTable(void *cont, int contLen);
+void *  tsdbFreeEncode(void *cont);
+
+// ---------- TSDB META HANDLE DEFINITION
 typedef struct {
-  int32_t  maxTables;
+  int32_t maxTables;  // Max number of tables
-  int32_t  nTables;
+
-  STable **tables;    // array of normal tables
+  int32_t nTables;  // Tables created
-  STable * stables;   // linked list of super tables // TODO use container to implement this
+
-  void *   tableMap;  // hash map of uid ==> STable *
+  STable **tables;  // table array
+
+  STable *superList;  // super table list TODO: change  it to list container
+
+  void *map; // table map of (uid ===> table)
+
+  SMetaFile *mfh; // meta file handle
 } STsdbMeta;
 
+STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables);
+int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
+
 // ---- Operation on STable
 #define TSDB_TABLE_ID(pTable) ((pTable)->tableId)
 #define TSDB_TABLE_UID(pTable) ((pTable)->uid)
@@ -97,21 +83,12 @@ typedef struct {
 #define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData)
 #define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex)
 
-STSchema *tsdbGetTableSchema(STable *pTable);
-
 // ---- Operation on SMetaHandle
 #define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables)
 #define TSDB_NUM_OF_SUPER_TABLES(pHandle) ((pHandle)->numOfSuperTables)
 #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
 #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */
 
-// Create a new meta handle with configuration
-STsdbMeta *tsdbCreateMeta(int32_t maxTables);
-int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
-
-// Recover the meta handle from the file
-STsdbMeta *tsdbOpenMeta(char *tsdbDir);
-
 int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
 int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
 STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);

src/vnode/tsdb/inc/tsdbMetaFile.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TSDB_META_FILE_
+#define _TSDB_META_FILE_
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TSDB_META_FILE_NAME "META"
+#define TSDB_META_HASH_FRACTION 1.1
+
+typedef struct {
+  int     fd;       // File descriptor
+  int     nDel;     // number of deletions
+  int     nRecord;  // Number of records
+  int64_t size;     // Total file size
+  void *  map;      // Map from uid ==> position
+} SMetaFile;
+
+SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables);
+int32_t    tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+int32_t    tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid);
+int32_t    tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
+void       tsdbCloseMetaFile(SMetaFile *mfh);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // _TSDB_META_FILE_

src/vnode/tsdb/src/tsdbCache.c

@@ -16,7 +16,7 @@
 
 #include "tsdbCache.h"
 
-STsdbCache *tsdbCreateCache(int32_t numOfBlocks) {
+STsdbCache *tsdbInitCache(int64_t maxSize) {
   STsdbCache *pCacheHandle = (STsdbCache *)malloc(sizeof(STsdbCache));
   if (pCacheHandle == NULL) {
     // TODO : deal with the error

src/vnode/tsdb/src/tsdbFile.c

@@ -18,6 +18,52 @@
 
 #include "tsdbFile.h"
 
+typedef struct {
+  int64_t offset;
+} SCompHeader;
+
+typedef struct {
+  int64_t uid;
+  int64_t last : 1;
+  int64_t numOfBlocks : 63;
+  int32_t delimiter;
+} SCompInfo;
+
+typedef struct {
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  int32_t numOfBlocks;
+  int32_t offset;
+} SCompIdx;
+
+typedef struct {
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+  int64_t offset;
+  int32_t len;
+  int32_t sversion;
+} SCompBlock;
+
+typedef struct {
+  int64_t uid;
+} SBlock;
+
+typedef struct {
+  int16_t colId;
+  int16_t bytes;
+  int32_t nNullPoints;
+  int32_t type:8;
+  int32_t offset:24;
+  int32_t len;
+  // fields for pre-aggregate
+  // TODO: pre-aggregation should be seperated
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIdx;
+  int16_t minIdx;
+} SField;
+
 const char *tsdbFileSuffix[] = {
     ".head",  // TSDB_FILE_TYPE_HEAD
     ".data",  // TSDB_FILE_TYPE_DATA

src/vnode/tsdb/src/tsdbMain.c

@@ -42,6 +42,9 @@
 #define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024)       // 4M
 #define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024)    // 1G
 
+#define TSDB_CFG_FILE_NAME "CONFIG"
+#define TSDB_DATA_DIR_NAME "data"
+
 enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING };
 
 typedef struct _tsdb_repo {
@@ -75,16 +78,18 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo);
 static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
 static int     tsdbOpenMetaFile(char *tsdbDir);
 static int     tsdbRecoverRepo(int fd, STsdbCfg *pCfg);
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlock *pBlock);
+static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
 
 #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
 #define TSDB_GET_TABLE_BY_NAME(pRepo, name)
 #define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE)
 #define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED)
 
-STsdbCfg *tsdbCreateDefaultCfg() {
+/**
-  STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg));
+ * Set the default TSDB configuration
-  if (pCfg == NULL) return NULL;
+ */
+void tsdbSetDefaultCfg(STsdbCfg *pCfg) {
+  if (pCfg == NULL) return;
 
   pCfg->precision = -1;
   pCfg->tsdbId = 0;
@@ -94,6 +99,18 @@ STsdbCfg *tsdbCreateDefaultCfg() {
   pCfg->maxRowsPerFileBlock = -1;
   pCfg->keep = -1;
   pCfg->maxCacheSize = -1;
+}
+
+/**
+ * Create a configuration for TSDB default
+ * @return a pointer to a configuration. the configuration object 
+ *         must call tsdbFreeCfg to free memory after usage
+ */
+STsdbCfg *tsdbCreateDefaultCfg() {
+  STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg));
+  if (pCfg == NULL) return NULL;
+
+  tsdbSetDefaultCfg(pCfg);
 
   return pCfg;
 }
@@ -102,7 +119,15 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
   if (pCfg != NULL) free(pCfg);
 }
 
-tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
+/**
+ * Create a new TSDB repository
+ * @param rootDir the TSDB repository root directory
+ * @param pCfg the TSDB repository configuration, upper layer need to free the pointer
+ * @param limiter the limitation tracker will implement in the future, make it void now
+ *
+ * @return a TSDB repository handle on success, NULL for failure
+ */
+tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) {
   if (rootDir == NULL) return NULL;
 
   if (access(rootDir, F_OK | R_OK | W_OK) == -1) return NULL;
@@ -120,35 +145,44 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter) {
   pRepo->config = *pCfg;
   pRepo->limiter = limiter;
 
-  pRepo->tsdbMeta = tsdbCreateMeta(pCfg->maxTables);
+  // Create the environment files and directories
-  if (pRepo->tsdbMeta == NULL) {
-    free(pRepo->rootDir);
-    free(pRepo);
-    return NULL;
-  }
-
-  pRepo->tsdbCache = tsdbCreateCache(5);
-  if (pRepo->tsdbCache == NULL) {
-    free(pRepo->rootDir);
-    tsdbFreeMeta(pRepo->tsdbMeta);
-    free(pRepo);
-    return NULL;
-  }
-
-  // Create the Meta data file and data directory
   if (tsdbSetRepoEnv(pRepo) < 0) {
     free(pRepo->rootDir);
-    tsdbFreeMeta(pRepo->tsdbMeta);
-    tsdbFreeCache(pRepo->tsdbCache);
     free(pRepo);
     return NULL;
   }
 
+  // Initialize meta
+  STsdbMeta *pMeta = tsdbInitMeta(rootDir, pCfg->maxTables);
+  if (pMeta == NULL) {
+    free(pRepo->rootDir);
+    free(pRepo);
+    return NULL;
+  }
+  pRepo->tsdbMeta = pMeta;
+
+  // Initialize cache
+  STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize);
+  if (pCache == NULL) {
+    free(pRepo->rootDir);
+    tsdbFreeMeta(pRepo->tsdbMeta);
+    free(pRepo);
+    return NULL;
+  }
+  pRepo->tsdbCache = pCache;
+
   pRepo->state = TSDB_REPO_STATE_ACTIVE;
 
   return (tsdb_repo_t *)pRepo;
 }
 
+/**
+ * Close and free all resources taken by the repository
+ * @param repo the TSDB repository handle. The interface will free the handle too, so upper
+ *              layer do NOT need to free the repo handle again.
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbDropRepo(tsdb_repo_t *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
@@ -169,6 +203,12 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) {
   return 0;
 }
 
+/**
+ * Open an existing TSDB storage repository
+ * @param tsdbDir the existing TSDB root directory
+ *
+ * @return a TSDB repository handle on success, NULL for failure and the error number is set
+ */
 tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
   if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) {
     return NULL;
@@ -191,7 +231,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
     return NULL;
   }
 
-  pRepo->tsdbCache = tsdbCreateCache(5);
+  pRepo->tsdbCache = tsdbInitCache(5);
   if (pRepo->tsdbCache == NULL) {
     // TODO: deal with error
     return NULL;
@@ -208,6 +248,13 @@ static int32_t tsdbFlushCache(STsdbRepo *pRepo) {
   return 0;
 }
 
+/**
+ * Close a TSDB repository. Only free memory resources, and keep the files.
+ * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
+ *              layer do NOT need to free the repo handle again.
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   if (pRepo == NULL) return 0;
@@ -223,6 +270,12 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) {
   return 0;
 }
 
+/**
+ * Change the configuration of a repository
+ * @param pCfg the repository configuration, the upper layer should free the pointer
+ *
+ * @return 0 for success, -1 for failure and the error number is set
+ */
 int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
@@ -231,22 +284,30 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
   return 0;
 }
 
+/**
+ * Get the TSDB repository information, including some statistics
+ * @param pRepo the TSDB repository handle
+ * @param error the error number to set when failure occurs
+ *
+ * @return a info struct handle on success, NULL for failure and the error number is set. The upper
+ *         layers should free the info handle themselves or memory leak will occur
+ */
 STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo) {
   // TODO
   return NULL;
 }
 
-int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
+int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg);
 }
 
-int32_t tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
+int tsdbAlterTable(tsdb_repo_t *pRepo, STableCfg *pCfg) {
   // TODO
   return 0;
 }
 
-int32_t tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
+int tsdbDropTable(tsdb_repo_t *repo, STableId tableId) {
   // TODO
   if (repo == NULL) return -1;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
@@ -261,18 +322,150 @@ STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tableId) {
 
 // TODO: need to return the number of data inserted
 int32_t tsdbInsertData(tsdb_repo_t *repo, SSubmitMsg *pMsg) {
-  SSubmitBlock *pBlock = (SSubmitBlock *)pMsg->data;
+  SSubmitMsgIter msgIter;
 
-  for (int i = 0; i < pMsg->numOfTables; i++) {  // Loop to deal with the submit message
+  tsdbInitSubmitMsgIter(pMsg, &msgIter);
+  SSubmitBlk *pBlock;
+  while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) {
     if (tsdbInsertDataToTable(repo, pBlock) < 0) {
       return -1;
     }
-    pBlock = (SSubmitBlock *)(((char *)pBlock) + sizeof(SSubmitBlock) + pBlock->len);
   }
 
   return 0;
 }
 
+/**
+ * Initialize a table configuration
+ */
+int tsdbInitTableCfg(STableCfg *config, TSDB_TABLE_TYPE type, int64_t uid, int32_t tid) {
+  if (config == NULL) return -1;
+  if (type != TSDB_NTABLE && type != TSDB_STABLE) return -1;
+
+  memset((void *)config, 0, sizeof(STableCfg));
+
+  config->type = type;
+  config->superUid = TSDB_INVALID_SUPER_TABLE_ID;
+  config->tableId.uid = uid;
+  config->tableId.tid = tid;
+  return 0;
+}
+
+/**
+ * Set the super table UID of the created table
+ */
+int tsdbTableSetSuperUid(STableCfg *config, int64_t uid) {
+  if (config->type != TSDB_STABLE) return -1;
+  if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1;
+
+  config->superUid = uid;
+  return 0;
+}
+
+/**
+ * Set the table schema in the configuration
+ * @param config the configuration to set
+ * @param pSchema the schema to set
+ * @param dup use the schema directly or duplicate one for use
+ * 
+ * @return 0 for success and -1 for failure
+ */
+int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+  if (dup) {
+    config->schema = tdDupSchema(pSchema);
+  } else {
+    config->schema = pSchema;
+  }
+  return 0;
+}
+
+/**
+ * Set the table schema in the configuration
+ * @param config the configuration to set
+ * @param pSchema the schema to set
+ * @param dup use the schema directly or duplicate one for use
+ * 
+ * @return 0 for success and -1 for failure
+ */
+int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) {
+  if (config->type != TSDB_STABLE) return -1;
+
+  if (dup) {
+    config->tagSchema = tdDupSchema(pSchema);
+  } else {
+    config->tagSchema = pSchema;
+  }
+  return 0;
+}
+
+int tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup) {
+  if (config->type != TSDB_STABLE) return -1;
+
+  if (dup) {
+    config->tagValues = tdDataRowDup(row);
+  } else {
+    config->tagValues = row;
+  }
+
+  return 0;
+}
+
+void tsdbClearTableCfg(STableCfg *config) {
+  if (config->schema) tdFreeSchema(config->schema);
+  if (config->tagSchema) tdFreeSchema(config->tagSchema);
+  if (config->tagValues) tdFreeDataRow(config->tagValues);
+}
+
+int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+  if (pBlock->len <= 0) return -1;
+  pIter->totalLen = pBlock->len;
+  pIter->len = 0;
+  pIter->row = (SDataRow)(pBlock->data);
+  return 0;
+}
+
+SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  SDataRow row = pIter->row;
+  if (row == NULL) return NULL;
+
+  pIter->len += dataRowLen(row);
+  if (pIter->len >= pIter->totalLen) {
+    pIter->row = NULL;
+  } else {
+    pIter->row = (char *)row + dataRowLen(row);
+  }
+
+  return row;
+}
+
+int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+  if (pMsg == NULL || pIter == NULL) return -1;
+
+  pIter->totalLen = pMsg->length;
+  pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE;
+  if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
+    pIter->pBlock = NULL;
+  } else {
+    pIter->pBlock = pMsg->blocks;
+  }
+
+  return 0;
+}
+
+SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
+  SSubmitBlk *pBlock = pIter->pBlock;
+  if (pBlock == NULL) return NULL;
+
+  pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len;
+  if (pIter->len >= pIter->totalLen) {
+    pIter->pBlock = NULL;
+  } else {
+    pIter->pBlock = (char *)pBlock + pBlock->len + sizeof(SSubmitBlk);
+  }
+
+  return pBlock;
+}
+
 // Check the configuration and set default options
 static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   // Check precision
@@ -285,7 +478,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   // Check tsdbId
   if (pCfg->tsdbId < 0) return -1;
 
-  // Check MaxTables
+  // Check maxTables
   if (pCfg->maxTables == -1) {
     pCfg->maxTables = TSDB_DEFAULT_TABLES;
   } else {
@@ -333,10 +526,18 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   return 0;
 }
 
-static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
+static int32_t tsdbGetCfgFname(STsdbRepo *pRepo, char *fname) {
-  char *metaFname = tsdbGetFileName(pRepo->rootDir, "tsdb", TSDB_FILE_TYPE_META);
+  if (pRepo == NULL) return -1;
+  sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_CFG_FILE_NAME);
+  return 0;
+}
 
-  int fd = open(metaFname, O_WRONLY | O_CREAT);
+static int32_t tsdbSaveConfig(STsdbRepo *pRepo) {
+  char fname[128] = "\0"; // TODO: get rid of the literal 128
+
+  if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+
+  int fd = open(fname, O_WRONLY | O_CREAT, 0755);
   if (fd < 0) {
     return -1;
   }
@@ -345,20 +546,46 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
     return -1;
   }
 
-  // Create the data file
+  close(fd);
-  char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2);
+  return 0;
-  if (dirName == NULL) {
+}
+
+static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) {
+  char fname[128] = "\0";
+
+  if (tsdbGetCfgFname(pRepo, fname) < 0) return -1;
+
+  int fd = open(fname, O_RDONLY);
+  if (fd < 0) {
     return -1;
   }
 
-  sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb");
+  if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) {
+    close(fd);
+    return -1;
+  }
+
+  close(fd);
+
+  return 0;
+}
+
+static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname) {
+  if (pRepo == NULL || pRepo->rootDir == NULL) return -1;
+  sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_DATA_DIR_NAME);
+  return 0;
+}
+
+static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) {
+  if (tsdbSaveConfig(pRepo) < 0) return -1;
+
+  char dirName[128] = "\0";
+  if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1;
+
   if (mkdir(dirName, 0755) < 0) {
-    free(dirName);
     return -1;
   }
 
-  free(dirName);
-
   return 0;
 }
 
@@ -417,7 +644,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   }
 
   pNode->level = level;
-  tdDataRowCpy(SL_GET_NODE_DATA(pNode), row);
+  dataRowCpy(SL_GET_NODE_DATA(pNode), row);
 
   // Insert the skiplist node into the data
   tsdbInsertRowToTableImpl(pNode, pTable);
@@ -425,23 +652,19 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
   return 0;
 }
 
-static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlock *pBlock) {
+static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
 
   STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, pBlock->tableId);
-  if (pTable == NULL) {
+  if (pTable == NULL) return -1;
-    return -1;
-  }
 
-  SDataRows     rows = pBlock->data;
+  SSubmitBlkIter blkIter;
-  SDataRowsIter rDataIter, *pIter;
-  pIter = &rDataIter;
   SDataRow row;
 
-  tdInitSDataRowsIter(rows, pIter);
+  tsdbInitSubmitBlkIter(pBlock, &blkIter);
-  while ((row = tdDataRowsNext(pIter)) != NULL) {
+  while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
     if (tdInsertRowToTable(pRepo, row, pTable) < 0) {
-      // TODO: deal with the error here
+      return -1;
     }
   }
 

src/vnode/tsdb/src/tsdbMeta.c

@@ -9,6 +9,7 @@
 #include "tsdbCache.h"
 
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
+#define TSDB_META_FILE_NAME "META"
 
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
@@ -16,24 +17,85 @@ static int     tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable);
 static int     tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable);
+static int     tsdbEstimateTableEncodeSize(STable *pTable);
 
-STsdbMeta *tsdbCreateMeta(int32_t maxTables) {
+/**
-  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
+ * Encode a TSDB table object as a binary content
-  if (pMeta == NULL) {
+ * ASSUMPTIONS: VALID PARAMETERS
-    return NULL;
+ * 
+ * @param pTable table object to encode
+ * @param contLen the encoded binary content length
+ * 
+ * @return binary content for success
+ *         NULL fro failure
+ */
+void *tsdbEncodeTable(STable *pTable, int *contLen) {
+  if (pTable == NULL) return NULL;
+
+  *contLen = tsdbEstimateTableEncodeSize(pTable);
+  if (*contLen < 0) return NULL;
+
+  void *ret = malloc(*contLen);
+  if (ret == NULL) return NULL;
+
+  // TODO: encode the object to the memory
+  {}
+
+  return ret;
+}
+
+/**
+ * Decode from an encoded binary
+ * ASSUMPTIONS: valid parameters
+ * 
+ * @param cont binary object
+ * @param contLen binary length
+ * 
+ * @return TSDB table object for success
+ *         NULL for failure
+ */
+STable *tsdbDecodeTable(void *cont, int contLen) {
+  STable *pTable = (STable *)calloc(1, sizeof(STable));
+  if (pTable == NULL) return NULL;
+
+  {
+    // TODO recover from the binary content
   }
 
+  return pTable;
+}
+
+void *tsdbFreeEncode(void *cont) {
+  if (cont != NULL) free(cont);
+}
+
+/**
+ * Initialize the meta handle
+ * ASSUMPTIONS: VALID PARAMETER
+ */
+STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
+  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
+  if (pMeta == NULL) return NULL;
+
   pMeta->maxTables = maxTables;
   pMeta->nTables = 0;
-  pMeta->stables = NULL;
+  pMeta->superList = NULL;
   pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *));
   if (pMeta->tables == NULL) {
     free(pMeta);
     return NULL;
   }
 
-  pMeta->tableMap = taosHashInit(maxTables + maxTables / 10, taosGetDefaultHashFunction, false);
+  pMeta->map = taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
-  if (pMeta->tableMap == NULL) {
+  if (pMeta->map == NULL) {
+    free(pMeta->tables);
+    free(pMeta);
+    return NULL;
+  }
+
+  pMeta->mfh = tsdbInitMetaFile(rootDir, maxTables);
+  if (pMeta->mfh == NULL) {
+    taosHashCleanup(pMeta->map);
     free(pMeta->tables);
     free(pMeta);
     return NULL;
@@ -45,6 +107,8 @@ STsdbMeta *tsdbCreateMeta(int32_t maxTables) {
 int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
   if (pMeta == NULL) return 0;
 
+  tsdbCloseMetaFile(pMeta->mfh);
+
   for (int i = 0; i < pMeta->maxTables; i++) {
     if (pMeta->tables[i] != NULL) {
       tsdbFreeTable(pMeta->tables[i]);
@@ -53,14 +117,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 
   free(pMeta->tables);
 
-  STable *pTable = pMeta->stables;
+  STable *pTable = pMeta->superList;
   while (pTable != NULL) {
     STable *pTemp = pTable;
     pTable = pTemp->next;
     tsdbFreeTable(pTemp);
   }
 
-  taosHashCleanup(pMeta->tableMap);
+  taosHashCleanup(pMeta->map);
 
   free(pMeta);
 
@@ -68,74 +132,65 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 }
 
 int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
-  if (tsdbCheckTableCfg(pCfg) < 0) {
+  if (tsdbCheckTableCfg(pCfg) < 0) return -1;
-    return -1;
-  }
 
-  STable *pSTable = NULL;
+  STable *super = NULL;
   int newSuper = 0;
 
-  if (IS_CREATE_STABLE(pCfg)) {  // to create a TSDB_STABLE, check if super table exists
+  if (pCfg->type == TSDB_STABLE) { 
-    pSTable = tsdbGetTableByUid(pMeta, pCfg->stableUid);
+    super = tsdbGetTableByUid(pMeta, pCfg->superUid);
-    if (pSTable == NULL) {  // super table not exists, try to create it
+    if (super == NULL) {  // super table not exists, try to create it
       newSuper = 1;
-      pSTable = (STable *)calloc(1, sizeof(STable));
+      // TODO: use function to implement create table object
-      if (pSTable == NULL) return -1;
+      super = (STable *)calloc(1, sizeof(STable));
+      if (super == NULL) return -1;
 
-      pSTable->tableId.uid = pCfg->stableUid;
+      super->type = TSDB_SUPER_TABLE;
-      pSTable->tableId.tid = -1;
+      super->tableId.uid = pCfg->superUid;
-      pSTable->type = TSDB_SUPER_TABLE;
+      super->tableId.tid = -1;
-      // pSTable->createdTime = pCfg->createdTime; // The created time is not required
+      super->superUid = TSDB_INVALID_SUPER_TABLE_ID;
-      pSTable->stableUid = -1;
+      super->schema = tdDupSchema(pCfg->schema);
-      pSTable->numOfCols = pCfg->numOfCols;
+      super->tagSchema = tdDupSchema(pCfg->tagSchema);
-      pSTable->pSchema = tdDupSchema(pCfg->schema);
+      super->tagVal = tdDataRowDup(pCfg->tagValues);
-      pSTable->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1,
+      super->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1,
                                                 0, NULL);  // Allow duplicate key, no lock
-      if (pSTable->content.pIndex == NULL) {
+
-        free(pSTable);
+      if (super->content.pIndex == NULL) {
+        tdFreeSchema(super->schema);
+        tdFreeSchema(super->tagSchema);
+        tdFreeDataRow(super->tagVal);
+        free(super);
         return -1;
       }
     } else {
-      if (pSTable->type != TSDB_SUPER_TABLE) return -1;
+      if (super->type != TSDB_SUPER_TABLE) return -1;
     }
   }
 
-  STable *pTable = (STable *)malloc(sizeof(STable));
+  STable *table = (STable *)malloc(sizeof(STable));
-  if (pTable == NULL) {
+  if (table == NULL) {
-    if (newSuper) tsdbFreeTable(pSTable);
+    if (newSuper) tsdbFreeTable(super);
     return -1;
   }
 
-  pTable->tableId = pCfg->tableId;
+  table->tableId = pCfg->tableId;
-  pTable->createdTime = pCfg->createdTime;
   if (IS_CREATE_STABLE(pCfg)) { // TSDB_STABLE
-    pTable->type = TSDB_STABLE;
+    table->type = TSDB_STABLE;
-    pTable->stableUid = pCfg->stableUid;
+    table->superUid = pCfg->superUid;
-    pTable->pTagVal = tdDataRowDup(pCfg->tagValues);
+    table->tagVal = tdDataRowDup(pCfg->tagValues);
   } else { // TSDB_NTABLE
-    pTable->type = TSDB_NTABLE;
+    table->type = TSDB_NTABLE;
-    pTable->stableUid = -1;
+    table->superUid = -1;
-    pTable->pSchema = tdDupSchema(pCfg->schema);
+    table->schema = tdDupSchema(pCfg->schema);
   }
-  pTable->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, 0, 8, 0, 0, NULL);
+  table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, 0, 8, 0, 0, NULL);
 
-  if (newSuper) tsdbAddTableToMeta(pMeta, pSTable);
+  if (newSuper) tsdbAddTableToMeta(pMeta, super);
-  tsdbAddTableToMeta(pMeta, pTable);
+  tsdbAddTableToMeta(pMeta, table);
 
   return 0;
 }
 
-STsdbMeta *tsdbOpenMeta(char *tsdbDir) {
-  // TODO : Open meta file for reading
-
-  STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
-  if (pMeta == NULL) {
-    return NULL;
-  }
-
-  return pMeta;
-}
-
 /**
  * Check if a table is valid to insert.
  * @return NULL for invalid and the pointer to the table if valid
@@ -183,9 +238,9 @@ int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) {
 static int tsdbFreeTable(STable *pTable) {
   // TODO: finish this function
   if (pTable->type == TSDB_STABLE) {
-    tdFreeDataRow(pTable->pTagVal);
+    tdFreeDataRow(pTable->tagVal);
   } else {
-    tdFreeSchema(pTable->pSchema);
+    tdFreeSchema(pTable->schema);
   }
 
   // Free content
@@ -205,7 +260,7 @@ static int32_t tsdbCheckTableCfg(STableCfg *pCfg) {
 }
 
 STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
-  void *ptr = taosHashGet(pMeta->tableMap, (char *)(&uid), sizeof(uid));
+  void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid));
 
   if (ptr == NULL) return NULL;
 
@@ -215,12 +270,12 @@ STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid) {
 static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable) {
   if (pTable->type == TSDB_SUPER_TABLE) { 
     // add super table to the linked list
-    if (pMeta->stables == NULL) {
+    if (pMeta->superList == NULL) {
-      pMeta->stables = pTable;
+      pMeta->superList = pTable;
       pTable->next = NULL;
     } else {
-      STable *pTemp = pMeta->stables;
+      STable *pTemp = pMeta->superList;
-      pMeta->stables = pTable;
+      pMeta->superList = pTable;
       pTable->next = pTemp;
     }
   } else {
@@ -244,7 +299,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) {
 static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) {
   // TODO: add the table to the map
   int64_t uid = pTable->tableId.uid;
-  if (taosHashPut(pMeta->tableMap, (char *)(&uid), sizeof(uid), (void *)(&pTable), sizeof(pTable)) < 0) {
+  if (taosHashPut(pMeta->map, (char *)(&uid), sizeof(uid), (void *)(&pTable), sizeof(pTable)) < 0) {
     return -1;
   }
   return 0;
@@ -259,4 +314,9 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   assert(pTable->type == TSDB_STABLE);
   // TODO
   return 0;
+}
+
+static int tsdbEstimateTableEncodeSize(STable *pTable) {
+  // TODO
+  return 0;
 }

src/vnode/tsdb/src/tsdbMetaFile.c

@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "taosdef.h"
+#include "hash.h"
+#include "tsdbMetaFile.h"
+
+#define TSDB_META_FILE_HEADER_SIZE 512
+
+typedef struct {
+  int32_t offset;
+  int32_t size;
+} SRecordInfo;
+
+static int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
+static int32_t tsdbCheckMetaHeader(int fd);
+static int32_t tsdbWriteMetaHeader(int fd);
+static int     tsdbCreateMetaFile(char *fname);
+static int     tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh);
+
+SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables) {
+  // TODO
+  char fname[128] = "\0";
+  if (tsdbGetMetaFileName(rootDir, fname) < 0) return NULL;
+
+  SMetaFile *mfh = (SMetaFile *)calloc(1, sizeof(SMetaFile));
+  if (mfh == NULL) return NULL;
+
+  // OPEN MAP
+  mfh->map =
+      taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false);
+  if (mfh->map == NULL) {
+    free(mfh);
+    return NULL;
+  }
+
+  // OPEN FILE
+  if (access(fname, F_OK) < 0) {  // file not exists
+    mfh->fd = tsdbCreateMetaFile(fname);
+    if (mfh->fd < 0) {
+      taosHashCleanup(mfh->map);
+      free(mfh);
+      return NULL;
+    }
+  } else {  // file exists, recover from file
+    if (tsdbRestoreFromMetaFile(fname, mfh) < 0) {
+      taosHashCleanup(mfh->map);
+      free(mfh);
+      return NULL;
+    }
+  }
+
+  return mfh;
+}
+
+int32_t tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+  if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) {
+    return -1;
+  }
+
+  SRecordInfo info;
+  info.offset = mfh->size;
+  info.size = contLen;  // TODO: Here is not correct
+
+  mfh->size += (contLen + sizeof(SRecordInfo));
+
+  if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  // TODO: make below a function to implement
+  if (fseek(mfh->fd, info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, cont, contLen) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  mfh->nRecord++;
+
+  return 0;
+}
+
+int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid) {
+  char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
+  if (ptr == NULL) return -1;
+
+  SRecordInfo info = *(SRecordInfo *)ptr;
+
+  // Remove record from hash table
+  taosHashRemove(mfh->map, (char *)(&uid), sizeof(uid));
+
+  // Remove record from file
+
+  info.offset = -info.offset;
+  if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  mfh->nDel++;
+
+  return 0;
+}
+
+int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen) {
+  char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid));
+  if (ptr == NULL) return -1;
+
+  SRecordInfo info = *(SRecordInfo *)ptr;
+  // Update the hash table
+  if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  // Update record in file
+  if (info.size >= contLen) {  // Just update it in place
+    info.size = contLen;
+
+  } else {  // Just append to the end of file
+    info.offset = mfh->size;
+    info.size = contLen;
+
+    mfh->size += contLen;
+  }
+  if (fseek(mfh->fd, -info.offset, SEEK_CUR) < 0) {
+    return -1;
+  }
+
+  if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) {
+    return -1;
+  }
+
+  fsync(mfh->fd);
+
+  return 0;
+}
+
+void tsdbCloseMetaFile(SMetaFile *mfh) {
+  if (mfh == NULL) return;
+  close(mfh);
+
+  taosHashCleanup(mfh->map);
+}
+
+static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) {
+  if (rootDir == NULL) return -1;
+  sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME);
+  return 0;
+}
+
+static int32_t tsdbCheckMetaHeader(int fd) {
+  // TODO: write the meta file header check function
+  return 0;
+}
+
+static int32_t tsdbWriteMetaHeader(int fd) {
+  // TODO: write the meta file header to file
+  return 0;
+}
+
+static int tsdbCreateMetaFile(char *fname) {
+  int fd = open(fname, O_RDWR | O_CREAT, 0755);
+  if (fd < 0) return -1;
+
+  if (tsdbWriteMetaHeader(fd) < 0) {
+    close(fd);
+    return NULL;
+  }
+
+  return fd;
+}
+
+static int tsdbCheckMetaFileIntegrety(int fd) {
+  // TODO
+  return 0;
+}
+
+static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) {
+  int fd = open(fname, O_RDWR);
+  if (fd < 0) return -1;
+
+  if (tsdbCheckMetaFileIntegrety(fd) < 0) {
+    // TODO: decide if to auto-recover the file
+    close(fd);
+    return -1;
+  }
+
+  if (fseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) {
+    // TODO: deal with the error
+    close(fd);
+    return -1;
+  }
+
+  mfh->fd = fd;
+  // TODO: iterate to read the meta file to restore the meta data
+
+  return 0;
+}

src/vnode/tsdb/tests/tsdbTests.cpp

@@ -3,92 +3,63 @@
 
 #include "tsdb.h"
 #include "dataformat.h"
-#include "tsdbMeta.h"
-
-TEST(TsdbTest, createTable) {
-  STsdbMeta *pMeta = tsdbCreateMeta(100);
-  ASSERT_NE(pMeta, nullptr);
-
-  STableCfg config;
-  config.tableId.tid = 0;
-  config.tableId.uid = 98868728187539L;
-  config.numOfCols = 5;
-  config.schema = tdNewSchema(config.numOfCols);
-  for (int i = 0; i < schemaNCols(config.schema); i++) {
-    STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
-    tdColCpy(schemaColAt(config.schema, i), pCol);
-    tdFreeCol(pCol);
-  }
-  config.tagValues = nullptr;
-
-  tsdbCreateTableImpl(pMeta, &config);
-
-  STable *pTable = tsdbGetTableByUid(pMeta, config.tableId.uid);
-  ASSERT_NE(pTable, nullptr);
-}
 
 TEST(TsdbTest, createRepo) {
-  STsdbCfg *pCfg = tsdbCreateDefaultCfg();
+  STsdbCfg config;
 
-  // Create a tsdb repository
+  // 1. Create a tsdb repository
-  tsdb_repo_t *pRepo = tsdbCreateRepo("/root/mnt/test/vnode0", pCfg, NULL);
+  tsdbSetDefaultCfg(&config);
+  tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
   ASSERT_NE(pRepo, nullptr);
-  tsdbFreeCfg(pCfg);
 
-  // create a normal table in this repository
+  // 2. Create a normal table
-  STableCfg config;
+  STableCfg tCfg;
-  config.tableId.tid = 0;
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
-  config.tableId.uid = 98868728187539L;
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NTABLE, 987607499877672L, 0), 0);
-  config.numOfCols = 5;
+
-  config.schema = tdNewSchema(config.numOfCols);
+  int       nCols = 5;
-  STColumn *pCol = tdNewCol(TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
+  STSchema *schema = tdNewSchema(nCols);
-  tdColCpy(schemaColAt(config.schema, 0), pCol);
+
-  tdFreeCol(pCol);
+  for (int i = 0; i < nCols; i++) {
-  for (int i = 1; i < schemaNCols(config.schema); i++) {
+    if (i == 0) {
-    pCol = tdNewCol(TSDB_DATA_TYPE_BIGINT, i, 0);
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
-    tdColCpy(schemaColAt(config.schema, i), pCol);
+    } else {
-    tdFreeCol(pCol);
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+    }
   }
 
-  tsdbCreateTable(pRepo, &config);
+  tsdbTableSetSchema(&tCfg, schema, true);
-  // Write some data
 
-  // int32_t size = sizeof(SSubmitMsg) + sizeof(SSubmitBlock) + tdMaxRowDataBytes(config.schema) * 10 + sizeof(int32_t);
+  tsdbCreateTable(pRepo, &tCfg);
 
-  // tdUpdateSchema(config.schema);
+  // // 3. Loop to write some simple data
+  int nRows = 10;
+  SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * nRows);
 
-  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(size);
+  SSubmitBlk *pBlock = pMsg->blocks;
-  // pMsg->numOfTables = 1;  // TODO: use api
+  pBlock->tableId = {.uid = 987607499877672L, .tid = 0};
+  pBlock->sversion = 0;
+  pBlock->len = 0;
+  int64_t start_time = 1584081000000;
+  for (int i = 0; i < nRows; i++) {
+    int64_t ttime = start_time + 1000 * i;
+    SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
+    tdInitDataRow(row, schema);
 
-  // SSubmitBlock *pBlock = (SSubmitBlock *)pMsg->data;
+    for (int j = 0; j < schemaNCols(schema); j++) {
-  // pBlock->tableId = {.uid = 98868728187539L, .tid = 0};
+      if (j == 0) { // Just for timestamp
-  // pBlock->sversion = 0;
+        tdAppendColVal(row, (void *)(&time), schemaColAt(schema, j));
-  // pBlock->len = sizeof(SSubmitBlock);
+      } else { // For int
+        int val = 10;
+        tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
+      }
 
-  // SDataRows rows = pBlock->data;
+    }
-  // dataRowsInit(rows);
+    pBlock->len += dataRowLen(row);
 
-  // SDataRow row = tdNewDataRow(tdMaxRowDataBytes(config.schema));
+  }
-  // int64_t ttime = 1583508800000;
+  pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
-  // for (int i = 0; i < 10; i++) {  // loop over rows
-  //   ttime += (10000 * i);
-  //   tdDataRowReset(row);
-  //   for (int j = 0; j < schemaNCols(config.schema); j++) {
-  //     if (j == 0) {  // set time stamp
-  //       tdAppendColVal(row, (void *)(&ttime), schemaColAt(config.schema, j), 40);
-  //     } else {       // set other fields
-  //       int32_t val = 10;
-  //       tdAppendColVal(row, (void *)(&val), schemaColAt(config.schema, j), 40);
-  //     }
-  //   }
 
-  //   tdDataRowsAppendRow(rows, row);
+  tsdbInsertData(pRepo, pMsg);
-  // }
+}
 
-  // tsdbInsertData(pRepo, pMsg);
-
-  // tdFreeDataRow(row);
-
-  tdFreeSchema(config.schema);
-  tsdbDropRepo(pRepo);
-}