[td-186] merge develop branch

.travis.yml

@@ -32,15 +32,15 @@ matrix:
       - cd debug
 
     script:
-      - cmake ..
+      - cmake .. > /dev/null
-      - make
+      - make > /dev/null
 
     after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
           cd ${TRAVIS_BUILD_DIR}/debug
-          make install || travis_terminate $?
+          make install > /dev/null || travis_terminate $?
 
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@@ -98,11 +98,11 @@ matrix:
 
         # Commands to prepare for build_command
         # ** likely specific to your build **
-        build_command_prepend: cmake .
+        build_command_prepend: cmake . > /dev/null
 
         # The command that will be added as an argument to "cov-build" to compile your project for analysis,
         # ** likely specific to your build **
-        build_command: make
+        build_command: make > /dev/null
 
         # Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'.
         # Take care in resource usage, and consider the build frequency allowances per
@@ -135,15 +135,15 @@ matrix:
       - cd debug
 
     script:
-      - cmake -DCOVER=true ..
+      - cmake -DCOVER=true .. > /dev/null
-      - make
+      - make > /dev/null
 
     after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
           cd ${TRAVIS_BUILD_DIR}/debug
-          make install || travis_terminate $?
+          make install > /dev/null || travis_terminate $?
 
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@@ -208,8 +208,8 @@ matrix:
       - cd debug
 
     script:
-      - cmake ..
+      - cmake .. > /dev/null
-      - make
+      - make > /dev/null
 
         #  - os: osx
         #    language: c
@@ -225,5 +225,5 @@ matrix:
         #      - cd ${TRAVIS_BUILD_DIR}
         #      - mkdir debug
         #      - cd debug
-        #      - cmake ..
+        #      - cmake .. > /dev/null
-        #      - make
+        #      - make > /dev/null

src/client/src/tscSQLParser.c

@@ -303,11 +303,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
       }
 
-//      SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0];
+      SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0];
-//      if (!validateIpAddress(pIpAddr->z, pIpAddr->n)) {
+      pIpAddr->n = strdequote(pIpAddr->z);
-//        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
-//      }
-
       break;
     }
 

src/client/src/tscServer.c

@@ -79,15 +79,18 @@ void tscSetMgmtIpListFromEdge() {
   if (tscMgmtIpSet.numOfIps != 1) {
     tscMgmtIpSet.numOfIps = 1;
     tscMgmtIpSet.inUse = 0;
-    taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+    taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
     tscTrace("edge mgmt IP list:");
     tscPrintMgmtIp();
   }
 }
 
 void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
-  tscTrace("mgmt IP list is changed for ufp is called");
   tscMgmtIpSet = *pIpSet;
+  tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse);
+  for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
+    tscTrace("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
+  }
 }
 
 void tscSetMgmtIpList(SRpcIpSet *pIpList) {

src/client/src/tscSql.c

@@ -80,8 +80,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     strcpy(tscMgmtIpSet.fqdn[0], ip);
     tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort;
   } else {
-    if (tsMaster[0] != 0) {
+    if (tsFirst[0] != 0) {
-      taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+      taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
       tscMgmtIpSet.numOfIps++;
     }
 

src/client/src/tscSystem.c

@@ -136,9 +136,9 @@ void taos_init_imp() {
 
   tscMgmtIpSet.inUse = 0;
   tscMgmtIpSet.numOfIps = 1;
-  taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+  taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
 
-  if (tsSecond[0] && strcmp(tsSecond, tsMaster) != 0) {
+  if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) {
     tscMgmtIpSet.numOfIps = 2;
     taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
   }

src/client/src/tscUtil.c

@@ -591,47 +591,44 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
   return TSDB_CODE_SUCCESS;
 }
 
-static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
+static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
-  int32_t firstPartLen = 0;
+  // TODO: optimize this function
+  int len = 0;
 
-  STableMeta* pTableMeta = pTableDataBlock->pTableMeta;
+  STableMeta*   pTableMeta = pTableDataBlock->pTableMeta;
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
-  SSchema* pSchema = tscGetTableSchema(pTableMeta);
+  SSchema*      pSchema = tscGetTableSchema(pTableMeta);
 
+  SSubmitBlk* pBlock = pDataBlock;
   memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
   pDataBlock += sizeof(SSubmitBlk);
 
-  int32_t total = sizeof(int32_t)*2;
+  int32_t flen = 0;
-  for(int32_t i = 0; i < tinfo.numOfColumns; ++i) {
+  for (int32_t i = 0; i < tinfo.numOfColumns; ++i) {
-    switch (pSchema[i].type) {
+    flen += TYPE_BYTES[pSchema[i].type];
-      case TSDB_DATA_TYPE_NCHAR:
-      case TSDB_DATA_TYPE_BINARY: {
-        assert(0);  // not support binary yet
-        firstPartLen += sizeof(int32_t);break;
-      }
-      default:
-        firstPartLen += tDataTypeDesc[pSchema[i].type].nSize;
-        total += tDataTypeDesc[pSchema[i].type].nSize;
-    }
   }
 
   char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
+  pBlock->len = 0;
+  for (int32_t i = 0; i < htons(pBlock->numOfRows); ++i) {
+    SDataRow trow = (SDataRow)pDataBlock;
+    dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen);
 
-  SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData;
+    int toffset = 0;
-  int32_t rows = htons(pBlock->numOfRows);
+    for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
+      tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset);
+      toffset += TYPE_BYTES[pSchema[j].type];
+      p += pSchema[j].bytes;
+    }
 
-  for(int32_t i = 0; i < rows; ++i) {
+    // p += pTableDataBlock->rowSize;
-    *(int32_t*) pDataBlock = total;
+    pDataBlock += dataRowLen(trow);
-    pDataBlock += sizeof(int32_t);
+    pBlock->len += dataRowLen(trow);
-    
-    *(int32_t*) pDataBlock = firstPartLen;
-    pDataBlock += sizeof(int32_t);
-    
-    memcpy(pDataBlock, p, pTableDataBlock->rowSize);
-    
-    p += pTableDataBlock->rowSize;
-    pDataBlock += pTableDataBlock->rowSize;
   }
+
+  len = pBlock->len;
+  pBlock->len = htonl(pBlock->len);
+  return len;
 }
 
 int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
@@ -694,7 +691,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
     pBlocks->len = htonl(len);
     
     // erase the empty space reserved for binary data
-    trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
+    len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
     dataBuf->size += (len + sizeof(SSubmitBlk));
     dataBuf->numOfTables += 1;
   }

src/common/inc/tdataformat.h

@@ -20,6 +20,7 @@
 #include <string.h>
 
 #include "taosdef.h"
+#include "tutil.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,7 +31,7 @@ typedef struct {
   int8_t  type;    // Column type
   int16_t colId;   // column ID
   int32_t bytes;   // column bytes
-  int32_t offset;  // point offset in a row data
+  int32_t offset;  // point offset in SDataRow after the header part
 } STColumn;
 
 #define colType(col) ((col)->type)
@@ -43,26 +44,25 @@ typedef struct {
 #define colSetBytes(col, b) (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
 
-STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes);
-void      tdFreeCol(STColumn *pCol);
-void      tdColCpy(STColumn *dst, STColumn *src);
-void      tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes);
-
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
+  int      totalCols;  // Total columns allocated
   int      numOfCols;  // Number of columns appended
-  int      padding;  // Total columns allocated
+  int      tlen;       // maximum length of a SDataRow without the header part
+  int      flen;       // First part length in a SDataRow after the header part
   STColumn columns[];
 } STSchema;
 
 #define schemaNCols(s) ((s)->numOfCols)
+#define schemaTotalCols(s) ((s)->totalCols)
+#define schemaTLen(s) ((s)->tlen)
+#define schemaFLen(s) ((s)->flen)
 #define schemaColAt(s, i) ((s)->columns + i)
 
 STSchema *tdNewSchema(int32_t nCols);
-int       tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
+#define   tdFreeSchema(s) tfree((s))
+int       tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
 STSchema *tdDupSchema(STSchema *pSchema);
-void      tdFreeSchema(STSchema *pSchema);
-void      tdUpdateSchema(STSchema *pSchema);
 int       tdGetSchemaEncodeSize(STSchema *pSchema);
 void *    tdEncodeSchema(void *dst, STSchema *pSchema);
 STSchema *tdDecodeSchema(void **psrc);
@@ -70,53 +70,100 @@ STSchema *tdDecodeSchema(void **psrc);
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * +----------+---------+---------------------------------+---------------------------------+
+ * |<------------------------------------- len ---------------------------------->|
- * | int32_t  | int32_t |                                 |                                 |
+ * |<--Head ->|<---------   flen -------------->|                                 |
- * +----------+---------+---------------------------------+---------------------------------+
+ * +----------+---------------------------------+---------------------------------+
- * |   len    |  flen   |           First part            |             Second part         |
+ * | int32_t  |                                 |                                 |
- * +----------+---------+---------------------------------+---------------------------------+
+ * +----------+---------------------------------+---------------------------------+
- * plen: first part length
+ * |   len    |           First part            |             Second part         |
- * len: the length including sizeof(row) + sizeof(len)
+ * +----------+---------------------------------+---------------------------------+
- * row: actual row data encoding
  */
 typedef void *SDataRow;
 
-
+#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
-#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
 
 #define dataRowLen(r) (*(int32_t *)(r))
-#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
+#define dataRowTuple(r) POINTER_DRIFT(r, TD_DATA_ROW_HEAD_SIZE)
-#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
 #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
-#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
-#define dataRowIdx(r, i) ((char *)(r) + i)
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
-#define dataRowAt(r, idx) ((char *)(r) + (idx))
+#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
 
-void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-int      tdMaxRowBytesFromSchema(STSchema *pSchema);
-SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema);
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
-int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
+void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-void     tdDataRowReset(SDataRow row, STSchema *pSchema);
+int      tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
 SDataRow tdDataRowDup(SDataRow row);
 
+// NOTE: offset here including the header size
+static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset));
+      break;
+    default:
+      return POINTER_DRIFT(row, offset);
+      break;
+  }
+}
+
 // ----------------- Data column structure
 typedef struct SDataCol {
-  int8_t  type;
+  int8_t          type;       // column type
-  int16_t colId;
+  int16_t         colId;      // column ID
-  int     bytes;
+  int             bytes;      // column data bytes defined
-  int     len;
+  int             offset;     // data offset in a SDataRow (including the header size)
-  int     offset;
+  int             spaceSize;  // Total space size for this column
-  void *  pData; // Original data
+  int             len;        // column data length
+  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
+  void *          pData;      // Actual data pointer
 } SDataCol;
 
+static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
+
+void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
+void dataColSetOffset(SDataCol *pCol, int nEle);
+
+bool isNEleNull(SDataCol *pCol, int nEle);
+void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
+
+// Get the data pointer from a column-wised data
+static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
+  switch (pCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]);
+      break;
+
+    default:
+      return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+      break;
+  }
+}
+
+static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
+  ASSERT(rows > 0);
+
+  switch (pDataCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
+      break;
+    default:
+      return TYPE_BYTES[pDataCol->type] * rows;
+  }
+}
+
+
 typedef struct {
   int      maxRowSize;
   int      maxCols;    // max number of columns
   int      maxPoints;  // max number of points
+  int      bufSize;
+
   int      numOfPoints;
   int      numOfCols;  // Total number of cols
   int      sversion;   // TODO: set sversion
@@ -125,7 +172,7 @@ typedef struct {
 } SDataCols;
 
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
-#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
 #define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
 #define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1)
 

src/common/inc/tglobal.h

@@ -51,7 +51,7 @@ extern int32_t tsVersion;
 extern int32_t tscEmbedded;
 extern int64_t tsMsPerDay[2];
 
-extern char  tsMaster[];
+extern char  tsFirst[];
 extern char  tsSecond[];
 extern char  tsLocalEp[];
 extern uint16_t tsServerPort;

src/common/inc/tulog.h

@@ -43,6 +43,8 @@ extern int32_t tscEmbedded;
   }
 #define uPrint(...) \
   { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
+#define uForcePrint(...) \
+  { taosPrintLog("ERROR UTL ", 255, __VA_ARGS__); }
 
 #define pError(...) \
   { taosPrintLog("ERROR APP ", 255, __VA_ARGS__); }  

src/common/src/tdataformat.c

@@ -13,72 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "tdataformat.h"
-#include "tutil.h"
+#include "wchar.h"
-
-static int tdFLenFromSchema(STSchema *pSchema);
-
-/**
- * Create a new STColumn object
- * ASSUMPTIONS: VALID PARAMETERS
- * 
- * @param type column type
- * @param colId column ID
- * @param bytes maximum bytes the col taken
- * 
- * @return a STColumn object on success
- *         NULL for failure
- */
-STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes) {
-  if (!isValidDataType(type, 0)) return NULL;
-
-  STColumn *pCol = (STColumn *)calloc(1, sizeof(STColumn));
-  if (pCol == NULL) return NULL;
-
-  colSetType(pCol, type);
-  colSetColId(pCol, colId);
-  colSetOffset(pCol, -1);
-  switch (type) {
-    case TSDB_DATA_TYPE_BINARY:
-    case TSDB_DATA_TYPE_NCHAR:
-      colSetBytes(pCol, bytes);
-      break;
-    default:
-      colSetBytes(pCol, TYPE_BYTES[type]);
-      break;
-  }
-
-  return pCol;
-}
-
-/**
- * Free a STColumn object CREATED with tdNewCol
- */
-void tdFreeCol(STColumn *pCol) {
-  if (pCol) free(pCol);
-}
-
-/**
- * Copy from source to destinition
- */
-void tdColCpy(STColumn *dst, STColumn *src) { memcpy((void *)dst, (void *)src, sizeof(STColumn)); }
-
-/**
- * Set the column
- */
-void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
-  colSetType(pCol, type);
-  colSetColId(pCol, colId);
-  switch (type)
-  {
-  case TSDB_DATA_TYPE_BINARY:
-  case TSDB_DATA_TYPE_NCHAR:
-    colSetBytes(pCol, bytes);
-    break;
-  default:
-    colSetBytes(pCol, TYPE_BYTES[type]);
-    break;
-  }
-}
 
 /**
  * Create a SSchema object with nCols columns
@@ -90,11 +25,15 @@ void tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes) {
  *         NULL for failure
  */
 STSchema *tdNewSchema(int32_t nCols) {
-  int32_t  size = sizeof(STSchema) + sizeof(STColumn) * nCols;
+  int32_t size = sizeof(STSchema) + sizeof(STColumn) * nCols;
 
   STSchema *pSchema = (STSchema *)calloc(1, size);
   if (pSchema == NULL) return NULL;
+
   pSchema->numOfCols = 0;
+  pSchema->totalCols = nCols;
+  pSchema->flen = 0;
+  pSchema->tlen = 0;
 
   return pSchema;
 }
@@ -102,25 +41,34 @@ STSchema *tdNewSchema(int32_t nCols) {
 /**
  * Append a column to the schema
  */
-int tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes) {
+int tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes) {
-  // if (pSchema->numOfCols >= pSchema->totalCols) return -1;
+  if (!isValidDataType(type, 0) || pSchema->numOfCols >= pSchema->totalCols) return -1;
-  if (!isValidDataType(type, 0)) return -1;
 
   STColumn *pCol = schemaColAt(pSchema, schemaNCols(pSchema));
   colSetType(pCol, type);
   colSetColId(pCol, colId);
-  colSetOffset(pCol, -1);
+  if (schemaNCols(pSchema) == 0) {
+    colSetOffset(pCol, 0);
+  } else {
+    STColumn *pTCol = schemaColAt(pSchema, schemaNCols(pSchema)-1);
+    colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
+  }
   switch (type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-      colSetBytes(pCol, bytes);
+      colSetBytes(pCol, bytes); // Set as maximum bytes
+      pSchema->tlen += (TYPE_BYTES[type] + sizeof(VarDataLenT) + bytes);
       break;
     default:
       colSetBytes(pCol, TYPE_BYTES[type]);
+      pSchema->tlen += TYPE_BYTES[type];
       break;
   }
 
   pSchema->numOfCols++;
+  pSchema->flen += TYPE_BYTES[type];
+
+  ASSERT(pCol->offset < pSchema->flen);
 
   return 0;
 }
@@ -138,40 +86,22 @@ STSchema *tdDupSchema(STSchema *pSchema) {
   return tSchema;
 }
 
-/**
- * Free the SSchema object created by tdNewSchema or tdDupSchema
- */
-void tdFreeSchema(STSchema *pSchema) {
-  if (pSchema != NULL) free(pSchema);
-}
-
-/**
- * Function to update each columns's offset field in the schema.
- * ASSUMPTIONS: VALID PARAMETERS
- */
-void tdUpdateSchema(STSchema *pSchema) {
-  STColumn *pCol = NULL;
-  int32_t offset = TD_DATA_ROW_HEAD_SIZE;
-  for (int i = 0; i < schemaNCols(pSchema); i++) {
-    pCol = schemaColAt(pSchema, i);
-    colSetOffset(pCol, offset);
-    offset += TYPE_BYTES[pCol->type];
-  }
-}
-
 /**
  * Return the size of encoded schema
  */
 int tdGetSchemaEncodeSize(STSchema *pSchema) {
-  return sizeof(STSchema) + schemaNCols(pSchema) * (T_MEMBER_SIZE(STColumn, type) + T_MEMBER_SIZE(STColumn, colId) +
+  return T_MEMBER_SIZE(STSchema, totalCols) +
-                                                    T_MEMBER_SIZE(STColumn, bytes));
+         schemaNCols(pSchema) *
+             (T_MEMBER_SIZE(STColumn, type) + T_MEMBER_SIZE(STColumn, colId) + T_MEMBER_SIZE(STColumn, bytes));
 }
 
 /**
  * Encode a schema to dst, and return the next pointer
  */
 void *tdEncodeSchema(void *dst, STSchema *pSchema) {
-  T_APPEND_MEMBER(dst, pSchema, STSchema, numOfCols);
+  ASSERT(pSchema->numOfCols == pSchema->totalCols);
+
+  T_APPEND_MEMBER(dst, pSchema, STSchema, totalCols);
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     T_APPEND_MEMBER(dst, pCol, STColumn, type);
@@ -186,13 +116,13 @@ void *tdEncodeSchema(void *dst, STSchema *pSchema) {
  * Decode a schema from a binary.
  */
 STSchema *tdDecodeSchema(void **psrc) {
-  int numOfCols = 0;
+  int totalCols = 0;
 
-  T_READ_MEMBER(*psrc, int, numOfCols);
+  T_READ_MEMBER(*psrc, int, totalCols);
 
-  STSchema *pSchema = tdNewSchema(numOfCols);
+  STSchema *pSchema = tdNewSchema(totalCols);
   if (pSchema == NULL) return NULL;
-  for (int i = 0; i < numOfCols; i++) {
+  for (int i = 0; i < totalCols; i++) {
     int8_t  type = 0;
     int16_t colId = 0;
     int32_t bytes = 0;
@@ -200,7 +130,7 @@ STSchema *tdDecodeSchema(void **psrc) {
     T_READ_MEMBER(*psrc, int16_t, colId);
     T_READ_MEMBER(*psrc, int32_t, bytes);
 
-    tdSchemaAppendCol(pSchema, type, colId, bytes);
+    tdSchemaAddCol(pSchema, type, colId, bytes);
   }
 
   return pSchema;
@@ -209,53 +139,18 @@ STSchema *tdDecodeSchema(void **psrc) {
 /**
  * Initialize a data row
  */
-void tdInitDataRow(SDataRow row, STSchema *pSchema) {
+void tdInitDataRow(SDataRow row, STSchema *pSchema) { dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); }
-  dataRowSetFLen(row, TD_DATA_ROW_HEAD_SIZE);
-  dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + tdFLenFromSchema(pSchema));
-}
 
-/**
+SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
- * Create a data row with maximum row length bytes.
+  int32_t size = dataRowMaxBytesFromSchema(pSchema);
- *
- * NOTE: THE AAPLICATION SHOULD MAKE SURE BYTES IS LARGE ENOUGH TO
- * HOLD THE WHOE ROW.
- *
- * @param bytes max bytes a row can take
- * @return SDataRow object for success
- *         NULL for failure
- */
-SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema) {
-  int32_t size = sizeof(int32_t) + bytes;
 
   SDataRow row = malloc(size);
   if (row == NULL) return NULL;
 
   tdInitDataRow(row, pSchema);
-
   return row;
 }
 
-/**
- * Get maximum bytes a data row from a schema
- * ASSUMPTIONS: VALID PARAMETER
- */
-int tdMaxRowBytesFromSchema(STSchema *pSchema) {
-  // TODO
-  int bytes = TD_DATA_ROW_HEAD_SIZE;
-  for (int i = 0; i < schemaNCols(pSchema); i++) {
-    STColumn *pCol = schemaColAt(pSchema, i);
-    bytes += TYPE_BYTES[pCol->type];
-
-    if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
-      bytes += pCol->bytes;
-    }
-  }
-
-  return bytes;
-}
-
-SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { return tdNewDataRow(tdMaxRowBytesFromSchema(pSchema), pSchema); }
-
 /**
  * Free the SDataRow object
  */
@@ -265,28 +160,49 @@ void tdFreeDataRow(SDataRow row) {
 
 /**
  * Append a column value to the data row
+ * @param type: column type
+ * @param bytes: column bytes
+ * @param offset: offset in the data row tuple, not including the data row header
  */
-int tdAppendColVal(SDataRow row, void *value, STColumn *pCol) {
+int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
-  switch (colType(pCol))
+  ASSERT(value != NULL);
-  {
+  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-  case TSDB_DATA_TYPE_BINARY:
+  char *  ptr = POINTER_DRIFT(row, dataRowLen(row));
-  case TSDB_DATA_TYPE_NCHAR:
+
-    *(int32_t *)dataRowAt(row, dataRowFLen(row)) = dataRowLen(row);
+  switch (type) {
-    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+    case TSDB_DATA_TYPE_BINARY:
-    memcpy((void *)dataRowAt(row, dataRowLen(row)), value, strlen(value));
+    case TSDB_DATA_TYPE_NCHAR:
-    dataRowLen(row) += strlen(value);
+      // set offset
-    break;
+      *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row);
-  default:
+
-    memcpy(dataRowAt(row, dataRowFLen(row)), value, TYPE_BYTES[colType(pCol)]);
+      // set length
-    dataRowFLen(row) += TYPE_BYTES[colType(pCol)];
+      VarDataLenT slen = 0;
-    break;
+      if (isNull(value, type)) {
+        slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t);
+      } else {
+        if (type == TSDB_DATA_TYPE_BINARY) {
+          slen = strnlen((char *)value, bytes);
+        } else {
+          slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE;
+        }
+      }
+
+      ASSERT(slen <= bytes);
+      *(VarDataLenT *)ptr = slen;
+      ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT));
+
+      memcpy((void *)ptr, value, slen);
+      dataRowLen(row) += (sizeof(int16_t) + slen);
+
+      break;
+    default:
+      memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]);
+      break;
   }
 
   return 0;
 }
 
-void tdDataRowReset(SDataRow row, STSchema *pSchema) { tdInitDataRow(row, pSchema); }
-
 SDataRow tdDataRowDup(SDataRow row) {
   SDataRow trow = malloc(dataRowLen(row));
   if (trow == NULL) return NULL;
@@ -295,6 +211,119 @@ SDataRow tdDataRowDup(SDataRow row) {
   return trow;
 }
 
+void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) {
+  pDataCol->type = colType(pCol);
+  pDataCol->colId = colColId(pCol);
+  pDataCol->bytes = colBytes(pCol);
+  pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
+
+  pDataCol->len = 0;
+  if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+    pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints;
+    pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
+    pDataCol->pData = POINTER_DRIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints);
+    *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints);
+  } else {
+    pDataCol->spaceSize = pDataCol->bytes * maxPoints;
+    pDataCol->dataOff = NULL;
+    pDataCol->pData = *pBuf;
+    *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize);
+  }
+
+}
+
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints) {
+  ASSERT(pCol != NULL && value != NULL);
+
+  switch (pCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      // set offset
+      pCol->dataOff[numOfPoints] = pCol->len;
+      // Copy data
+      memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+      // Update the length
+      pCol->len += varDataTLen(value);
+      break;
+    default:
+      ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+      memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, pCol->bytes);
+      pCol->len += pCol->bytes;
+      break;
+  }
+}
+
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) {
+  int pointsLeft = numOfPoints - pointsToPop;
+
+  ASSERT(pointsLeft > 0);
+
+  if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
+    ASSERT(pCol->len > 0);
+    VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
+    pCol->len = pCol->len - toffset;
+    ASSERT(pCol->len > 0);
+    memmove(pCol->pData, POINTER_DRIFT(pCol->pData, toffset), pCol->len);
+    dataColSetOffset(pCol, pointsLeft);
+  } else {
+    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints);
+    pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
+    memmove(pCol->pData, POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
+  }
+}
+
+bool isNEleNull(SDataCol *pCol, int nEle) {
+  switch (pCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      for (int i = 0; i < nEle; i++) {
+        if (!isNull(varDataVal(tdGetColDataOfRow(pCol, i)), pCol->type)) return false;
+      }
+      return true;
+    default:
+      for (int i = 0; i < nEle; i++) {
+        if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
+      }
+      return true;
+  }
+}
+
+void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
+  char *ptr = NULL;
+  switch (pCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      pCol->len = 0;
+      for (int i = 0; i < nEle; i++) {
+        pCol->dataOff[i] = pCol->len;
+        ptr = (char *)pCol->pData + pCol->len;
+        varDataLen(ptr) = (pCol->type == TSDB_DATA_TYPE_BINARY) ? sizeof(char) : TSDB_NCHAR_SIZE;
+        setNull(ptr + sizeof(VarDataLenT), pCol->type, pCol->bytes);
+        pCol->len += varDataTLen(ptr);
+      }
+
+      break;
+    default:
+      setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
+      pCol->len = TYPE_BYTES[pCol->type] * nEle;
+      break;
+  }
+}
+
+void dataColSetOffset(SDataCol *pCol, int nEle) {
+  ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
+
+  void * tptr = pCol->pData;
+  // char *tptr = (char *)(pCol->pData);
+
+  VarDataOffsetT offset = 0;
+  for (int i = 0; i < nEle; i++) {
+    pCol->dataOff[i] = offset;
+    offset += varDataTLen(tptr);
+    tptr = POINTER_DRIFT(tptr, varDataTLen(tptr));
+  }
+}
+
 SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
   SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols);
   if (pCols == NULL) return NULL;
@@ -302,8 +331,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
   pCols->maxRowSize = maxRowSize;
   pCols->maxCols = maxCols;
   pCols->maxPoints = maxRows;
+  pCols->bufSize = maxRowSize * maxRows;
 
-  pCols->buf = malloc(maxRowSize * maxRows);
+  pCols->buf = malloc(pCols->bufSize);
   if (pCols->buf == NULL) {
     free(pCols);
     return NULL;
@@ -317,24 +347,16 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   tdResetDataCols(pCols);
   pCols->numOfCols = schemaNCols(pSchema);
 
-  pCols->cols[0].pData = pCols->buf;
+  void *ptr = pCols->buf;
-  int offset = TD_DATA_ROW_HEAD_SIZE;
   for (int i = 0; i < schemaNCols(pSchema); i++) {
-    if (i > 0) {
+    dataColInit(pCols->cols + i, schemaColAt(pSchema, i), &ptr, pCols->maxPoints);
-      pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints;
+    ASSERT((char *)ptr - (char *)(pCols->buf) <= pCols->bufSize);
-    }
-    pCols->cols[i].type = colType(schemaColAt(pSchema, i));
-    pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i));
-    pCols->cols[i].offset = offset;
-    pCols->cols[i].colId = colColId(schemaColAt(pSchema, i));
-
-    offset += TYPE_BYTES[pCols->cols[i].type];
   }
 }
 
 void tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
-    if (pCols->buf) free(pCols->buf);
+    tfree(pCols->buf);
     free(pCols);
   }
 }
@@ -351,11 +373,24 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
     pRet->cols[i].type = pDataCols->cols[i].type;
     pRet->cols[i].colId = pDataCols->cols[i].colId;
     pRet->cols[i].bytes = pDataCols->cols[i].bytes;
-    pRet->cols[i].len = pDataCols->cols[i].len;
     pRet->cols[i].offset = pDataCols->cols[i].offset;
+
+    pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
     pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
 
-    if (keepData) memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pRet->cols[i].bytes * pDataCols->numOfPoints);
+    if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+      ASSERT(pDataCols->cols[i].dataOff != NULL);
+      pRet->cols[i].dataOff =
+          (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
+    }
+
+    if (keepData) {
+      pRet->cols[i].len = pDataCols->cols[i].len;
+      memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
+      if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) {
+        memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
+      }
+    }
   }
 
   return pRet;
@@ -364,57 +399,60 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 void tdResetDataCols(SDataCols *pCols) {
   pCols->numOfPoints = 0;
   for (int i = 0; i < pCols->maxCols; i++) {
-    pCols->cols[i].len = 0;
+    dataColReset(pCols->cols + i);
   }
 }
 
 void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) {
+  ASSERT(dataColsKeyLast(pCols) < dataRowKey(row));
+
   for (int i = 0; i < pCols->numOfCols; i++) {
     SDataCol *pCol = pCols->cols + i;
-    memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes);
+    void *    value = tdGetRowDataOfCol(row, pCol->type, pCol->offset);
-    pCol->len += pCol->bytes;
+
+    dataColAppendVal(pCol, value, pCols->numOfPoints, pCols->maxPoints);
   }
   pCols->numOfPoints++;
 }
+
 // Pop pointsToPop points from the SDataCols
 void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
   int pointsLeft = pCols->numOfPoints - pointsToPop;
+  if (pointsLeft <= 0) {
+    tdResetDataCols(pCols);
+    return;
+  }
 
   for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
-    SDataCol *p_col = pCols->cols + iCol;
+    SDataCol *pCol = pCols->cols + iCol;
-    if (p_col->len > 0) {
+    dataColPopPoints(pCol, pointsToPop, pCols->numOfPoints);
-      p_col->len = TYPE_BYTES[p_col->type] * pointsLeft;
-      if (pointsLeft > 0) {
-        memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len);
-      }
-    }
   }
   pCols->numOfPoints = pointsLeft;
 }
 
-/**
- * Return the first part length of a data row for a schema
- */
-static int tdFLenFromSchema(STSchema *pSchema) {
-  int ret = 0;
-  for (int i = 0; i < schemaNCols(pSchema); i++) {
-    STColumn *pCol = schemaColAt(pSchema, i);
-    ret += TYPE_BYTES[pCol->type];
-  }
-
-  return ret;
-}
-
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfPoints);
+  ASSERT(target->numOfPoints + rowsToMerge <= target->maxPoints);
+  ASSERT(target->numOfCols == source->numOfCols);
 
-  SDataCols *pTarget = tdDupDataCols(target, true);
+  SDataCols *pTarget = NULL;
-  if (pTarget == NULL) goto _err;
-  // tdResetDataCols(target);
 
-  int iter1 = 0;
+  if (dataColsKeyLast(target) < dataColsKeyFirst(source)) {  // No overlap
-  int iter2 = 0;
+    for (int i = 0; i < rowsToMerge; i++) {
-  tdMergeTwoDataCols(target,pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
+      for (int j = 0; j < source->numOfCols; j++) {
+        dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i), target->numOfPoints,
+                         target->maxPoints);
+      }
+    }
+    target->numOfPoints++;
+  } else {
+    pTarget = tdDupDataCols(target, true);
+    if (pTarget == NULL) goto _err;
+
+    int iter1 = 0;
+    int iter2 = 0;
+    tdMergeTwoDataCols(target, pTarget, &iter1, source, &iter2, pTarget->numOfPoints + rowsToMerge);
+  }
 
   tdFreeDataCols(pTarget);
   return 0;
@@ -425,6 +463,7 @@ _err:
 }
 
 void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows) {
+  // TODO: add resolve duplicate key here
   tdResetDataCols(target);
 
   while (target->numOfPoints < tRows) {
@@ -436,10 +475,8 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
     if (key1 < key2) {
       for (int i = 0; i < src1->numOfCols; i++) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
-        memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
+        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfPoints,
-               (void *)((char *)(src1->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * (*iter1)),
+                         target->maxPoints);
-               TYPE_BYTES[target->cols[i].type]);
-        target->cols[i].len += TYPE_BYTES[target->cols[i].type];
       }
 
       target->numOfPoints++;
@@ -447,15 +484,14 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCol
     } else if (key1 > key2) {
       for (int i = 0; i < src2->numOfCols; i++) {
         ASSERT(target->cols[i].type == src2->cols[i].type);
-        memcpy((void *)((char *)(target->cols[i].pData) + TYPE_BYTES[target->cols[i].type] * target->numOfPoints),
+        dataColAppendVal(target->cols[i].pData, tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfPoints,
-               (void *)((char *)(src2->cols[i].pData) + TYPE_BYTES[src2->cols[i].type] * (*iter2)),
+                         target->maxPoints);
-               TYPE_BYTES[target->cols[i].type]);
-        target->cols[i].len += TYPE_BYTES[target->cols[i].type];
       }
 
       target->numOfPoints++;
       (*iter2)++;
     } else {
+      // TODO: deal with duplicate keys
       ASSERT(false);
     }
   }

src/common/src/tglobal.c

@@ -25,6 +25,7 @@
 #include "tutil.h"
 #include "tlocale.h"
 #include "ttimezone.h"
+#include "tsync.h"
 
 char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
 char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
@@ -60,8 +61,9 @@ int32_t tscEmbedded = 0;
  */
 int64_t tsMsPerDay[] = {86400000L, 86400000000L};
 
-char  tsMaster[TSDB_FQDN_LEN] = {0};
+char  tsFirst[TSDB_FQDN_LEN] = {0};
 char  tsSecond[TSDB_FQDN_LEN] = {0};
+char  tsArbitrator[TSDB_FQDN_LEN] = {0};
 char  tsLocalEp[TSDB_FQDN_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 uint16_t tsMnodeShellPort = 6030;   // udp[6030-6034] tcp[6030]
@@ -141,6 +143,7 @@ int32_t qdebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
+int32_t sDebugFlag = 131;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -269,9 +272,9 @@ static void doInitGlobalConfig() {
   SGlobalCfg cfg = {0};
   
   // ip address
-  cfg.option = "master";
+  cfg.option = "first";
-  cfg.ptr = tsMaster;
+  cfg.ptr = tsFirst;
-  cfg.valType = TAOS_CFG_VTYPE_IPSTR;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -281,7 +284,7 @@ static void doInitGlobalConfig() {
 
   cfg.option = "second";
   cfg.ptr = tsSecond;
-  cfg.valType = TAOS_CFG_VTYPE_IPSTR;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -341,6 +344,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "arbitrator";
+  cfg.ptr = tsArbitrator;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // dnode configs
   cfg.option = "numOfThreadsPerCore";
   cfg.ptr = &tsNumOfThreadsPerCore;
@@ -1012,6 +1025,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "sDebugFlag";
+  cfg.ptr = &sDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "sdbDebugFlag";
   cfg.ptr = &sdbDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1181,9 +1204,10 @@ void taosInitGlobalCfg() {
 bool taosCheckGlobalCfg() {
   taosGetFqdn(tsLocalEp);
   sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
+  uPrint("localEp is %s", tsLocalEp);
 
-  if (tsMaster[0] == 0) {
+  if (tsFirst[0] == 0) {
-    strcpy(tsMaster, tsLocalEp);
+    strcpy(tsFirst, tsLocalEp);
   }
 
   if (tsSecond[0] == 0) {

src/common/src/tlocale.c

@@ -27,27 +27,20 @@
  * In case that the setLocale failed to be executed, the right charset needs to be set.
  */
 void tsSetLocale() {
-  char msgLocale[] = "Invalid locale:%s, please set the valid locale in config file\n";
-  char msgCharset[] = "Invalid charset:%s, please set the valid charset in config file\n";
-  char msgCharset1[] = "failed to get charset, please set the valid charset in config file\n";
-
   char *locale = setlocale(LC_CTYPE, tsLocale);
 
   // default locale or user specified locale is not valid, abort launch
   if (locale == NULL) {
-    printf(msgLocale, tsLocale);
+    uForcePrint("Invalid locale:%s, please set the valid locale in config file", tsLocale);
-    uPrint(msgLocale, tsLocale);
   }
 
   if (strlen(tsCharset) == 0) {
-    printf("%s\n", msgCharset1);
+    uForcePrint("failed to get charset, please set the valid charset in config file");
-    uPrint(msgCharset1);
     exit(-1);
   }
 
   if (!taosValidateEncodec(tsCharset)) {
-    printf(msgCharset, tsCharset);
+    uForcePrint("Invalid charset:%s, please set the valid charset in config file", tsCharset);
-    uPrint(msgCharset, tsCharset);
     exit(-1);
   }
 }

src/common/src/ttypes.c

@@ -16,33 +16,34 @@
 
 #include "taosdef.h"
 #include "ttokendef.h"
+#include "tscompression.h"
 
 const int32_t TYPE_BYTES[11] = {
-    -1,               // TSDB_DATA_TYPE_NULL
+    -1,                      // TSDB_DATA_TYPE_NULL
-    sizeof(int8_t),   // TSDB_DATA_TYPE_BOOL
+    sizeof(int8_t),          // TSDB_DATA_TYPE_BOOL
-    sizeof(int8_t),   // TSDB_DATA_TYPE_TINYINT
+    sizeof(int8_t),          // TSDB_DATA_TYPE_TINYINT
-    sizeof(int16_t),  // TSDB_DATA_TYPE_SMALLINT
+    sizeof(int16_t),         // TSDB_DATA_TYPE_SMALLINT
-    sizeof(int32_t),  // TSDB_DATA_TYPE_INT
+    sizeof(int32_t),         // TSDB_DATA_TYPE_INT
-    sizeof(int64_t),  // TSDB_DATA_TYPE_BIGINT
+    sizeof(int64_t),         // TSDB_DATA_TYPE_BIGINT
-    sizeof(float),    // TSDB_DATA_TYPE_FLOAT
+    sizeof(float),           // TSDB_DATA_TYPE_FLOAT
-    sizeof(double),   // TSDB_DATA_TYPE_DOUBLE
+    sizeof(double),          // TSDB_DATA_TYPE_DOUBLE
-    sizeof(int32_t),  // TSDB_DATA_TYPE_BINARY
+    sizeof(VarDataOffsetT),  // TSDB_DATA_TYPE_BINARY
-    sizeof(TSKEY),    // TSDB_DATA_TYPE_TIMESTAMP
+    sizeof(TSKEY),           // TSDB_DATA_TYPE_TIMESTAMP
-    sizeof(int32_t)   // TSDB_DATA_TYPE_NCHAR
+    sizeof(VarDataOffsetT)   // TSDB_DATA_TYPE_NCHAR
 };
 
 tDataTypeDescriptor tDataTypeDesc[11] = {
-  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE"},
+  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL"},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool},
-  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT"},
+  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint},
-  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT"},
+  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint},
-  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT"},
+  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt},
-  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT"},
+  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint},
-  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT"},
+  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat},
-  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE"},
+  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble},
-  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY"},
+  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP"},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp},
-  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR"},
+  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString},
 };
 
 char tTokenTypeSwitcher[13] = {

src/dnode/src/dnodeMClient.c

@@ -83,17 +83,19 @@ int32_t dnodeInitMClient() {
     memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
     memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
     tsMnodeIpSet.numOfIps = 1;
-    taosGetFqdnPortFromEp(tsMaster, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
+    taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
     tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE;
-    if (strcmp(tsSecond, tsMaster) != 0) {
+    if (strcmp(tsSecond, tsFirst) != 0) {
       tsMnodeIpSet.numOfIps = 2;
       taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
+      tsMnodeIpSet.port[1] += TSDB_PORT_MNODEDNODE;
     }
   } else {
     tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
     tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
     for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
       taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
+      tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
     }
   }
 

src/dnode/src/dnodeMain.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "taos.h"
 #include "tglobal.h"
 #include "trpc.h"
 #include "tutil.h"
@@ -190,6 +191,7 @@ static void dnodeCleanUpSystem() {
     dnodeCleanupWrite();
     dnodeCleanupRead();
     dnodeCleanUpModules();
+    taos_cleanup();
     dnodeCleanupStorage();
     taosCloseLog();
   }
@@ -236,5 +238,5 @@ static int32_t dnodeInitStorage() {
 static void dnodeCleanupStorage() {}
 
 bool  dnodeIsFirstDeploy() {
-  return strcmp(tsMaster, tsLocalEp) == 0;
+  return strcmp(tsFirst, tsLocalEp) == 0;
 }

src/inc/taosdef.h

@@ -32,6 +32,13 @@ extern "C" {
 #define TSKEY int64_t
 #endif
 
+// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
+typedef int32_t VarDataOffsetT;
+typedef int16_t VarDataLenT;
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
+#define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT)))
+
 // this data type is internally used only in 'in' query to hold the values
 #define TSDB_DATA_TYPE_ARRAY      (TSDB_DATA_TYPE_NCHAR + 1)
 
@@ -121,6 +128,10 @@ typedef struct tDataTypeDescriptor {
   int16_t nameLen;
   int32_t nSize;
   char *  aName;
+  int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                  char algorithm, char *const buffer, int bufferSize);
+  int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
+                    int outputSize, char algorithm, char *const buffer, int bufferSize);
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypeDesc[11];
@@ -190,7 +201,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_LOCALE_LEN           64
 #define TSDB_TIMEZONE_LEN         64
 
-#define TSDB_FQDN_LEN             64
+#define TSDB_FQDN_LEN             72
 #define TSDB_IPv4ADDR_LEN      	  16
 #define TSDB_FILENAME_LEN         128
 #define TSDB_METER_VNODE_BITS     20

src/inc/tsdb.h

@@ -104,9 +104,10 @@ void tsdbClearTableCfg(STableCfg *config);
 int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val);
 int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name);
 
-int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
+int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
-int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
+int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
-int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
+int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {

src/inc/tsync.h

@@ -45,8 +45,6 @@ typedef struct {
 typedef struct {
   int8_t     quorum;    // number of confirms required, >=1 
   int8_t     replica;   // number of replications, >=1
-  uint16_t   arbitratorPort;  // arbitrator port
-  char       arbitratorFqdn[TSDB_FQDN_LEN]; // arbitrator IP address
   SNodeInfo  nodeInfo[TAOS_SYNC_MAX_REPLICA];
 } SSyncCfg;
 
@@ -108,6 +106,7 @@ extern  int   tsMaxWatchFiles;
 extern  int   tsSyncTimer;
 extern  int   tsMaxFwdInfo; 
 extern  int   sDebugFlag;
+extern  char  tsArbitrator[];
 extern  uint16_t tsSyncPort;
 
 #ifdef __cplusplus

src/mnode/inc/mgmtDb.h

@@ -32,6 +32,7 @@ int32_t mgmtInitDbs();
 void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
+void *  mgmtGetNextDb(void *pNode, SDbObj **pDb);
 void    mgmtIncDbRef(SDbObj *pDb);
 void    mgmtDecDbRef(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);

src/mnode/inc/mgmtDef.h

@@ -237,7 +237,6 @@ typedef struct {
 
 typedef struct {
   uint8_t  msgType;
-  int8_t   usePublicIp;
   int8_t   received;
   int8_t   successed;
   int8_t   expected;

src/mnode/inc/mgmtMnode.h

@@ -33,16 +33,19 @@ void    mgmtCleanupMnodes();
 
 int32_t mgmtAddMnode(int32_t dnodeId);
 int32_t mgmtDropMnode(int32_t dnodeId);
+void    mgmtDropMnodeLocal(int32_t dnodeId);
 
 void *  mgmtGetMnode(int32_t mnodeId);
 int32_t mgmtGetMnodesNum();
 void *  mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
-void    mgmtReleaseMnode(struct SMnodeObj *pMnode);
+void    mgmtIncMnodeRef(struct SMnodeObj *pMnode);
+void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 
 char *  mgmtGetMnodeRoleStr();
 void    mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
 void    mgmtGetMnodeInfos(void *mnodes);
 
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtProfile.h

@@ -24,14 +24,6 @@ extern "C" {
 int32_t mgmtInitProfile();
 void    mgmtCleanUpProfile();
 
-bool    mgmtCheckQhandle(uint64_t qhandle);
-void    mgmtSaveQhandle(void *qhandle);
-void    mgmtFreeQhandle(void *qhandle);
-
-void *  mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
-void *  mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
-void    mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
-
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtSdb.h

@@ -72,6 +72,7 @@ void    sdbCleanUp();
 void *  sdbOpenTable(SSdbTableDesc *desc);
 void    sdbCloseTable(void *handle);
 bool    sdbIsMaster();
+bool    sdbIsServing();
 void    sdbUpdateMnodeRoles();
 
 int32_t sdbInsertRow(SSdbOper *pOper);

src/mnode/inc/mgmtShell.h

@@ -34,6 +34,14 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
 void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg);
 void mgmtSendSimpleResp(void *thandle, int32_t code);
 
+bool  mgmtCheckQhandle(uint64_t qhandle);
+void *mgmtSaveQhandle(void *qhandle, int32_t size);
+void  mgmtFreeQhandle(void *qhandle, bool forceRemove);
+
+void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
+void *mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
+void  mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtTable.h

@@ -22,13 +22,15 @@ extern "C" {
 
 #include "mgmtDef.h"
 
-int32_t     mgmtInitTables();
+int32_t mgmtInitTables();
-void        mgmtCleanUpTables();
+void    mgmtCleanUpTables();
-STableObj*  mgmtGetTable(char* tableId);
+void *  mgmtGetTable(char *tableId);
-void        mgmtIncTableRef(void *pTable); 
+void    mgmtIncTableRef(void *pTable);
-void        mgmtDecTableRef(void *pTable);
+void    mgmtDecTableRef(void *pTable);
-void        mgmtDropAllChildTables(SDbObj *pDropDb);
+void *  mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable);
-void        mgmtDropAllSuperTables(SDbObj *pDropDb);
+void *  mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable);
+void    mgmtDropAllChildTables(SDbObj *pDropDb);
+void    mgmtDropAllSuperTables(SDbObj *pDropDb);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtUser.h

@@ -27,7 +27,7 @@ SUserObj *mgmtGetUser(char *name);
 void *    mgmtGetNextUser(void *pNode, SUserObj **pUser);
 void      mgmtIncUserRef(SUserObj *pUser);
 void      mgmtDecUserRef(SUserObj *pUser);
-SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
+SUserObj *mgmtGetUserFromConn(void *pConn);
 int32_t   mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
 void      mgmtDropAllUsers(SAcctObj *pAcct);
 

src/mnode/inc/mgmtVgroup.h

@@ -32,7 +32,8 @@ void    mgmtCleanUpVgroups();
 SVgObj *mgmtGetVgroup(int32_t vgId);
 void    mgmtIncVgroupRef(SVgObj *pVgroup);
 void    mgmtDecVgroupRef(SVgObj *pVgroup);
-void    mgmtDropAllVgroups(SDbObj *pDropDb);
+void    mgmtDropAllDbVgroups(SDbObj *pDropDb);
+void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 
 void *  mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);

src/mnode/src/mgmtAcct.c

@@ -27,8 +27,8 @@
 #include "mgmtUser.h"
 
 void *  tsAcctSdb = NULL;
-int32_t tsAcctUpdateSize;
+static int32_t tsAcctUpdateSize;
-static void mgmtCreateRootAcct();
+static void    mgmtCreateRootAcct();
 
 static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;

src/mnode/src/mgmtBalance.c

@@ -35,7 +35,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   void *     pNode = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1.0;
+  float      vnodeUsage = 1000.0;
 
   while (1) {
     pNode = mgmtGetNextDnode(pNode, &pDnode);

src/mnode/src/mgmtDServer.c

@@ -107,6 +107,7 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
     
     SRpcIpSet ipSet = {0};
     dnodeGetMnodeDnodeIpSet(&ipSet);
+    
     mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
       mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);

src/mnode/src/mgmtDb.c

@@ -36,7 +36,7 @@
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-void *  tsDbSdb = NULL;
+static void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
 static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
@@ -82,7 +82,7 @@ static int32_t mgmtDbActionDelete(SSdbOper *pOper) {
   mgmtDropDbFromAcct(pAcct, pDb);
   mgmtDropAllChildTables(pDb);
   mgmtDropAllSuperTables(pDb);
-  mgmtDropAllVgroups(pDb);
+  mgmtDropAllDbVgroups(pDb);
   mgmtDecAcctRef(pAcct);
   
   return TSDB_CODE_SUCCESS;
@@ -95,6 +95,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDb, pOper->rowSize);
     free(pDb);
   }
+  mgmtDecDbRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -154,6 +155,10 @@ int32_t mgmtInitDbs() {
   return 0;
 }
 
+void *mgmtGetNextDb(void *pNode, SDbObj **pDb) {
+  return sdbFetchRow(tsDbSdb, pNode, (void **)pDb);
+}
+
 SDbObj *mgmtGetDb(char *db) {
   return (SDbObj *)sdbGetRow(tsDbSdb, db);
 }
@@ -174,7 +179,7 @@ SDbObj *mgmtGetDbByTableId(char *tableId) {
   memset(db, 0, sizeof(db));
   strncpy(db, tableId, pos - tableId);
 
-  return (SDbObj *)sdbGetRow(tsDbSdb, db);
+  return mgmtGetDb(db);
 }
 
 static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
@@ -346,8 +351,27 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
   return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
 }
 
+#if 0
+void mgmtPrintVgroups(SDbObj *pDb, char *oper) {
+  mPrint("db:%s, vgroup link from head, oper:%s", pDb->name, oper);  
+  SVgObj *pVgroup = pDb->pHead;
+  while (pVgroup != NULL) {
+    mPrint("vgId:%d", pVgroup->vgId);
+    pVgroup = pVgroup->next;
+  }
+
+  mPrint("db:%s, vgroup link from tail", pDb->name, pDb->numOfVgroups);
+  pVgroup = pDb->pTail;
+  while (pVgroup != NULL) {
+    mPrint("vgId:%d", pVgroup->vgId);
+    pVgroup = pVgroup->prev;
+  }
+}
+#endif
+
 void mgmtAddVgroupIntoDb(SVgObj *pVgroup) {
   SDbObj *pDb = pVgroup->pDb;
+
   pVgroup->next = pDb->pHead;
   pVgroup->prev = NULL;
 
@@ -397,7 +421,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
   int32_t cols = 0;
 
   SSchema *pSchema = pMeta->schema;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   pShow->bytes[cols] = TSDB_DB_NAME_LEN;
@@ -545,11 +569,11 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
   SDbObj *pDb = NULL;
   char *  pWrite;
   int32_t cols = 0;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   while (numOfRows < rows) {
-    pShow->pNode = sdbFetchRow(tsDbSdb, pShow->pNode, (void **) &pDb);
+    pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb);
     if (pDb == NULL) break;
 
     cols = 0;
@@ -674,8 +698,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsDbSdb,
-    .pObj = pDb,
+    .pObj = pDb
-    .rowSize = tsDbUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -803,8 +826,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
       .table = tsDbSdb,
-      .pObj = pDb,
+      .pObj = pDb
-      .rowSize = tsDbUpdateSize
     };
 
     int32_t code = sdbUpdateRow(&oper);
@@ -839,21 +861,21 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SDbObj *pDb = pMsg->pDb = mgmtGetDb(pAlter->db);
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pAlter->db);
-  if (pDb == NULL) {
+  if (pMsg->pDb == NULL) {
     mError("db:%s, failed to alter, invalid db", pAlter->db);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
     return;
   }
 
-  int32_t code = mgmtAlterDb(pDb, pAlter);
+  int32_t code = mgmtAlterDb(pMsg->pDb, pAlter);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, failed to alter, invalid db option", pAlter->db);
     mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  mTrace("db:%s, all vgroups is altered", pDb->name);
+  mTrace("db:%s, all vgroups is altered", pMsg->pDb->name);
   mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
 }
 
@@ -884,8 +906,8 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SDbObj *pDb = pMsg->pDb = mgmtGetDb(pDrop->db);
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pDrop->db);
-  if (pDb == NULL) {
+  if (pMsg->pDb == NULL) {
     if (pDrop->ignoreNotExists) {
       mTrace("db:%s, db is not exist, think drop success", pDrop->db);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@@ -897,30 +919,32 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
     }
   }
 
-  if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
+  if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
     mError("db:%s, can't drop monitor database", pDrop->db);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN);
     return;
   }
 
-  int32_t code = mgmtSetDbDropping(pDb);
+  int32_t code = mgmtSetDbDropping(pMsg->pDb);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
     mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  SVgObj *pVgroup = pDb->pHead;
+#if 0
+  SVgObj *pVgroup = pMsg->pDb->pHead;
   if (pVgroup != NULL) {
-    mPrint("vgroup:%d, will be dropped", pVgroup->vgId);
+    mPrint("vgId:%d, will be dropped", pVgroup->vgId);
     SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
     newMsg->ahandle = pVgroup;
     newMsg->expected = pVgroup->numOfVnodes;
     mgmtDropVgroup(pVgroup, newMsg);
     return;
   }
+#endif  
 
-  mTrace("db:%s, all vgroups is dropped", pDb->name);
+  mTrace("db:%s, all vgroups is dropped", pMsg->pDb->name);
   mgmtDropDb(pMsg);
 }
 
@@ -932,7 +956,7 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
   mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user);
 
   while (1) {
-    pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
+    pNode = mgmtGetNextDb(pNode, &pDb);
     if (pDb == NULL) break;
 
     if (pDb->pAcct == pAcct) {

src/mnode/src/mgmtDnode.c

@@ -36,9 +36,9 @@
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-void   *tsDnodeSdb = NULL;
-int32_t tsDnodeUpdateSize = 0;
 int32_t tsAccessSquence = 0;
+static void   *tsDnodeSdb = NULL;
+static int32_t tsDnodeUpdateSize = 0;
 extern void *  tsMnodeSdb;
 extern void *  tsVgroupSdb;
 
@@ -73,39 +73,12 @@ static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
 
 static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
-  void *     pNode = NULL;
-  void *     pLastNode = NULL;
-  SVgObj *   pVgroup = NULL;
-  int32_t    numOfVgroups = 0;
-
-  while (1) {
-    pLastNode = pNode;
-    pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
-    if (pVgroup == NULL) break;
-
-    if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
-      SSdbOper oper = {
-        .type = SDB_OPER_LOCAL,
-        .table = tsVgroupSdb,
-        .pObj = pVgroup,
-      };
-      sdbDeleteRow(&oper);
-      pNode = pLastNode;
-      numOfVgroups++;
-      continue;
-    }
-  }
-
-  SMnodeObj *pMnode = mgmtGetMnode(pDnode->dnodeId);
-  if (pMnode != NULL) {
-    SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
-    sdbDeleteRow(&oper);
-    mgmtReleaseMnode(pMnode);
-  }
  
+  mgmtDropAllDnodeVgroups(pDnode);
+  mgmtDropMnodeLocal(pDnode->dnodeId);
   balanceNotify();
 
-  mTrace("dnode:%d, all vgroups:%d is dropped from sdb", pDnode->dnodeId, numOfVgroups);
+  mTrace("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -116,6 +89,7 @@ static int32_t mgmtDnodeActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDnode, pOper->rowSize);
     free(pDnode);
   }
+  mgmtDecDnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -212,7 +186,7 @@ void *mgmtGetDnodeByIp(char *ep) {
   void *     pNode = NULL;
 
   while (1) {
-    pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode);
+    pNode = mgmtGetNextDnode(pNode, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
       return pDnode;
@@ -235,8 +209,7 @@ void mgmtUpdateDnode(SDnodeObj *pDnode) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsDnodeSdb,
-    .pObj = pDnode,
+    .pObj = pDnode
-    .rowSize = tsDnodeUpdateSize
   };
 
   sdbUpdateRow(&oper);
@@ -336,7 +309,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
     SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId);
     if (pVgroup == NULL) {
       SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
-      mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
+      mPrint("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
       mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
     } else {
       mgmtUpdateVgroupStatus(pVgroup, pDnode, pVload);
@@ -387,6 +360,7 @@ static int32_t mgmtCreateDnode(char *ep) {
 
   SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
   if (pDnode != NULL) {
+    mgmtDecDnodeRef(pDnode);
     mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort);
     return TSDB_CODE_DNODE_ALREADY_EXIST;
   }
@@ -440,6 +414,7 @@ static int32_t mgmtDropDnodeByIp(char *ep) {
     return TSDB_CODE_DNODE_NOT_EXIST;
   }
 
+  mgmtDecDnodeRef(pDnode);
   if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) {
     mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
     return TSDB_CODE_NO_REMOVE_MASTER;
@@ -464,6 +439,7 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) {
     if (rpcRsp.code == TSDB_CODE_SUCCESS) {
       SDnodeObj *pDnode = mgmtGetDnodeByIp(pCreate->ep);
       mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
+      mgmtDecDnodeRef(pDnode);
     } else {
       mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(rpcRsp.code));
     }
@@ -492,7 +468,7 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
 }
 
 static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->pAcct->user, "root") != 0) {
@@ -609,7 +585,7 @@ static bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
 static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->user, "root") != 0)  {
@@ -719,7 +695,7 @@ static bool mgmtCheckConfigShow(SGlobalCfg *cfg) {
 static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->user, "root") != 0)  {
@@ -806,7 +782,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo
 
 static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
   
   if (strcmp(pUser->user, "root") != 0)  {

src/mnode/src/mgmtMain.c

@@ -36,7 +36,7 @@
 #include "mgmtTable.h"
 #include "mgmtShell.h"
 
-void *tsMgmtTmr = NULL;
+extern void *tsMgmtTmr;
 static bool tsMgmtIsRunning = false;
 
 int32_t mgmtStartSystem() {
@@ -51,12 +51,6 @@ int32_t mgmtStartSystem() {
     mkdir(tsMnodeDir, 0755);
   }
 
-  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
-  if (tsMgmtTmr == NULL) {
-    mError("failed to init timer");
-    return -1;
-  }
-
   if (mgmtInitAccts() < 0) {
     mError("failed to init accts");
     return -1;
@@ -130,7 +124,7 @@ int32_t mgmtInitSystem() {
 
   struct stat dirstat;
   bool fileExist  = (stat(tsMnodeDir, &dirstat) == 0);
-  bool asMaster = (strcmp(tsMaster, tsLocalEp) == 0);
+  bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0);
 
   if (asMaster || fileExist) {
     if (mgmtStartSystem() != 0) {

src/mnode/src/mgmtMnode.c

@@ -30,7 +30,7 @@
 #include "mgmtShell.h"
 #include "mgmtUser.h"
 
-void * tsMnodeSdb = NULL;
+static void *  tsMnodeSdb = NULL;
 static int32_t tsMnodeUpdateSize = 0;
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -71,7 +71,7 @@ static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pMnode, pOper->rowSize);
     free(pMnode);
   }
-
+  mgmtDecMnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -97,7 +97,7 @@ static int32_t mgmtMnodeActionRestored() {
     mgmtGetNextMnode(NULL, &pMnode);
     if (pMnode != NULL) {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
   return TSDB_CODE_SUCCESS;
@@ -148,7 +148,11 @@ void *mgmtGetMnode(int32_t mnodeId) {
   return sdbGetRow(tsMnodeSdb, &mnodeId);
 }
 
-void mgmtReleaseMnode(SMnodeObj *pMnode) {
+void mgmtIncMnodeRef(SMnodeObj *pMnode) {
+  sdbIncRef(tsMnodeSdb, pMnode);
+}
+
+void mgmtDecMnodeRef(SMnodeObj *pMnode) {
   sdbDecRef(tsMnodeSdb, pMnode);
 }
 
@@ -187,7 +191,7 @@ void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
 
     ipSet->numOfIps++;
     
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 }
 
@@ -209,7 +213,7 @@ void mgmtGetMnodeInfos(void *param) {
     }
 
     index++;
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 
   mnodes->nodeNum = index;
@@ -235,8 +239,17 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
   return code;
 }
 
+void mgmtDropMnodeLocal(int32_t dnodeId) {
+  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
+  if (pMnode != NULL) {
+    SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
+    sdbDeleteRow(&oper);
+    mgmtDecMnodeRef(pMnode);
+  }
+}
+
 int32_t mgmtDropMnode(int32_t dnodeId) {
-  SMnodeObj *pMnode = sdbGetRow(tsMnodeSdb, &dnodeId);
+  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
   if (pMnode == NULL) {
     return TSDB_CODE_DNODE_NOT_EXIST;
   }
@@ -258,7 +271,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
 
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   sdbUpdateMnodeRoles();
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->pAcct->user, "root") != 0)  {
@@ -339,7 +352,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     
     numOfRows++;
 
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 
   pShow->numOfReads += numOfRows;

src/mnode/src/mgmtProfile.c

@@ -561,17 +561,6 @@ int32_t mgmtKillConnection(char *qidstr, void *pConn) {
   return TSDB_CODE_INVALID_CONNECTION;
 }
 
-bool mgmtCheckQhandle(uint64_t qhandle) {
-  return true;
-}
-
-void mgmtSaveQhandle(void *qhandle) {
-  mTrace("qhandle:%p is allocated", qhandle);
-}
-
-void mgmtFreeQhandle(void *qhandle) {
-  mTrace("qhandle:%p is freed", qhandle);
-}
 
 int mgmtGetConns(SShowObj *pShow, void *pConn) {
   //  SAcctObj * pAcct = pConn->pAcct;
@@ -686,7 +675,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
 void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -710,7 +699,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
 void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -734,7 +723,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
 void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -771,52 +760,3 @@ int32_t mgmtInitProfile() {
 
 void mgmtCleanUpProfile() {
 }
-
-void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
-  bool usePublicIp = false;
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp);
-  if (pUser == NULL) {
-    return NULL;
-  }
-
-  SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
-  pMsg->thandle = rpcMsg->handle;
-  pMsg->msgType = rpcMsg->msgType;
-  pMsg->contLen = rpcMsg->contLen;
-  pMsg->pCont = rpcMsg->pCont;
-  pMsg->pUser = pUser;
-  pMsg->usePublicIp = usePublicIp;
-
-  return pMsg;
-}
-
-void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
-  if (pMsg != NULL) {
-    rpcFreeCont(pMsg->pCont);
-    if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
-    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
-    if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
-    if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
-    if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
-    if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
-    free(pMsg);
-  }
-}
-
-void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
-  SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
-  
-  pDestMsg->thandle = pSrcMsg->thandle;
-  pDestMsg->msgType = pSrcMsg->msgType;
-  pDestMsg->pCont   = pSrcMsg->pCont;
-  pDestMsg->contLen = pSrcMsg->contLen;
-  pDestMsg->retry   = pSrcMsg->retry;
-  pDestMsg->maxRetry= pSrcMsg->maxRetry;
-  pDestMsg->pUser   = pSrcMsg->pUser;
-  pDestMsg->usePublicIp = pSrcMsg->usePublicIp;
-
-  pSrcMsg->pCont = NULL;
-  pSrcMsg->pUser = NULL;
-  
-  return pDestMsg;
-}

src/mnode/src/mgmtSdb.c

@@ -40,7 +40,7 @@ typedef enum {
 typedef enum {
   SDB_STATUS_OFFLINE,
   SDB_STATUS_SERVING,
-  SDB_ACTION_CLOSING
+  SDB_STATUS_CLOSING
 } ESdbStatus;
 
 typedef struct _SSdbTable {
@@ -107,6 +107,10 @@ bool sdbIsMaster() {
   return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER; 
 }
 
+bool sdbIsServing() {
+  return tsSdbObj.status == SDB_STATUS_SERVING; 
+}
+
 static char *sdbGetActionStr(int32_t action) {
   switch (action) {
     case SDB_ACTION_INSERT:
@@ -180,7 +184,7 @@ void sdbUpdateMnodeRoles() {
     if (pMnode != NULL) {
       pMnode->role = roles.role[i];
       sdbPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
 }
@@ -248,18 +252,12 @@ void sdbUpdateSync() {
       strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
       index++;
 
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
 
   syncCfg.replica = index;
-  syncCfg.arbitratorPort = syncCfg.nodeInfo[0].nodePort;
+  syncCfg.quorum = (syncCfg.replica == 1) ? 1:2;
-  strcpy(syncCfg.arbitratorFqdn, syncCfg.nodeInfo[0].nodeFqdn);
-  if (syncCfg.replica == 1) {
-    syncCfg.quorum = 1;
-  } else {
-    syncCfg.quorum = 2;
-  }
 
   bool hasThisDnode = false;
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
@@ -272,9 +270,9 @@ void sdbUpdateSync() {
   if (!hasThisDnode) return;
   if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
 
-  sdbPrint("work as mnode, replica:%d arbitrator:%s", syncCfg.replica, syncCfg.arbitratorFqdn);
+  sdbPrint("work as mnode, replica:%d", syncCfg.replica);
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
-    sdbPrint("mnode:%d, ip:%s", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn);
+    sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
   }
 
   SSyncInfo syncInfo;
@@ -320,6 +318,7 @@ int32_t sdbInit() {
 void sdbCleanUp() {
   if (tsSdbObj.status != SDB_STATUS_SERVING) return;
 
+  tsSdbObj.status = SDB_STATUS_CLOSING;
   syncStop(tsSdbObj.sync);
   free(tsSdbObj.sync);
   walClose(tsSdbObj.wal);
@@ -333,7 +332,7 @@ void sdbIncRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     atomic_add_fetch_32(pRefCount, 1);
-    if (0 && strcmp(pTable->tableName, "accounts") == 0) {
+    if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
       sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
                *pRefCount);
     }
@@ -345,7 +344,7 @@ void sdbDecRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
-    if (0 && strcmp(pTable->tableName, "accounts") == 0) {
+    if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
       sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
                *pRefCount);
     }

src/mnode/src/mgmtShell.c

@@ -23,6 +23,7 @@
 #include "ttimer.h"
 #include "tgrant.h"
 #include "tglobal.h"
+#include "tcache.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtLog.h"
@@ -50,10 +51,11 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
 
-extern void *tsMgmtTmr;
+void *tsMgmtTmr;
 static void *tsMgmtShellRpc = NULL;
 static void *tsMgmtTranQhandle = NULL;
 static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
+static void *tsQhandleCache = NULL;
 static SShowMetaFp     tsMgmtShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMgmtShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
 
@@ -64,7 +66,9 @@ int32_t mgmtInitShell() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mgmtProcessUseMsg);
   
+  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
   tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
+  tsQhandleCache = taosCacheInit(tsMgmtTmr, 2);
 
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0;
   if (numOfThreads < 1) {
@@ -102,6 +106,12 @@ void mgmtCleanUpShell() {
     tsMgmtShellRpc = NULL;
     mPrint("server connection to shell is closed");
   }
+
+  if (tsQhandleCache) {
+    taosCacheEmpty(tsQhandleCache);
+    taosCacheCleanup(tsQhandleCache);
+    tsQhandleCache = NULL;
+  }
 }
 
 void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SQueuedMsg *queuedMsg)) {
@@ -233,14 +243,15 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SShowObj *pShow = (SShowObj *) calloc(1, sizeof(SShowObj) + htons(pShowMsg->payloadLen));
+  int32_t showObjSize = sizeof(SShowObj) + htons(pShowMsg->payloadLen);
+  SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
   pShow->signature  = pShow;
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
   strcpy(pShow->db, pShowMsg->db);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
-  mgmtSaveQhandle(pShow);
+  pShow = mgmtSaveQhandle(pShow, showObjSize);
   pShowRsp->qhandle = htobe64((uint64_t) pShow);
 
   mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type));
@@ -255,10 +266,10 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
     rpcSendResponse(&rpcRsp);
   } else {
     mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code));
-    mgmtFreeQhandle(pShow);
+    mgmtFreeQhandle(pShow, false);
     SRpcMsg rpcRsp = {
-      .handle  = pMsg->thandle,
+      .handle = pMsg->thandle,
-      .code    = code
+      .code   = code
     };
     rpcSendResponse(&rpcRsp);
   }
@@ -284,26 +295,20 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
   SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
   mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type));
 
-  if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
+  if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
-    mError("pShow:%p, query memory is corrupted", pShow);
+    rowsToRead = pShow->numOfRows - pShow->numOfReads;
-    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MEMORY_CORRUPTED);
-    return;
-  } else {
-    if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
-      rowsToRead = pShow->numOfRows - pShow->numOfReads;
-    }
-
-    /* return no more than 100 meters in one round trip */
-    if (rowsToRead > 100) rowsToRead = 100;
-
-    /*
-     * the actual number of table may be larger than the value of pShow->numOfRows, if a query is
-     * issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
-     */
-    if (rowsToRead < 0) rowsToRead = 0;
-    size = pShow->rowSize * rowsToRead;
   }
 
+  /* return no more than 100 meters in one round trip */
+  if (rowsToRead > 100) rowsToRead = 100;
+
+  /*
+   * the actual number of table may be larger than the value of pShow->numOfRows, if a query is
+   * issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
+   */
+  if (rowsToRead < 0) rowsToRead = 0;
+  size = pShow->rowSize * rowsToRead;
+
   size += 100;
   SRetrieveTableRsp *pRsp = rpcMallocCont(size);
 
@@ -313,6 +318,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
 
   if (rowsRead < 0) {  // TSDB_CODE_ACTION_IN_PROGRESS;
     rpcFreeCont(pRsp);
+    mgmtFreeQhandle(pShow, false);
     return;
   }
 
@@ -329,7 +335,9 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 
   if (rowsToRead == 0) {
-    mgmtFreeQhandle(pShow);
+    mgmtFreeQhandle(pShow, true);
+  } else {
+    mgmtFreeQhandle(pShow, false);
   }
 }
 
@@ -413,6 +421,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
       code = TSDB_CODE_INVALID_DB;
       goto connect_over;
     }
+    mgmtDecDbRef(pDb);
   }
 
   SCMConnectRsp *pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
@@ -446,9 +455,8 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
   SCMUseDbMsg *pUseDbMsg = pMsg->pCont;
   
   // todo check for priority of current user
-  pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
-  
   int32_t code = TSDB_CODE_SUCCESS;
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
   if (pMsg->pDb == NULL) {
     code = TSDB_CODE_INVALID_DB;
   }
@@ -462,7 +470,7 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
  */
 static bool mgmtCheckTableMetaMsgReadOnly(SQueuedMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->pCont;
-  pMsg->pTable = mgmtGetTable(pInfo->tableId);
+  if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
   if (pMsg->pTable != NULL) return true;
 
   // If table does not exists and autoCreate flag is set, we add the handler into task queue
@@ -511,3 +519,79 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) {
   };
   rpcSendResponse(&rpcRsp);
 }
+
+bool mgmtCheckQhandle(uint64_t qhandle) {
+  void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle);
+  if (pSaved == (void *)qhandle) {
+    mTrace("qhandle:%p is retrived", qhandle);
+    return true;
+  } else {
+    mTrace("qhandle:%p is already freed", qhandle);
+    return false;
+  }
+}
+
+void* mgmtSaveQhandle(void *qhandle, int32_t size) {
+  if (tsQhandleCache != NULL) {
+    char key[24];
+    sprintf(key, "show:%p", qhandle);
+    void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60);
+    free(qhandle);
+    
+    mTrace("qhandle:%p is saved", newQhandle);
+    return newQhandle;
+  }
+  
+  return NULL;
+}
+
+void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
+  mTrace("qhandle:%p is freed", qhandle);
+  taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
+}
+
+void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
+  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  if (pUser == NULL) {
+    return NULL;
+  }
+
+  SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
+  pMsg->thandle = rpcMsg->handle;
+  pMsg->msgType = rpcMsg->msgType;
+  pMsg->contLen = rpcMsg->contLen;
+  pMsg->pCont = rpcMsg->pCont;
+  pMsg->pUser = pUser;
+
+  return pMsg;
+}
+
+void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
+  if (pMsg != NULL) {
+    rpcFreeCont(pMsg->pCont);
+    if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
+    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
+    if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
+    if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
+    if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
+    if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
+    free(pMsg);
+  }
+}
+
+void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
+  SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
+  
+  pDestMsg->thandle = pSrcMsg->thandle;
+  pDestMsg->msgType = pSrcMsg->msgType;
+  pDestMsg->pCont   = pSrcMsg->pCont;
+  pDestMsg->contLen = pSrcMsg->contLen;
+  pDestMsg->retry   = pSrcMsg->retry;
+  pDestMsg->maxRetry= pSrcMsg->maxRetry;
+  pDestMsg->pUser   = pSrcMsg->pUser;
+ 
+  pSrcMsg->pCont = NULL;
+  pSrcMsg->pUser = NULL;
+  
+  return pDestMsg;
+}

src/mnode/src/mgmtTable.c

@@ -41,8 +41,8 @@
 #include "mgmtVgroup.h"
 #include "tcompare.h"
 
-void *  tsChildTableSdb;
+static void *  tsChildTableSdb;
-void *  tsSuperTableSdb;
+static void *  tsSuperTableSdb;
 static int32_t tsChildTableUpdateSize;
 static int32_t tsSuperTableUpdateSize;
 static void *  mgmtGetChildTable(char *tableId);
@@ -97,14 +97,14 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
 
   SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
   if (pVgroup == NULL) {
-    mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId);
+    mError("ctable:%s, not in vgId:%d", pTable->info.tableId, pTable->vgId);
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
   mgmtDecVgroupRef(pVgroup);
 
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
-    mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
+    mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
   mgmtDecDbRef(pDb);
@@ -117,6 +117,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) {
   mgmtDecAcctRef(pAcct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
+    // add ref
     pTable->superTable = mgmtGetSuperTable(pTable->superTableId);
     mgmtAddTableIntoStable(pTable->superTable, pTable);
     grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
@@ -146,7 +147,7 @@ static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) {
 
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
-    mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
+    mError("ctable:%s, vgId:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
     return TSDB_CODE_INVALID_DB;
   }
   mgmtDecDbRef(pDb);
@@ -186,6 +187,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) {
     free(oldSql);
     free(oldSchema);
   }
+  mgmtDecTableRef(pTable);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -250,7 +252,7 @@ static int32_t mgmtChildTableActionRestored() {
   while (1) {
     pLastNode = pNode;
     mgmtDecTableRef(pTable);
-    pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
+    pNode = mgmtGetNextChildTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId);
@@ -268,7 +270,7 @@ static int32_t mgmtChildTableActionRestored() {
 
     SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId);
     if (pVgroup == NULL) {
-      mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
+      mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid);
       pTable->vgId = 0;
       SSdbOper desc = {0};
       desc.type = SDB_OPER_LOCAL;
@@ -281,7 +283,7 @@ static int32_t mgmtChildTableActionRestored() {
     mgmtDecVgroupRef(pVgroup);
 
     if (strcmp(pVgroup->dbName, pDb->name) != 0) {
-      mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it",
+      mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it",
              pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid);
       pTable->vgId = 0;
       SSdbOper desc = {0};
@@ -294,7 +296,7 @@ static int32_t mgmtChildTableActionRestored() {
     }
 
     if (pVgroup->tableList == NULL) {
-      mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId);
+      mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId);
       pTable->vgId = 0;
       SSdbOper desc = {0};
       desc.type = SDB_OPER_LOCAL;
@@ -361,7 +363,7 @@ static void mgmtCleanUpChildTables() {
 
 static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
   if (pStable->vgLen == 0) {
-    pStable->vgLen = 10;
+    pStable->vgLen = 8;
     pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t));
   }
   
@@ -431,11 +433,11 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
     void *oldSchema = pTable->schema;
     memcpy(pTable, pNew, pOper->rowSize);
     pTable->schema = pNew->schema;
-    free(pNew);
     free(pNew->vgList);
+    free(pNew);
     free(oldSchema);
   }
-
+  mgmtDecTableRef(pTable);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -558,20 +560,28 @@ static void *mgmtGetSuperTable(char *tableId) {
   return sdbGetRow(tsSuperTableSdb, tableId);
 }
 
-STableObj *mgmtGetTable(char *tableId) {
+void *mgmtGetTable(char *tableId) {
-  STableObj *tableInfo = sdbGetRow(tsSuperTableSdb, tableId);
+  void *pTable = mgmtGetSuperTable(tableId);
-  if (tableInfo != NULL) {
+  if (pTable != NULL) {
-    return tableInfo;
+    return pTable;
   }
 
-  tableInfo = sdbGetRow(tsChildTableSdb, tableId);
+  pTable = mgmtGetChildTable(tableId);
-  if (tableInfo != NULL) {
+  if (pTable != NULL) {
-    return tableInfo;
+    return pTable;
   }
 
   return NULL;
 }
 
+void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) {
+  return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable);
+}
+
+void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) {
+  return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable);
+}
+
 void mgmtIncTableRef(void *p1) {
   STableObj *pTable = (STableObj *)p1;
   if (pTable->type == TSDB_SUPER_TABLE) {
@@ -613,11 +623,18 @@ static void mgmtExtractTableName(char* tableId, char* name) {
 static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
   SCMCreateTableMsg *pCreate = pMsg->pCont;
   
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pCreate->db);
+  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
+    mError("table:%s, failed to create, db not selected", pCreate->tableId);
+    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
+    return;
+  }
+
   if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId);
   if (pMsg->pTable != NULL && pMsg->retry == 0) {
     if (pCreate->getMeta) {
       mTrace("table:%s, continue to get meta", pCreate->tableId);
-      mgmtProcessTableMetaMsg(pMsg);
+      mgmtGetChildTableMeta(pMsg);
     } else if (pCreate->igExists) {
       mTrace("table:%s, is already exist", pCreate->tableId);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@@ -628,13 +645,6 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pCreate->db);
-  if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
-    mError("table:%s, failed to create, db not selected", pCreate->tableId);
-    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
-    return;
-  }
-
   if (pCreate->numOfTags != 0) {
     mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->thandle);
     mgmtProcessCreateSuperTableMsg(pMsg);
@@ -646,7 +656,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) {
 
 static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
   SCMDropTableMsg *pDrop = pMsg->pCont;
-  pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId);
   if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to drop table, db not selected", pDrop->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
@@ -659,8 +669,8 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  pMsg->pTable = mgmtGetTable(pDrop->tableId);
+  if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pDrop->tableId);
-  if (pMsg->pTable  == NULL) {
+  if (pMsg->pTable == NULL) {
     if (pDrop->igNotExists) {
       mTrace("table:%s, table is not exist, think drop success", pDrop->tableId);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@@ -683,7 +693,8 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) {
 
 static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->pCont;
-  mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle);
+  pInfo->createFlag = htons(pInfo->createFlag);
+  mTrace("table:%s, table meta msg is received from thandle:%p, createFlag:%d", pInfo->tableId, pMsg->thandle, pInfo->createFlag);
 
   if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId);
   if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
@@ -694,7 +705,7 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) {
 
   if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
   if (pMsg->pTable == NULL) {
-    if (htons(pInfo->createFlag) != 1) {
+    if (!pInfo->createFlag) {
       mError("table:%s, failed to get table meta, table not exist", pInfo->tableId);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
     } else {
@@ -786,8 +797,6 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
         mgmtDecVgroupRef(pVgroup);
       }
     }
-    //mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables);
-    //mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
   } else {
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
@@ -845,8 +854,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
-    .pObj = pStable,
+    .pObj = pStable
-    .rowSize = tsSuperTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -873,8 +881,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
-    .pObj = pStable,
+    .pObj = pStable
-    .rowSize = tsSuperTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -910,8 +917,7 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
-    .pObj = pStable,
+    .pObj = pStable
-    .rowSize = tsSuperTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -976,8 +982,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
-    .pObj = pStable,
+    .pObj = pStable
-    .rowSize = tsSuperTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -1014,8 +1019,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsSuperTableSdb,
-    .pObj = pStable,
+    .pObj = pStable
-    .rowSize = tsSuperTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -1098,7 +1102,8 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
   char stableName[TSDB_TABLE_NAME_LEN] = {0};
 
   while (numOfRows < rows) {    
-    pShow->pNode = sdbFetchRow(tsSuperTableSdb, pShow->pNode, (void **) &pTable);
+    mgmtDecTableRef(pTable);
+    pShow->pNode = mgmtGetNextSuperTable(pShow->pNode, &pTable);
     if (pTable == NULL) break;
     if (strncmp(pTable->info.tableId, prefix, prefixLen)) {
       continue;
@@ -1134,8 +1139,6 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v
     cols++;
 
     numOfRows++;
-    mgmtDecTableRef(pTable);
-
   }
 
   pShow->numOfReads += numOfRows;
@@ -1154,7 +1157,8 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) {
   mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **)&pTable);
+    pLastNode = pNode;
+    pNode = mgmtGetNextSuperTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1448,17 +1452,21 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
   }
 
   int32_t sid = taosAllocateId(pVgroup->idPool);
-  if (sid < 0) {
+  if (sid <= 0) {
-    mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId);
+    mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId);
     mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb);
     return;
   }
 
   if (pMsg->retry == 0) {
-    pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
+    if (pMsg->pTable == NULL) {
+      pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid);
+      mgmtIncTableRef(pMsg->pTable);
+    }
   } else {
-    pMsg->pTable = mgmtGetTable(pCreate->tableId);
+    if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId);
   }
+
   if (pMsg->pTable == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, terrno);
     return;
@@ -1474,7 +1482,6 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   newMsg->ahandle = pMsg->pTable;
   newMsg->maxRetry = 5;
-  mgmtIncTableRef(pMsg->pTable);
   SRpcMsg rpcMsg = {
       .handle  = newMsg,
       .pCont   = pMDCreate,
@@ -1488,8 +1495,8 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) {
 
 static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
   SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
-  SVgObj *pVgroup = pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
+  if (pMsg->pVgroup == NULL) pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
-  if (pVgroup == NULL) {
+  if (pMsg->pVgroup == NULL) {
     mError("table:%s, failed to drop ctable, vgroup not exist", pTable->info.tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS);
     return;
@@ -1508,7 +1515,7 @@ static void mgmtProcessDropChildTableMsg(SQueuedMsg *pMsg) {
   pDrop->sid     = htonl(pTable->sid);
   pDrop->uid     = htobe64(pTable->uid);
 
-  SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
+  SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pMsg->pVgroup);
 
   mTrace("table:%s, send drop ctable msg", pDrop->tableId);
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
@@ -1574,8 +1581,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsChildTableSdb,
-    .pObj = pTable,
+    .pObj = pTable
-    .rowSize = tsChildTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -1607,8 +1613,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsChildTableSdb,
-    .pObj = pTable,
+    .pObj = pTable
-    .rowSize = tsChildTableUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -1656,21 +1661,21 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) {
     pMeta->contLen      = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); 
   }
   
-  SVgObj *pVgroup = pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
+  if (pMsg->pVgroup == NULL) pMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
-  if (pVgroup == NULL) {
+  if (pMsg->pVgroup == NULL) {
     mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId);
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
 
-  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+  for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) {
-    SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId);
+    SDnodeObj *pDnode = mgmtGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId);
     if (pDnode == NULL) break;
     strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn);
     pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL);
     pMeta->vgroup.numOfIps++;
     mgmtDecDnodeRef(pDnode);
   }
-  pMeta->vgroup.vgId = htonl(pVgroup->vgId);
+  pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId);
 
   mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid);
 
@@ -1691,10 +1696,12 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) {
   strcpy(pCreateMsg->db, pMsg->pDb->name);
   pCreateMsg->igExists = 1;
   pCreateMsg->getMeta = 1;
+  pCreateMsg->contLen = htonl(contLen);
   memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData));
 
   SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
   pMsg->pCont = newMsg->pCont;
+  newMsg->msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
   newMsg->pCont = pCreateMsg;
 
   mTrace("table:%s, start to create on demand", pInfo->tableId);
@@ -1730,7 +1737,8 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) {
   mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name);
 
   while (1) {
-    pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
+    pLastNode = pNode;
+    pNode = mgmtGetNextChildTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) {
@@ -1758,7 +1766,8 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
   mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables);
 
   while (1) {
-    pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable);
+    pLastNode = pNode;
+    pNode = mgmtGetNextChildTable(pNode, &pTable);
     if (pTable == NULL) break;
 
     if (pTable->superTable == pStable) {
@@ -1778,16 +1787,13 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) {
   mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
 }
 
-static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) {
+static SChildTableObj* mgmtGetTableByPos(int32_t vnode, int32_t sid) {
-  SDnodeObj *pObj = mgmtGetDnode(dnodeId);
   SVgObj *pVgroup = mgmtGetVgroup(vnode);
+  if (pVgroup == NULL) return NULL;
 
-  if (pObj == NULL || pVgroup == NULL) {
+  SChildTableObj *pTable = pVgroup->tableList[sid - 1];
-    return NULL;
-  }
-
-  SChildTableObj *pTable = pVgroup->tableList[sid];
   mgmtIncTableRef((STableObj *)pTable);
+
   mgmtDecVgroupRef(pVgroup);
   return pTable;
 }
@@ -1799,7 +1805,7 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
   pCfg->sid   = htonl(pCfg->sid);
   mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
 
-  SChildTableObj *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid);
+  SChildTableObj *pTable = mgmtGetTableByPos(pCfg->vnode, pCfg->sid);
   if (pTable == NULL) {
     mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid);
     mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE);
@@ -1814,6 +1820,7 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
     mgmtDecTableRef(pTable);
     return;
   }
+
   SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnode);
   SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
   SRpcMsg rpcRsp = {
@@ -1824,7 +1831,9 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) {
       .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE
   };
   mgmtSendMsgToDnode(&ipSet, &rpcRsp);
+
   mgmtDecTableRef(pTable);
+  mgmtDecDnodeRef(pDnode);
 }
 
 // handle drop child response
@@ -1845,8 +1854,8 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
     return;
   }
 
-  SVgObj *pVgroup = queueMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
+  if (queueMsg->pVgroup == NULL) queueMsg->pVgroup = mgmtGetVgroup(pTable->vgId);
-  if (pVgroup == NULL) {
+  if (queueMsg->pVgroup == NULL) {
     mError("table:%s, failed to get vgroup", pTable->info.tableId);
     mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID);
     return;
@@ -1865,9 +1874,9 @@ static void mgmtProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
     return;
   }
 
-  if (pVgroup->numOfTables <= 0) {
+  if (queueMsg->pVgroup->numOfTables <= 0) {
-    mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId);
+    mPrint("vgId:%d, all tables is dropped, drop vgroup", queueMsg->pVgroup->vgId);
-    mgmtDropVgroup(pVgroup, NULL);
+    mgmtDropVgroup(queueMsg->pVgroup, NULL);
   }
 
   mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS);
@@ -1908,9 +1917,10 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
   } else {
     mTrace("table:%s, created in dnode, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle,
            tstrerror(rpcMsg->code));
-
+    SCMCreateTableMsg *pCreate = queueMsg->pCont;
-    if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) {
+    if (pCreate->getMeta) {
-      mTrace("table:%s, start to get meta", pTable->info.tableId);
+      mTrace("table:%s, continue to get meta", pTable->info.tableId);
+      queueMsg->retry = 0;
       mgmtAddToShellQueue(queueMsg);
     } else {
       mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code);
@@ -1943,8 +1953,8 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) {
     SChildTableObj *pTable = mgmtGetChildTable(tableId);
     if (pTable == NULL) continue;
 
-    SDbObj *pDb = mgmtGetDbByTableId(tableId);
+    if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(tableId);
-    if (pDb == NULL) continue;
+    if (pMsg->pDb == NULL) continue;
 
     int availLen = totalMallocLen - pMultiMeta->contLen;
     if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) {
@@ -2043,7 +2053,8 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
   int32_t prefixLen = strlen(prefix);
 
   while (numOfRows < rows) {
-    pShow->pNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable);
+    mgmtDecTableRef(pTable);
+    pShow->pNode = mgmtGetNextChildTable(pShow->pNode, &pTable);
     if (pTable == NULL) break;
 
     // not belong to current db
@@ -2087,7 +2098,6 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows,
     cols++;
 
     numOfRows++;
-    mgmtDecTableRef(pTable);
   }
 
   pShow->numOfReads += numOfRows;
@@ -2103,7 +2113,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
   SCMAlterTableMsg *pAlter = pMsg->pCont;
   mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle);
 
-  pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId);
   if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
     mError("table:%s, failed to alter table, db not selected", pAlter->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED);
@@ -2116,7 +2126,7 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  pMsg->pTable = mgmtGetTable(pAlter->tableId);
+  if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pAlter->tableId);
   if (pMsg->pTable == NULL) {
     mError("table:%s, failed to alter table, table not exist", pMsg->pTable->tableId);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);

src/mnode/src/mgmtUser.c

@@ -168,8 +168,7 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsUserSdb,
-    .pObj = pUser,
+    .pObj = pUser
-    .rowSize = tsUserUpdateSize
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -249,7 +248,7 @@ static int32_t mgmtDropUser(SUserObj *pUser) {
 }
 
 static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) {
     return TSDB_CODE_NO_USER_FROM_CONN;
   }
@@ -298,7 +297,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   char     *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = sdbFetchRow(tsUserSdb, pShow->pNode, (void **) &pUser);
+    pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser);
     if (pUser == NULL) break;
     
     cols = 0;
@@ -329,12 +328,9 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   return numOfRows;
 }
 
-SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
+SUserObj *mgmtGetUserFromConn(void *pConn) {
   SRpcConnInfo connInfo;
   if (rpcGetConnInfo(pConn, &connInfo) == 0) {
-    if (usePublicIp) {
-      *usePublicIp = (connInfo.serverIp == tsPublicIpInt);
-    }
     return mgmtGetUser(connInfo.user);
   } else {
     mError("can not get user from conn:%p", pConn);
@@ -510,7 +506,7 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
 
   while (1) {
     pLastNode = pNode;
-    pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
+    pNode = mgmtGetNextUser(pNode, &pUser);
     if (pUser == NULL) break;
 
     if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {

src/mnode/src/mgmtVgroup.c

@@ -36,8 +36,8 @@
 #include "mgmtTable.h"
 #include "mgmtVgroup.h"
 
-void   *tsVgroupSdb = NULL;
+static void   *tsVgroupSdb = NULL;
-int32_t tsVgUpdateSize = 0;
+static int32_t tsVgUpdateSize = 0;
 
 static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -62,6 +62,8 @@ static int32_t mgmtVgroupActionDestroy(SSdbOper *pOper) {
 
 static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
   SVgObj *pVgroup = pOper->pObj;
+
+  // refer to db
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
     return TSDB_CODE_INVALID_DB;
@@ -74,13 +76,13 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
   int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
   pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
   if (pVgroup->tableList == NULL) {
-    mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
+    mError("vgId:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
     return -1;
   }
 
   pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables);
   if (pVgroup->idPool == NULL) {
-    mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
+    mError("vgId:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
     tfree(pVgroup->tableList);
     return -1;
   }
@@ -140,6 +142,7 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
       if (pDnode != NULL) {
         atomic_add_fetch_32(&pDnode->openVnodes, 1);
       }
+      mgmtDecDnodeRef(pDnode);
     }
   }
 
@@ -147,14 +150,15 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
   SDbObj *pDb = pVgroup->pDb;
   if (pDb != NULL) {
     if (pDb->cfg.maxTables != oldTables) {
-      mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
+      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
       taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
       int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
       pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
     }
   }
 
-  mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
+  mgmtDecVgroupRef(pVgroup);
+  mTrace("vgId:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -237,8 +241,7 @@ void mgmtUpdateVgroup(SVgObj *pVgroup) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsVgroupSdb,
-    .pObj = pVgroup,
+    .pObj = pVgroup
-    .rowSize = tsVgUpdateSize
   };
 
   sdbUpdateRow(&oper);
@@ -261,7 +264,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
 
   if (!dnodeExist) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
-    mError("vgroup:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
+    mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
     mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
     return;
   }
@@ -273,7 +276,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
   }
 
   if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) {
-    mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
+    mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
            pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
            pVgroup->numOfVnodes);
     mgmtSendCreateVgroupMsg(pVgroup, NULL);
@@ -317,9 +320,9 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
     return;
   }
 
-  mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+  mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    mPrint("vgroup:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
+    mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
   }
 
   pMsg->ahandle = pVgroup;
@@ -331,7 +334,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
   if (ahandle != NULL) {
     mgmtSendDropVgroupMsg(pVgroup, ahandle);
   } else {
-    mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
+    mTrace("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
     mgmtSendDropVgroupMsg(pVgroup, NULL);
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
@@ -379,6 +382,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   if (pShow->payloadLen > 0 ) {
     pTable = mgmtGetTable(pShow->payload);
     if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
+      mgmtDecTableRef(pTable);
       return TSDB_CODE_INVALID_TABLE_ID;
     }
     mgmtDecTableRef(pTable);
@@ -505,25 +509,28 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
 }
 
 void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
-  if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) {
+  if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] == NULL) {
-    pVgroup->tableList[pTable->sid] = pTable;
+    pVgroup->tableList[pTable->sid - 1] = pTable;
     taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables++;
   }
   
-  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
+  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) {
-    mgmtAddVgroupIntoDbTail(pVgroup);
+    mgmtMoveVgroupToTail(pVgroup);
+  }
+
+  mgmtIncVgroupRef(pVgroup);
 }
 
 void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
-  if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) {
+  if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] != NULL) {
-    pVgroup->tableList[pTable->sid] = NULL;
+    pVgroup->tableList[pTable->sid - 1] = NULL;
     taosFreeId(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables--;
   }
 
-  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
+  mgmtMoveVgroupToHead(pVgroup);
-    mgmtAddVgroupIntoDbTail(pVgroup);
+  mgmtDecVgroupRef(pVgroup);
 }
 
 SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
@@ -588,7 +595,7 @@ SRpcIpSet mgmtGetIpSetFromIp(char *ep) {
 }
 
 void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
   SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup);
   SRpcMsg rpcMsg = {
     .handle  = ahandle,
@@ -601,7 +608,7 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
 }
 
 void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
     mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
@@ -619,7 +626,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   }
 
   SVgObj *pVgroup = queueMsg->ahandle;
-  mTrace("vgroup:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
+  mTrace("vgId:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
          pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
          queueMsg->thandle, rpcMsg->handle);
 
@@ -654,7 +661,7 @@ static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) {
 }
 
 void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgroup:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
+  mTrace("vgId:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
   SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId);
   SRpcMsg rpcMsg = {
       .handle  = ahandle,
@@ -667,7 +674,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
 }
 
 static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
     mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle);
@@ -675,7 +682,7 @@ static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
 }
 
 static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
-  mTrace("drop vnode rsp is received");
+  mTrace("drop vnode rsp is received, handle:%p", rpcMsg->handle);
   if (rpcMsg->handle == NULL) return;
 
   SQueuedMsg *queueMsg = rpcMsg->handle;
@@ -686,7 +693,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
   }
 
   SVgObj *pVgroup = queueMsg->ahandle;
-  mTrace("vgroup:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
+  mTrace("vgId:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
          pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
          queueMsg->thandle, rpcMsg->handle);
 
@@ -736,7 +743,33 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
   mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
 }
 
-void mgmtDropAllVgroups(SDbObj *pDropDb) {
+void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
+  void *  pNode = NULL;
+  void *  pLastNode = NULL;
+  SVgObj *pVgroup = NULL;
+  int32_t numOfVgroups = 0;
+
+  while (1) {
+    pLastNode = pNode;
+    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
+      SSdbOper oper = {
+        .type = SDB_OPER_LOCAL,
+        .table = tsVgroupSdb,
+        .pObj = pVgroup,
+      };
+      sdbDeleteRow(&oper);
+      pNode = pLastNode;
+      numOfVgroups++;
+      continue;
+    }
+    mgmtDecVgroupRef(pVgroup);
+  }
+}
+
+void mgmtDropAllDbVgroups(SDbObj *pDropDb) {
   void *pNode = NULL;
   void *pLastNode = NULL;
   int32_t numOfVgroups = 0;
@@ -744,7 +777,8 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
 
   mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
   while (1) {
-    pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
+    pLastNode = pNode;
+    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->pDb == pDropDb) {

src/os/darwin/src/darwinPlatform.c

@@ -151,67 +151,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
   return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
 }
 
-int taosOpenUDClientSocket(char *ip, uint16_t port) {
-  int                sockFd = 0;
-  struct sockaddr_un serverAddr;
-  int                ret;
-  char               name[128];
-  sprintf(name, "%s.%hu", ip, port);
-
-  sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
-
-  if (sockFd < 0) {
-    uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  memset((char *)&serverAddr, 0, sizeof(serverAddr));
-  serverAddr.sun_family = AF_UNIX;
-  strcpy(serverAddr.sun_path + 1, name);
-
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
-
-  if (ret != 0) {
-    uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
-    sockFd = -1;
-  }
-
-  return sockFd;
-}
-
-int taosOpenUDServerSocket(char *ip, uint16_t port) {
-  struct sockaddr_un serverAdd;
-  int                sockFd;
-  char               name[128];
-
-  uTrace("open ud socket:%s", name);
-  sprintf(name, "%s.%hu", ip, port);
-
-  bzero((char *)&serverAdd, sizeof(serverAdd));
-  serverAdd.sun_family = AF_UNIX;
-  strcpy(serverAdd.sun_path + 1, name);
-  unlink(name);
-
-  if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
-    uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  /* bind socket to server address */
-  if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
-    uError("bind socket:%s failed, reason:%s", name, strerror(errno));
-    tclose(sockFd);
-    return -1;
-  }
-
-  if (listen(sockFd, 10) < 0) {
-    uError("listen socket:%s failed, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  return sockFd;
-}
-
 int taosInitTimer(void (*callback)(int), int ms) {
   signal(SIGALRM, callback);
 

src/os/linux/src/linuxPlatform.c

@@ -143,68 +143,6 @@ int taosSetNonblocking(int sock, int on) {
 int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
   return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
 }
-
-int taosOpenUDClientSocket(char *ip, uint16_t port) {
-  int                sockFd = 0;
-  struct sockaddr_un serverAddr;
-  int                ret;
-  char               name[128];
-  sprintf(name, "%s.%hu", ip, port);
-
-  sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
-
-  if (sockFd < 0) {
-    uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  memset((char *)&serverAddr, 0, sizeof(serverAddr));
-  serverAddr.sun_family = AF_UNIX;
-  strcpy(serverAddr.sun_path + 1, name);
-
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
-
-  if (ret != 0) {
-    uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
-    sockFd = -1;
-  }
-
-  return sockFd;
-}
-
-int taosOpenUDServerSocket(char *ip, uint16_t port) {
-  struct sockaddr_un serverAdd;
-  int                sockFd;
-  char               name[128];
-
-  uTrace("open ud socket:%s", name);
-  sprintf(name, "%s.%hu", ip, port);
-
-  bzero((char *)&serverAdd, sizeof(serverAdd));
-  serverAdd.sun_family = AF_UNIX;
-  strcpy(serverAdd.sun_path + 1, name);
-  unlink(name);
-
-  if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
-    uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  /* bind socket to server address */
-  if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
-    uError("bind socket:%s failed, reason:%s", name, strerror(errno));
-    tclose(sockFd);
-    return -1;
-  }
-
-  if (listen(sockFd, 10) < 0) {
-    uError("listen socket:%s failed, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  return sockFd;
-}
-
 static void taosDeleteTimer(void *tharg) {
   timer_t *pTimer = tharg;
   timer_delete(*pTimer);

src/os/linux/src/linuxSysPara.c

@@ -225,10 +225,11 @@ static void taosGetSystemLocale() {  // get and set default locale
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
     locale = setlocale(LC_CTYPE, "");
     if (locale == NULL) {
-      uError("can't get locale from system");
+      uForcePrint("can't get locale from system, set it to en_US.UTF-8");
+      strcpy(tsLocale, "en_US.UTF-8");
     } else {
       strncpy(tsLocale, locale, tListLen(tsLocale));
-      uPrint("locale not configured, set to system default:%s", tsLocale);
+      uForcePrint("locale not configured, set to system default:%s", tsLocale);
     }
   }
 
@@ -243,7 +244,10 @@ static void taosGetSystemLocale() {  // get and set default locale
       strncpy(tsCharset, revisedCharset, tListLen(tsCharset));
 
       free(revisedCharset);
-      uPrint("charset not configured, set to system default:%s", tsCharset);
+      uForcePrint("charset not configured, set to system default:%s", tsCharset);
+    } else {
+      strcpy(tsCharset, "UTF-8");
+      uForcePrint("can't get locale and charset from system, set it to UTF-8");
     }
   }
 }

src/plugins/http/src/httpServer.c

@@ -508,7 +508,7 @@ void httpAcceptHttpConnection(void *arg) {
               pServer->serverPort);
     return;
   } else {
-    httpPrint("http service init success at ip:%s:%u", pServer->serverIp, pServer->serverPort);
+    httpPrint("http service init success at %u", pServer->serverPort);
     pServer->online = true;
   }
 

src/query/tests/astTest.cpp

@@ -582,7 +582,7 @@ void exprSerializeTest1() {
   tExprTreeDestroy(&p1, nullptr);
   tExprTreeDestroy(&p2, nullptr);
   
-  tbufClose(&bw);
+  // tbufClose(&bw);
 }
 
 void exprSerializeTest2() {
@@ -627,7 +627,7 @@ void exprSerializeTest2() {
   tExprTreeDestroy(&p1, nullptr);
   tExprTreeDestroy(&p2, nullptr);
 
-  tbufClose(&bw);
+  // tbufClose(&bw);
 }
 }  // namespace
 TEST(testCase, astTest) {

src/rpc/src/rpcMain.c

@@ -643,10 +643,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
 
   if (sid) {
     pConn = pRpc->connList + sid;
-  } else {
+    if (pConn->user[0] == 0) pConn = NULL;
-    pConn = rpcAllocateServerConn(pRpc, pRecv);
   } 
 
+  if (pConn == NULL) pConn = rpcAllocateServerConn(pRpc, pRecv);
+
   if (pConn) {
     if (pConn->linkUid != pHead->linkUid) {
       tTrace("%s %p, linkUid:0x%x not matched, received:0x%x", pRpc->label, pConn, pConn->linkUid, pHead->linkUid);

src/tsdb/inc/tsdbMain.h

@@ -153,17 +153,16 @@ typedef struct {
 } SCacheMem;
 
 typedef struct {
-  int              maxBytes;
   int              cacheBlockSize;
   int              totalCacheBlocks;
   STsdbCachePool   pool;
   STsdbCacheBlock *curBlock;
   SCacheMem *      mem;
   SCacheMem *      imem;
-  TsdbRepoT *    pRepo;
+  TsdbRepoT *      pRepo;
 } STsdbCache;
 
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo);
+STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo);
 void        tsdbFreeCache(STsdbCache *pCache);
 void *      tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
 
@@ -297,7 +296,7 @@ typedef struct {
 // TODO: take pre-calculation into account
 typedef struct {
   int16_t colId;  // Column ID
-  int16_t len;    // Column length
+  int16_t len;    // Column length // TODO: int16_t is not enough
   int32_t type : 8;
   int32_t offset : 24;
 } SCompCol;
@@ -426,6 +425,8 @@ typedef struct {
   SCompData *pCompData;
   SDataCols *pDataCols[2];
 
+  void *blockBuffer;  // Buffer to hold the whole data block
+  void *compBuffer;   // Buffer for temperary compress/decompress purpose
 } SRWHelper;
 
 // --------- Helper state
@@ -445,13 +446,11 @@ typedef struct {
 
 int  tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
 int  tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
-// int  tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
 void tsdbDestroyHelper(SRWHelper *pHelper);
 void tsdbResetHelper(SRWHelper *pHelper);
 
 // --------- For set operations
 int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
-// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
 void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
 int  tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
 

src/tsdb/src/tsdbCache.c

@@ -21,29 +21,25 @@ static int  tsdbAllocBlockFromPool(STsdbCache *pCache);
 static void tsdbFreeBlockList(SList *list);
 static void tsdbFreeCacheMem(SCacheMem *mem);
 
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
+STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo) {
   STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
   if (pCache == NULL) return NULL;
 
   if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
   cacheBlockSize *= (1024 * 1024);
 
-  if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS;
+  if (totalBlocks <= 1) totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
 
-  pCache->maxBytes = maxBytes;
   pCache->cacheBlockSize = cacheBlockSize;
+  pCache->totalCacheBlocks = totalBlocks;
   pCache->pRepo = pRepo;
 
-  int nBlocks = maxBytes / cacheBlockSize + 1;
-  if (nBlocks <= 1) nBlocks = 2;
-  pCache->totalCacheBlocks = nBlocks;
-
   STsdbCachePool *pPool = &(pCache->pool);
   pPool->index = 0;
   pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *));
   if (pPool->memPool == NULL) goto _err;
 
-  for (int i = 0; i < nBlocks; i++) {
+  for (int i = 0; i < totalBlocks; i++) {
     STsdbCacheBlock *pBlock = (STsdbCacheBlock *)malloc(sizeof(STsdbCacheBlock) + cacheBlockSize);
     if (pBlock == NULL) {
       goto _err;

src/tsdb/src/tsdbMain.c

@@ -5,6 +5,7 @@
 #include "tsdb.h"
 #include "tsdbMain.h"
 #include "tscompression.h"
+#include "tchecksum.h"
 
 #define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI  // default precision
 #define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
@@ -157,7 +158,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
   tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
   while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
     if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
-    for (int i = 0; i < pRepo->config.maxTables; i++) {
+    for (int i = 1; i < pRepo->config.maxTables; i++) {
       STable *  pTable = pMeta->tables[i];
       SCompIdx *pIdx = &rhelper.pCompIdx[i];
 
@@ -202,7 +203,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
     return NULL;
   }
 
-  pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo);
+  pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TsdbRepoT *)pRepo);
   if (pRepo->tsdbCache == NULL) {
     tsdbFreeMeta(pRepo->tsdbMeta);
     free(pRepo->rootDir);
@@ -259,7 +260,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   }
   pRepo->commit = 1;
   // Loop to move pData to iData
-  for (int i = 0; i < pRepo->config.maxTables; i++) {
+  for (int i = 1; i < pRepo->config.maxTables; i++) {
     STable *pTable = pRepo->tsdbMeta->tables[i];
     if (pTable != NULL && pTable->mem != NULL) {
       pTable->imem = pTable->mem;
@@ -312,7 +313,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
   }
   pRepo->commit = 1;
   // Loop to move pData to iData
-  for (int i = 0; i < pRepo->config.maxTables; i++) {
+  for (int i = 1; i < pRepo->config.maxTables; i++) {
     STable *pTable = pRepo->tsdbMeta->tables[i];
     if (pTable != NULL && pTable->mem != NULL) {
       pTable->imem = pTable->mem;
@@ -367,6 +368,15 @@ int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
   return 0;
 }
 
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+  STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
+  if (pTable == NULL) return -1;
+
+  return TSDB_GET_TABLE_LAST_KEY(pTable);
+}
+
 int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
   if (repo == NULL) return -1;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
@@ -601,6 +611,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) return -1;
   }
 
+  // Since tableId starts from 1, we increase maxTables by 1
+  // TODO: take a fancier way to do this
+  pCfg->maxTables++;
+
   // Check daysPerFile
   if (pCfg->daysPerFile == -1) {
     pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@@ -823,7 +837,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
 static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) {
   if (iters == NULL) return;
 
-  for (int tid = 0; tid < maxTables; tid++) {
+  for (int tid = 1; tid < maxTables; tid++) {
     if (iters[tid] == NULL) continue;
     tSkipListDestroyIter(iters[tid]);
   }
@@ -835,7 +849,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
   SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *));
   if (iters == NULL) return NULL;
 
-  for (int tid = 0; tid < maxTables; tid++) {
+  for (int tid = 1; tid < maxTables; tid++) {
     STable *pTable = pMeta->tables[tid];
     if (pTable == NULL || pTable->imem == NULL) continue;
 
@@ -902,7 +916,7 @@ _exit:
   free(pCache->imem);
   pCache->imem = NULL;
   pRepo->commit = 0;
-  for (int i = 0; i < pCfg->maxTables; i++) {
+  for (int i = 1; i < pCfg->maxTables; i++) {
     STable *pTable = pMeta->tables[i];
     if (pTable && pTable->imem) {
       tsdbFreeMemTable(pTable->imem);
@@ -936,7 +950,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
   if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
 
   // Loop to commit data in each table
-  for (int tid = 0; tid < pCfg->maxTables; tid++) {
+  for (int tid = 1; tid < pCfg->maxTables; tid++) {
     STable *           pTable = pMeta->tables[tid];
     if (pTable == NULL) continue;
 
@@ -974,7 +988,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
 
     // Write the SCompBlock part
     if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
- 
   }
 
   if (tsdbWriteCompIdx(pHelper) < 0) goto _err;

src/tsdb/src/tsdbMeta.c

@@ -129,7 +129,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
 void tsdbOrgMeta(void *pHandle) {
   STsdbMeta *pMeta = (STsdbMeta *)pHandle;
 
-  for (int i = 0; i < pMeta->maxTables; i++) {
+  for (int i = 1; i < pMeta->maxTables; i++) {
     STable *pTable = pMeta->tables[i];
     if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) {
       tsdbAddTableIntoIndex(pMeta, pTable);
@@ -179,7 +179,7 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 
   tsdbCloseMetaFile(pMeta->mfh);
 
-  for (int i = 0; i < pMeta->maxTables; i++) {
+  for (int i = 1; i < pMeta->maxTables; i++) {
     if (pMeta->tables[i] != NULL) {
       tsdbFreeTable(pMeta->tables[i]);
     }
@@ -246,7 +246,7 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t
   assert(pCol != NULL);
   
   SDataRow row = (SDataRow)pTable->tagVal;
-  char* d = dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
+  char* d = dataRowTuple(row);
   
   *val = d;
   *type  = pCol->type;
@@ -455,9 +455,8 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
   // Update the pMeta->maxCols and pMeta->maxRowBytes
   if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE) {
     if (schemaNCols(pTable->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pTable->schema);
-    int bytes = tdMaxRowBytesFromSchema(pTable->schema);
+    int bytes = dataRowMaxBytesFromSchema(pTable->schema);
     if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
-    tdUpdateSchema(pTable->schema);
   }
 
   return tsdbAddTableIntoMap(pMeta, pTable);
@@ -528,5 +527,5 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
 char *getTupleKey(const void * data) {
   SDataRow row = (SDataRow)data;
 
-  return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
+  return POINTER_DRIFT(row, TD_DATA_ROW_HEAD_SIZE);
 }

src/tsdb/src/tsdbRWHelper.c

@@ -131,6 +131,11 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t
   // Init block part
   if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
 
+  pHelper->blockBuffer =
+      tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols +
+              pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM));
+  if (pHelper->blockBuffer == NULL) goto _err;
+
   return 0;
 
 _err:
@@ -149,6 +154,8 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
 
 void tsdbDestroyHelper(SRWHelper *pHelper) {
   if (pHelper) {
+    tzfree(pHelper->blockBuffer);
+    tzfree(pHelper->compBuffer);
     tsdbDestroyHelperFile(pHelper);
     tsdbDestroyHelperTable(pHelper);
     tsdbDestroyHelperBlock(pHelper);
@@ -330,7 +337,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
     int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks);
 
     if (pCompBlock == NULL) {  // No key overlap, must has last block, just merge with the last block
-      ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last);
+      ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last);
       rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
       if (rowsToWrite < 0) goto _err;
     } else {  // Has key overlap
@@ -552,61 +559,97 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx,
   return 0;
 }
 
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfPoints,
+                                        int maxPoints, char *buffer, int bufferSize) {
+  // Verify by checksum
+  if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
+
+  // Decode the data
+  if (comp) {
+    // // Need to decompress
+    pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
+        content, len - sizeof(TSCKSUM), numOfPoints, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
+    if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+      pDataCol->len += (sizeof(int32_t) * maxPoints);
+      dataColSetOffset(pDataCol, numOfPoints);
+    }
+  } else {
+    // No need to decompress, just memcpy it
+    switch (pDataCol->type) {
+      case TSDB_DATA_TYPE_BINARY:
+      case TSDB_DATA_TYPE_NCHAR:
+        pDataCol->len = sizeof(int32_t) * maxPoints;
+        memcpy((char *)pDataCol->pData + pDataCol->len, content, len - sizeof(TSCKSUM));
+        pDataCol->len += (len - sizeof(TSCKSUM));
+        dataColSetOffset(pDataCol, numOfPoints);
+        break;
+
+      default:
+        pDataCol->len = len - sizeof(TSCKSUM);
+        memcpy(pDataCol->pData, content, pDataCol->len);
+        break;
+    }
+  }
+  return 0;
+}
+
 /**
  * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1)
  */
 static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
   ASSERT(pCompBlock->numOfSubBlocks <= 1);
 
-  SCompData *pCompData = (SCompData *)malloc(pCompBlock->len);
+  ASSERT(tsizeof(pHelper->blockBuffer) >= pCompBlock->len);
-  if (pCompData == NULL) return -1;
+
+  SCompData *pCompData = (SCompData *)pHelper->blockBuffer;
 
   int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
   if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
   if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err;
   ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
 
-  // TODO : check the checksum
+  int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
-  size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
   if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
-  for (int i = 0; i < pCompData->numOfCols; i++) {
-    // TODO: check the data checksum
-    // if (!taosCheckChecksumWhole())
-  }
-
-  ASSERT(pCompBlock->numOfCols == pCompData->numOfCols);
 
   pDataCols->numOfPoints = pCompBlock->numOfPoints;
 
-  int ccol = 0, dcol = 0;
+  // Recover the data
-  while (true) {
+  int ccol = 0;
-    if (ccol >= pDataCols->numOfCols) {
+  int dcol = 0;
-      // TODO: Fill rest NULL
+  while (dcol < pDataCols->numOfCols) {
-      break;
+    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+    if (ccol >= pCompData->numOfCols) {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dcol++;
+      continue;
     }
-    if (dcol >= pCompData->numOfCols) break;
 
     SCompCol *pCompCol = &(pCompData->cols[ccol]);
-    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
 
     if (pCompCol->colId == pDataCol->colId) {
-      // TODO: uncompress
+      if (pCompBlock->algorithm == TWO_STAGE_COMP) {
-      memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tsize + pCompCol->offset), pCompCol->len);
+        pHelper->compBuffer = trealloc(pHelper->compBuffer, pCompCol->len + COMP_OVERFLOW_BYTES);
+        if (pHelper->compBuffer == NULL) goto _err;
+      }
+      if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
+                                       pCompBlock->algorithm, pCompBlock->numOfPoints, pDataCols->maxPoints,
+                                       pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
+        goto _err;
+      dcol++;
+      ccol++;
+    } else if (pCompCol->colId < pDataCol->colId) {
       ccol++;
-      dcol++;
-    } else if (pCompCol->colId > pDataCol->colId) {
-      // TODO: Fill NULL
-      dcol++;
     } else {
-      ccol++;
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dcol++;
     }
   }
 
-  tfree(pCompData);
   return 0;
 
 _err:
-  tfree(pCompData);
   return -1;
 }
 
@@ -634,36 +677,6 @@ _err:
   return -1;
 }
 
-// static int tsdbCheckHelperCfg(SHelperCfg *pCfg) {
-//   // TODO
-//   return 0;
-// }
-
-// static void tsdbClearHelperFile(SHelperFile *pHFile) {
-//   pHFile->fid = -1;
-//   if (pHFile->headF.fd > 0) {
-//     close(pHFile->headF.fd);
-//     pHFile->headF.fd = -1;
-//   }
-//   if (pHFile->dataF.fd > 0) {
-//     close(pHFile->dataF.fd);
-//     pHFile->dataF.fd = -1;
-//   }
-//   if (pHFile->lastF.fd > 0) {
-//     close(pHFile->lastF.fd);
-//     pHFile->lastF.fd = -1;
-//   }
-//   if (pHFile->nHeadF.fd > 0) {
-//     close(pHFile->nHeadF.fd);
-//     pHFile->nHeadF.fd = -1;
-//   }
-//   if (pHFile->nLastF.fd > 0) {
-//     close(pHFile->nLastF.fd);
-//     pHFile->nLastF.fd = -1;
-//   }
-
-// }
-
 static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
   ASSERT(pHelper->files.lastF.fd > 0);
   struct stat st;
@@ -677,81 +690,94 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
   ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
          rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
 
-  SCompData *pCompData = NULL;
+  SCompData *pCompData = (SCompData *)(pHelper->blockBuffer);
   int64_t offset = 0;
 
   offset = lseek(pFile->fd, 0, SEEK_END);
   if (offset < 0) goto _err;
 
-  pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols + sizeof(TSCKSUM));
-  if (pCompData == NULL) goto _err;
-
   int nColsNotAllNull = 0;
-  int32_t toffset = 0;
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
     SDataCol *pDataCol = pDataCols->cols + ncol;
     SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
 
-    if (0) {
+    if (isNEleNull(pDataCol, rowsToWrite)) {
-      // TODO: all data to commit are NULL
+      // all data to commit are NULL, just ignore it
       continue;
     }
 
-    // Compress the data here
-    {
-      // TODO
-    }
-
     pCompCol->colId = pDataCol->colId;
     pCompCol->type = pDataCol->type;
-    pCompCol->len = TYPE_BYTES[pCompCol->type] * rowsToWrite; // TODO: change it
-    pCompCol->offset = toffset;
     nColsNotAllNull++;
-
-    toffset += pCompCol->len;
   }
 
   ASSERT(nColsNotAllNull > 0 && nColsNotAllNull <= pDataCols->numOfCols);
 
+  // Compress the data if neccessary
+  int     tcol = 0;
+  int32_t toffset = 0;
+  int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
+  int32_t lsize = tsize;
+  for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
+    if (tcol >= nColsNotAllNull) break;
+
+    SDataCol *pDataCol = pDataCols->cols + ncol;
+    SCompCol *pCompCol = pCompData->cols + tcol;
+
+    if (pDataCol->colId != pCompCol->colId) continue;
+    void *tptr = (void *)((char *)pCompData + lsize);
+
+    pCompCol->offset = toffset;
+
+    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
+
+    if (pHelper->config.compress) {
+      if (pHelper->config.compress == TWO_STAGE_COMP) {
+        pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES);
+        if (pHelper->compBuffer == NULL) goto _err;
+      }
+
+      pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
+          (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->blockBuffer) - lsize,
+          pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer));
+    } else {
+      pCompCol->len = tlen;
+      memcpy(tptr, pDataCol->pData, pCompCol->len);
+    }
+
+    // Add checksum
+    pCompCol->len += sizeof(TSCKSUM);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len);
+
+    toffset += pCompCol->len;
+    lsize += pCompCol->len;
+    tcol++;
+  }
+
   pCompData->delimiter = TSDB_FILE_DELIMITER;
   pCompData->uid = pHelper->tableInfo.uid;
   pCompData->numOfCols = nColsNotAllNull;
 
-  // Write SCompData + SCompCol part
-  size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
   taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
-  if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err;
-  // Write true data part
-  int nCompCol = 0;
-  for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
-    ASSERT(nCompCol < nColsNotAllNull);
 
-    SDataCol *pDataCol = pDataCols->cols + ncol;
+  // Write the whole block to file
-    SCompCol *pCompCol = pCompData->cols + nCompCol;
+  if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) goto _err;
-
-    if (pDataCol->colId == pCompCol->colId) {
-      if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err;
-      tsize += pCompCol->len;
-      nCompCol++;
-    }
-  }
 
+  // Update pCompBlock membership vairables
   pCompBlock->last = isLast;
   pCompBlock->offset = offset;
   pCompBlock->algorithm = pHelper->config.compress;
   pCompBlock->numOfPoints = rowsToWrite;
   pCompBlock->sversion = pHelper->tableInfo.sversion;
-  pCompBlock->len = (int32_t)tsize;
+  pCompBlock->len = (int32_t)lsize;
   pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
   pCompBlock->numOfCols = nColsNotAllNull;
   pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
   pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
 
-  tfree(pCompData);
   return 0;
 
   _err:
-  tfree(pCompData);
   return -1;
 }
 
@@ -782,7 +808,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   TSKEY keyFirst = dataColsKeyFirst(pDataCols);
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
-  ASSERT(blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx < pIdx->numOfBlocks);
 
   // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
   ASSERT(blockAtIdx(pHelper, blkIdx)->numOfSubBlocks >= 1);
@@ -790,7 +816,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
 
   if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
-    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfSuperBlocks-1);
+    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
     int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5;  // TODO: make a interface
 
     rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints);
@@ -961,7 +987,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
 static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
 
-  ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks);
   ASSERT(pCompBlock->numOfSubBlocks == 1);
 
   // Adjust memory if no more room
@@ -1004,7 +1030,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
   ASSERT(pCompBlock->numOfSubBlocks == 0);
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
-  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
 
   SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
   ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS);
@@ -1088,7 +1114,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
 
-  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
 
   SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
 

src/tsdb/tests/tsdbTests.cpp

@@ -27,7 +27,7 @@ typedef struct {
 
 static int insertData(SInsertInfo *pInfo) {
   SSubmitMsg *pMsg =
-      (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
+      (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
   if (pMsg == NULL) return -1;
   TSKEY start_time = pInfo->startTime;
 
@@ -52,11 +52,12 @@ static int insertData(SInsertInfo *pInfo) {
       tdInitDataRow(row, pInfo->pSchema);
 
       for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
+        STColumn *pTCol = schemaColAt(pInfo->pSchema, j);
         if (j == 0) {  // Just for timestamp
-          tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j));
+          tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset);
         } else {  // For int
           int val = 10;
-          tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j));
+          tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset);
         }
       }
       pBlock->len += dataRowLen(row);
@@ -105,9 +106,9 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
 
   for (int i = 0; i < nCols; i++) {
     if (i == 0) {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
     } else {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
     }
   }
 
@@ -149,9 +150,9 @@ TEST(TsdbTest, createRepo) {
 
   for (int i = 0; i < nCols; i++) {
     if (i == 0) {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
     } else {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
     }
   }
 
@@ -244,7 +245,7 @@ TEST(TsdbTest, DISABLED_openRepo) {
 //   tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
 
   // STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
-  // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
+  // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
   // tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
 
 //   tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);

src/util/CMakeLists.txt

@@ -11,7 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil pthread os m rt)
+  TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
     ADD_DEFINITIONS(-DUSE_LIBICONV)
@@ -68,7 +68,7 @@ ELSEIF (TD_WINDOWS_64)
   LIST(APPEND SRC ./src/tutil.c)
   LIST(APPEND SRC ./src/version.c)
   ADD_LIBRARY(tutil ${SRC}) 
-  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32)
+  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   LIST(APPEND SRC ./src/hash.c)
@@ -105,7 +105,7 @@ ELSEIF(TD_DARWIN_64)
   LIST(APPEND SRC ./src/version.c)
   LIST(APPEND SRC ./src/hash.c)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil iconv pthread os)
+  TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
 ENDIF()
 
 #  TARGET_LINK_LIBRARIES(tutil mstorage)

src/util/inc/tcache.h

@@ -69,6 +69,7 @@ typedef struct {
   SHashObj *   pHashTable;
   int          numOfElemsInTrash;  // number of element in trash
   int16_t      deleting;           // set the deleting flag to stop refreshing ASAP.
+  T_REF_DECLARE()
 
 #if defined(LINUX)
   pthread_rwlock_t lock;

src/util/inc/tscompression.h

@@ -21,7 +21,9 @@ extern "C" {
 #endif
 
 #include "taosdef.h"
+#include "tutil.h"
 
+#define COMP_OVERFLOW_BYTES 2
 #define BITS_PER_BYTE 8
 // Masks
 #define INT64MASK(_x) ((1ul << _x) - 1)
@@ -32,43 +34,220 @@ extern "C" {
 #define ONE_STAGE_COMP 1
 #define TWO_STAGE_COMP 2
 
-int tsCompressTinyint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
+extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-                      char* const buffer, int bufferSize);
+extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-int tsCompressSmallint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
+extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
-                       char* const buffer, int bufferSize);
+extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
-int tsCompressInt(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
+extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
-                  char* const buffer, int bufferSize);
+extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
-int tsCompressBigint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
+extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
-                     char* const buffer, int bufferSize);
+extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
-int tsCompressBool(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
+extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
-                   char* const buffer, int bufferSize);
+extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
-int tsCompressString(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
+extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
-                     char* const buffer, int bufferSize);
+extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
-int tsCompressFloat(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                    char* const buffer, int bufferSize);
-int tsCompressDouble(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                     char* const buffer, int bufferSize);
-int tsCompressTimestamp(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                        char* const buffer, int bufferSize);
 
-int tsDecompressTinyint(const char* const input, int compressedSize, const int nelements, char* const output,
+static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                        int outputSize, char algorithm, char* const buffer, int bufferSize);
+                      char *const buffer, int bufferSize) {
-int tsDecompressSmallint(const char* const input, int compressedSize, const int nelements, char* const output,
+  if (algorithm == ONE_STAGE_COMP) {
-                         int outputSize, char algorithm, char* const buffer, int bufferSize);
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-int tsDecompressInt(const char* const input, int compressedSize, const int nelements, char* const output, int outputSize,
+  } else if (algorithm == TWO_STAGE_COMP) {
-                    char algorithm, char* const buffer, int bufferSize);
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
-int tsDecompressBigint(const char* const input, int compressedSize, const int nelements, char* const output,
+    return tsCompressStringImp(buffer, len, output, outputSize);
-                       int outputSize, char algorithm, char* const buffer, int bufferSize);
+  } else {
-int tsDecompressBool(const char* const input, int compressedSize, const int nelements, char* const output,
+    assert(0);
-                     int outputSize, char algorithm, char* const buffer, int bufferSize);
+  }
-int tsDecompressString(const char* const input, int compressedSize, const int nelements, char* const output,
+}
-                       int outputSize, char algorithm, char* const buffer, int bufferSize);
+
-int tsDecompressFloat(const char* const input, int compressedSize, const int nelements, char* const output,
+static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
-                      int outputSize, char algorithm, char* const buffer, int bufferSize);
+                        int outputSize, char algorithm, char *const buffer, int bufferSize) {
-int tsDecompressDouble(const char* const input, int compressedSize, const int nelements, char* const output,
+  if (algorithm == ONE_STAGE_COMP) {
-                       int outputSize, char algorith, char* const buffer, int bufferSize);
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-int tsDecompressTimestamp(const char* const input, int compressedSize, const int nelements, char* const output,
+  } else if (algorithm == TWO_STAGE_COMP) {
-                          int outputSize, char algorithm, char* const buffer, int bufferSize);
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
+                       char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
+                         int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
+                  char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
+                    int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, 
+                   char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressBoolImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressBoolImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
+                     int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressBoolImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressBoolImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  return tsCompressStringImp(input, inputSize, output, outputSize);
+}
+
+static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  return tsDecompressStringImp(input, compressedSize, output, outputSize);
+}
+
+static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                    char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressFloatImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressFloatImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
+                      int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressFloatImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressFloatImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressDoubleImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressDoubleImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressDoubleImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressDoubleImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                        char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressTimestampImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressTimestampImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
+                          int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressTimestampImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressTimestampImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
 
 #ifdef __cplusplus
 }

src/util/inc/tsocket.h

@@ -33,10 +33,6 @@ int  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int  taosKeepTcpAlive(int sockFd);
 void taosCloseTcpSocket(int sockFd);
 
-int taosOpenUDServerSocket(uint32_t ip, uint16_t port);
-int taosOpenUDClientSocket(uint32_t ip, uint16_t port);
-int taosOpenRawSocket(uint32_t ip);
-
 int      taosGetFqdn(char *);
 uint32_t taosGetIpFromFqdn(const char *);
 void     tinet_ntoa(char *ipstr, unsigned int ip);

src/util/inc/tutil.h

@@ -44,7 +44,10 @@ extern "C" {
 
 #define tclose(x) taosCloseSocket(x)
 
-#ifdef ASSERTION
+// Pointer p drift right by b bytes
+#define POINTER_DRIFT(p, b) ((void *)((char *)(p) + (b)))
+
+#ifndef NDEBUG
 #define ASSERT(x) assert(x)
 #else
 #define ASSERT(x)

src/util/src/hash.c

@@ -520,8 +520,8 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
 
 static SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
   assert(pIter != NULL);
-  pIter->entryIndex += 1;
 
+  pIter->entryIndex++;
   while (pIter->entryIndex < pIter->pHashObj->capacity) {
     SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
     if (pEntry->next == NULL) {

src/util/src/tcache.c

@@ -324,13 +324,26 @@ static void doCleanupDataCache(SCacheObj *pCacheObj) {
 static void taosCacheRefresh(void *handle, void *tmrId) {
   SCacheObj *pCacheObj = (SCacheObj *)handle;
   
-  if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) {
+  if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) {
     uTrace("object is destroyed. no refresh retry");
     return;
   }
   
-  if (pCacheObj->deleting == 1) {
+  int16_t ref = T_REF_INC(pCacheObj);
-    doCleanupDataCache(pCacheObj);
+  if (ref == 1) {
+    T_REF_DEC(pCacheObj);
+    return;
+  }
+  
+  // todo add the ref before start the timer
+  int32_t num = taosHashGetSize(pCacheObj->pHashTable);
+  if (num == 0) {
+    ref = T_REF_DEC(pCacheObj);
+    if (ref == 0) {
+      doCleanupDataCache(pCacheObj);
+    } else {
+      taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
+    }
     return;
   }
   
@@ -356,12 +369,15 @@ static void taosCacheRefresh(void *handle, void *tmrId) {
   
   taosHashDestroyIter(pIter);
 
-  if (pCacheObj->deleting == 1) {  // clean up resources and abort
-    doCleanupDataCache(pCacheObj);
-  } else {
     taosTrashEmpty(pCacheObj, false);
-    taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
+    
-  }
+    ref = T_REF_DEC(pCacheObj);
+    if (ref == 0)  {
+      doCleanupDataCache(pCacheObj);
+      return;
+    } else {
+      taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
+    }
 }
 
 SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
@@ -399,6 +415,7 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
     return NULL;
   }
   
+  T_REF_INC(pCacheObj);
   return pCacheObj;
 }
 
@@ -552,5 +569,8 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
     return;
   }
   
-  pCacheObj->deleting = 1;
+  int32_t ref = T_REF_DEC(pCacheObj);
+  if (ref == 0) {
+    doCleanupDataCache(pCacheObj);
+  }
 }

src/util/src/tcompression.c

@@ -56,223 +56,6 @@ const int TEST_NUMBER = 1;
 #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
 #define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L)
 
-// Function declarations
-int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
-int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
-int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
-int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
-int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
-int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
-int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
-int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
-int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
-int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
-
-/* ----------------------------------------------Compression function used by
- * others ---------------------------------------------- */
-int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                      char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
-                        int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                       char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
-                         int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                  char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
-                    int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, 
-                   char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressBoolImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressBoolImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
-                     int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressBoolImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressBoolImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  return tsCompressStringImp(input, inputSize, output, outputSize);
-}
-
-int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  return tsDecompressStringImp(input, compressedSize, output, outputSize);
-}
-
-int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                    char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressFloatImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressFloatImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
-                      int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressFloatImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressFloatImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressDoubleImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressDoubleImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressDoubleImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressDoubleImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                        char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressTimestampImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressTimestampImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
-                          int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressTimestampImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressTimestampImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
 bool safeInt64Add(int64_t a, int64_t b) {
   if ((a > 0 && b > INT64_MAX - a) || (a < 0 && b < INT64_MIN - a)) return false;
   return true;

src/util/src/tsocket.c

@@ -14,7 +14,6 @@
  */
 
 #include "os.h"
-#include "tglobal.h"
 #include "tulog.h"
 #include "tsocket.h"
 #include "tutil.h"
@@ -26,8 +25,13 @@ int taosGetFqdn(char *fqdn) {
 
   struct hostent* h;
   h = gethostbyname(hostname);
-  strcpy(fqdn, h->h_name);
+  if (h != NULL) {
-  return 0;
+    strcpy(fqdn, h->h_name);
+    return 0;
+  } else {
+    uError("failed to get host name");
+    return -1;
+  }
 }
 
 uint32_t taosGetIpFromFqdn(const char *fqdn) {
@@ -394,38 +398,6 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
   return sockFd;
 }
 
-int taosOpenRawSocket(uint32_t ip) {
-  int                fd, hold;
-  struct sockaddr_in rawAdd;
-
-  uTrace("open udp raw socket:%s", ip);
-
-  fd = (int)socket(AF_INET, SOCK_RAW, IPPROTO_UDP);
-  if (fd < 0) {
-    uError("failed to open raw socket: %d (%s)", errno, strerror(errno));
-    return -1;
-  }
-
-  hold = 1;
-  if (taosSetSockOpt(fd, IPPROTO_IP, IP_HDRINCL, (void *)&hold, sizeof(hold)) < 0) {
-    uError("failed to set hold option: %d (%s)", errno, strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  bzero((char *)&rawAdd, sizeof(rawAdd));
-  rawAdd.sin_family = AF_INET;
-  rawAdd.sin_addr.s_addr = ip;
-
-  if (bind(fd, (struct sockaddr *)&rawAdd, sizeof(rawAdd)) < 0) {
-    uError("failed to bind RAW socket:(%s)", strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  return fd;
-}
-
 void tinet_ntoa(char *ipstr, unsigned int ip) {
   sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
 }

src/util/tests/CMakeLists.txt

@@ -10,6 +10,7 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
     INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
     AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-    ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
+#    ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
+    ADD_EXECUTABLE(utilTest hashTest.cpp cacheTest.cpp)
     TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread)
 ENDIF()

src/util/tests/cacheTest.cpp

@@ -25,7 +25,7 @@ TEST(testCase, client_cache_test) {
   const char* key1 = "test1";
   char data1[] = "test11";
 
-  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1);
+  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1)+1, 1);
   sleep(REFRESH_TIME_IN_SEC+1);
 
   printf("obj is still valid: %s\n", cachedObj);
@@ -34,19 +34,17 @@ TEST(testCase, client_cache_test) {
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
 
   /* the object is cleared by cache clean operation */
-  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20);
+  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2)+1, 20);
   printf("after updated: %s\n", cachedObj);
 
   printf("start to remove data from cache\n");
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
   printf("end of removing data from cache\n");
 
-  getchar();
-
   const char* key3 = "test2";
   const char* data3 = "kkkkkkk";
 
-  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1);
+  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3) + 1, 1);
   printf("%s\n", cachedObj2);
 
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
@@ -57,18 +55,18 @@ TEST(testCase, client_cache_test) {
 
   char key5[] = "test5";
   char data5[] = "data5kkkkk";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5) + 1, 20);
 
   const char* data6= "new Data after updated";
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
 
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6) + 1, 20);
   printf("%s\n", cachedObj2);
 
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
 
   const char* data7 = "add call update procedure";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7) + 1, 20);
   printf("%s\n=======================================\n\n", cachedObj2);
 
   char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5);
@@ -137,6 +135,4 @@ TEST(testCase, cache_resize_test) {
   printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num);
 
   taosCacheCleanup(pCache);
-  taosMsleep(20000);
-  getchar();
 }

src/vnode/src/vnodeMain.c

@@ -96,14 +96,16 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
   }
 
   STsdbCfg tsdbCfg = {0};
-  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
-  tsdbCfg.compression         = pVnodeCfg->cfg.compression;;
   tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
+  tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
+  tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
   tsdbCfg.maxTables           = pVnodeCfg->cfg.maxTables;
   tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
+  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
   tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
   tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
-  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
+  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
+  tsdbCfg.compression         = pVnodeCfg->cfg.compression;;
   
   char tsdbDir[TSDB_FILENAME_LEN] = {0};
   sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);

src/vnode/src/vnodeWrite.c

@@ -123,7 +123,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
 
   STSchema *pDestSchema = tdNewSchema(numOfColumns);
   for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+    tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
   tsdbTableSetName(&tCfg, pTable->tableId, false);
@@ -131,7 +131,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
   if (numOfTags != 0) {
     STSchema *pDestTagSchema = tdNewSchema(numOfTags);
     for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+      tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
     tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
     tsdbTableSetSName(&tCfg, pTable->superTableId, false);
@@ -142,7 +142,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
     for (int i = 0; i < numOfTags; i++) {
-      tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
+      STColumn *pTCol = schemaColAt(pDestSchema, i);
+      tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
       accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);
@@ -189,14 +190,14 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 
   STSchema *pDestSchema = tdNewSchema(numOfColumns);
   for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+    tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
 
   if (numOfTags != 0) {
     STSchema *pDestTagSchema = tdNewSchema(numOfTags);
     for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+      tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
     tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
 
@@ -205,7 +206,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
     for (int i = 0; i < numOfTags; i++) {
-      tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
+      STColumn *pTCol = schemaColAt(pDestTagSchema, i);
+      tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
       accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);

tests/pytest/smoketest.sh

@@ -23,6 +23,17 @@ sleep 1
 python3 ./test.py $1 -f insert/tinyint.py
 python3 ./test.py -s $1
 sleep 1
+
+python3 ./test.py $1 -f table/column_name.py
+python3 ./test.py -s $1
+sleep 1
+python3 ./test.py $1 -f table/column_num.py
+python3 ./test.py -s $1
+sleep 1
+python3 ./test.py $1 -f table/db_table.py
+python3 ./test.py -s $1
+sleep 1
+
 python3 ./test.py $1 -f import_merge/importDataLastTO.py 
 python3 ./test.py -s $1
 sleep 1

tests/pytest/table/column_name.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        # TSIM: system sh/stop_dnodes.sh
+        # TSIM:
+        # TSIM: system sh/ip.sh -i 1 -s up
+        # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+        # TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0
+        # TSIM: system sh/exec.sh -n dnode1 -s start
+        # TSIM:
+        # TSIM: sleep 3000
+        # TSIM: sql connect
+        # TSIM:
+        # TSIM: $i = 0
+        # TSIM: $dbPrefix = lm_cm_db
+        # TSIM: $tbPrefix = lm_cm_tb
+        # TSIM: $db = $dbPrefix . $i
+        # TSIM: $tb = $tbPrefix . $i
+        # TSIM:
+        # TSIM: print =============== step1
+        tdLog.info('=============== step1')
+        # TSIM: sql create database $db
+        # TSIM: sql use $db
+        # TSIM:
+        # TSIM: sql drop table dd -x step0
+        tdLog.info('drop table dd -x step0')
+        tdSql.error('drop table dd')
+        # TSIM: return -1
+        # TSIM: step0:
+        # TSIM:
+        # TSIM: sql create table $tb(ts timestamp, int) -x step1
+        tdLog.info('create table tb(ts timestamp, int) -x step1')
+        tdSql.error('create table tb(ts timestamp, int)')
+        # TSIM: return -1
+        # TSIM: step1:
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step2
+        tdLog.info('=============== step2')
+        # TSIM: sql create table $tb (ts timestamp, s int)
+        tdLog.info('create table tb (ts timestamp, s int)')
+        tdSql.execute('create table tb (ts timestamp, s int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step3
+        tdLog.info('=============== step3')
+        # TSIM: sql create table $tb (ts timestamp, a0123456789 int)
+        tdLog.info('create table tb (ts timestamp, a0123456789 int)')
+        tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step4
+        tdLog.info('=============== step4')
+        # TSIM: sql create table $tb (ts timestamp,
+        # a0123456789012345678901234567890123456789 int)
+        tdLog.info(
+            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+        tdSql.execute(
+            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step5
+        tdLog.info('=============== step5')
+        # TSIM: sql create table $tb (ts timestamp, a0123456789 int)
+        tdLog.info('create table tb (ts timestamp, a0123456789 int)')
+        tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql insert into $tb values (now , 1)
+        tdLog.info("insert into tb values (now , 1)")
+        tdSql.execute("insert into tb values (now , 1)")
+        # TSIM: sql select * from $tb
+        tdLog.info('select * from tb')
+        tdSql.query('select * from tb')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop database $db
+        tdLog.info('drop database db')
+        tdSql.execute('drop database db')
+        # TSIM: sql show databases
+        tdLog.info('show databases')
+        tdSql.query('show databases')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM:
+        # TSIM:
+        # TSIM:
+# convert end
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/table/column_num.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        # TSIM: system sh/stop_dnodes.sh
+        # TSIM:
+        # TSIM: system sh/ip.sh -i 1 -s up
+        # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+        # TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0
+        # TSIM: system sh/exec.sh -n dnode1 -s start
+        # TSIM:
+        # TSIM: sleep 3000
+        # TSIM: sql connect
+        # TSIM:
+        # TSIM: $i = 0
+        # TSIM: $dbPrefix = lm_cn_db
+        # TSIM: $tbPrefix = lm_cn_tb
+        # TSIM: $db = $dbPrefix . $i
+        # TSIM: $tb = $tbPrefix . $i
+        # TSIM:
+        # TSIM: print =============== step1
+        tdLog.info('=============== step1')
+        # TSIM: sql create database $db
+        # TSIM: sql use $db
+        # TSIM:
+        # TSIM: sql create table $tb() -x step1
+        tdLog.info('create table tb() -x step1')
+        tdSql.error('create table tb()')
+        # TSIM: return -1
+        # TSIM: step1:
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step2
+        tdLog.info('=============== step2')
+        # TSIM: sql create table $tb (ts timestamp)  -x step2
+        tdLog.info('create table tb (ts timestamp)  -x step2')
+        tdSql.error('create table tb (ts timestamp) ')
+        # TSIM: return -1
+        # TSIM: step2:
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step3
+        tdLog.info('=============== step3')
+        # TSIM: sql create table $tb (ts int)  -x step3
+        tdLog.info('create table tb (ts int)  -x step3')
+        tdSql.error('create table tb (ts int) ')
+        # TSIM: return -1
+        # TSIM: step3:
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step4
+        tdLog.info('=============== step4')
+        # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4
+        # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12
+        # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int,
+        # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27
+        # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int,
+        # b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12
+        # int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int,
+        # b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27
+        # int, b28 int,b29 int,b30 int,b31 int,b32 int)
+        tdLog.info('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)')
+        tdSql.execute('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)')
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step5
+        tdLog.info('=============== step5')
+        # TSIM: $i = 1
+        # TSIM: $tb = $tbPrefix . $i
+        # TSIM:
+        # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4
+        # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12
+        # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int,
+        # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27
+        # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int,
+        # b4 int, b5 int)
+        tdLog.info('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)')
+        tdSql.execute('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)')
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 2 then
+        tdLog.info('tdSql.checkRow(2)')
+        tdSql.checkRows(2)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql insert into $tb values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8
+        # , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22
+        # , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)
+        tdLog.info("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)")
+        tdSql.execute("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)")
+        # TSIM: sql select * from $tb
+        tdLog.info('select * from tb1')
+        tdSql.query('select * from tb1')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb1')
+        tdSql.execute('drop table tb1')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop database $db
+        tdLog.info('drop database db')
+        tdSql.execute('drop database db')
+        # TSIM: sql show databases
+        tdLog.info('show databases')
+        tdSql.query('show databases')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+# convert end
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/table/db_table.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.info('=============== step1')
+        tdLog.info('=============== step2')
+        tdLog.info('create table tb (ts timestamp, speed int)')
+        tdSql.execute('create table tb (ts timestamp, speed int)')
+        tdLog.info('=============== step3')
+        tdLog.info("insert into tb values (now, 1)")
+        tdSql.execute("insert into tb values (now, 1)")
+        tdLog.info('=============== step4')
+        tdLog.info('select * from tb')
+        tdSql.query('select * from tb')
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        tdLog.info('tdSql.checkData(0, 1, 1)')
+        tdSql.checkData(0, 1, 1)
+        tdLog.info('=============== step5')
+        tdLog.info('describe tb')
+        tdSql.query('describe tb')
+        tdLog.info('=============== step6')
+        tdLog.info('drop database db')
+        tdSql.execute('drop database db')
+        tdLog.info('show databases')
+        tdSql.query('show databases')
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+# convert end
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/util/log.py

@@ -23,14 +23,14 @@ class TDLog:
         self.path = ""
 
     def info(self, info):
-        printf("%s %s" % (datetime.datetime.now(), info))
+        print("%s %s" % (datetime.datetime.now(), info))
 
     def sleep(self, sec):
-        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
+        print("%s sleep %d seconds" % (datetime.datetime.now(), sec))
         time.sleep(sec)
 
     def debug(self, err):
-        printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
+        print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
 
     def success(self, info):
         printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
@@ -43,7 +43,7 @@ class TDLog:
         sys.exit(1)
 
     def printNoPrefix(self, info):
-        printf("\033[1;36m%s\033[0m" % (info))
+        print("\033[1;36m%s\033[0m" % (info))
 
 
 tdLog = TDLog()

tests/script/basicSuite.sim

@@ -1,8 +1,28 @@
 ################################# 
+run general/db/basic1.sim
+run general/db/basic2.sim
+run general/db/basic3.sim
+run general/db/basic4.sim
+run general/db/basic5.sim
 
-run general/db/testSuite.sim
+run general/table/basic1.sim
-run general/insert/testSuite.sim
+run general/table/basic2.sim
-run general/table/testSuite.sim
+run general/table/basic3.sim
-run general/user/basicSuite.sim
+run general/table/column_num.sim
+run general/table/column_name.sim
+run general/table/bigint.sim
+run general/table/bool.sim
+run general/table/double.sim
+run general/table/float.sim
+run general/table/int.sim
+run general/table/smallint.sim
+run general/table/tinyint.sim
+run general/table/db.table.sim
+
+run general/user/basic1.sim
+run general/user/pass_alter.sim
+run general/user/pass_len.sim
+run general/user/user_create.sim
+run general/user/user_len.sim
 
 ################################## 

tests/script/general/agg/fill.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/agg/stream.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 
 print ========== step1

tests/script/general/alter/cached_schema_after_alter.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/alter/count.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1

tests/script/general/alter/import.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1

tests/script/general/alter/insert1.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/alter/insert2.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/alter/metrics.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/alter/table.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/cache/cache_balance.sim

@@ -1,8 +1,8 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/ip.sh -i 2 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
-system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2
+system sh/deploy.sh -n dnode2 -i 2
 
 system sh/cfg.sh -n dnode1 -c commitLog           -v 0
 system sh/cfg.sh -n dnode2 -c commitLog           -v 0

tests/script/general/cache/new_metrics.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10

tests/script/general/cache/new_stream.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10

tests/script/general/cache/restart_metrics.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10
@@ -47,7 +47,7 @@ endi
 
 print =============== step2
 system sh/exec.sh -n dnode1 -s stop
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10

tests/script/general/cache/restart_stream.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 1
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 1
@@ -97,7 +97,7 @@ endi
 
 print =============== step4
 system sh/exec.sh -n dnode1 -s stop
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 1
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 1

tests/script/general/cache/restart_table.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10
@@ -35,7 +35,7 @@ endi
 
 print =============== step2
 system sh/exec.sh -n dnode1 -s stop
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c meterMetaKeepTimer -v 10
 system sh/cfg.sh -n dnode1 -c metricMetaKeepTimer -v 10

tests/script/general/column/commit.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/column/metrics.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/column/stream.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 
 print ========== step1

tests/script/general/column/table.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/compress/commitlog.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 1
 system sh/cfg.sh -n dnode1 -c cacheBlockSize -v 1024
 system sh/cfg.sh -n dnode1 -c compression -v 1

tests/script/general/compress/compress.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c cacheBlockSize -v 1024
 system sh/cfg.sh -n dnode1 -c compression -v 1

tests/script/general/compress/compress2.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c cacheBlockSize -v 1024
 system sh/cfg.sh -n dnode1 -c compression -v 2

tests/script/general/compress/uncompress.sim

@@ -1,7 +1,7 @@
 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/cfg.sh -n dnode1 -c cacheBlockSize -v 1024
 system sh/cfg.sh -n dnode1 -c compression -v 1

tests/script/general/compute/avg.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/compute/bottom.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/compute/count.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/compute/diff.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/compute/diff2.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/compute/first.sim

@@ -1,6 +1,6 @@
 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
+
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000