Merge branch '3.0' into test/3.0_math_functions

.gitignore

@@ -40,6 +40,7 @@ taosdalipu/
 Target/
 *.failed
 *.sql
+sim
 sim/
 psim/
 pysim/
@@ -107,3 +108,4 @@ TAGS
 contrib/*
 !contrib/CMakeLists.txt
 !contrib/test
+sql

example/src/tmq.c

@@ -18,7 +18,6 @@
 #include <string.h>
 #include <time.h>
 #include "taos.h"
-#include "osSleep.h"
 
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {

include/client/taos.h

@@ -93,14 +93,14 @@ typedef struct taosField {
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-typedef struct TAOS_BIND_v2 {
+typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
   void     *buffer;
-  int32_t  buffer_length;
+  uintptr_t buffer_length;
   int32_t  *length;
   char     *is_null;
   int       num;
-} TAOS_BIND_v2;
+} TAOS_MULTI_BIND;
 
 typedef enum {
   SET_CONF_RET_SUCC = 0,
@@ -131,16 +131,16 @@ const char *taos_data_type(int type);
 
 DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
 DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags);
+DLL_EXPORT int         taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
 DLL_EXPORT int         taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
 DLL_EXPORT int         taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
 DLL_EXPORT int         taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
 DLL_EXPORT int         taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
 DLL_EXPORT int         taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int         taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int         taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx);
+DLL_EXPORT int         taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
 DLL_EXPORT int         taos_stmt_add_batch(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);

include/common/tcommon.h

@@ -99,6 +99,15 @@ typedef struct SColumnInfoData {
   };
 } SColumnInfoData;
 
+typedef struct SQueryTableDataCond {
+  STimeWindow  twindow;
+  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} SQueryTableDataCond;
+
 void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
@@ -122,59 +131,6 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
 
 static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
 
-static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
-  int32_t tlen = 0;
-  int32_t sz = 0;
-  // tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
-  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
-  tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return tlen;
-  tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema);
-  if (pRsp->pBlockData) {
-    sz = taosArrayGetSize(pRsp->pBlockData);
-  }
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
-    tlen += tEncodeDataBlock(buf, pBlock);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
-  int32_t sz;
-  // buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
-  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
-  buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
-  if (pRsp->numOfTopics == 0) return buf;
-  pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-  if (pRsp->schema == NULL) return NULL;
-  buf = taosDecodeSSchemaWrapper(buf, pRsp->schema);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock block = {0};
-    tDecodeDataBlock(buf, &block);
-    taosArrayPush(pRsp->pBlockData, &block);
-  }
-  return buf;
-}
-
-static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
-  if (pRsp->schema) {
-    if (pRsp->schema->nCols) {
-      taosMemoryFreeClear(pRsp->schema->pSchema);
-    }
-    taosMemoryFree(pRsp->schema);
-  }
-  taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
-  pRsp->pBlockData = NULL;
-}
-
 //======================================================================================================================
 // the following structure shared by parser and executor
 typedef struct SColumn {
@@ -205,12 +161,12 @@ typedef struct STableBlockDistInfo {
   int32_t  firstSeekTimeUs;
   uint32_t numOfRowsInMemTable;
   uint32_t numOfSmallBlocks;
-  SArray   *dataBlockInfos;
+  SArray*  dataBlockInfos;
 } STableBlockDistInfo;
 
 enum {
   FUNC_PARAM_TYPE_VALUE = 0x1,
-  FUNC_PARAM_TYPE_COLUMN= 0x2,
+  FUNC_PARAM_TYPE_COLUMN = 0x2,
 };
 
 typedef struct SFunctParam {
@@ -229,7 +185,6 @@ typedef struct SResSchame {
   char    name[TSDB_COL_NAME_LEN];
 } SResSchema;
 
-// TODO move away to executor.h
 typedef struct SExprBasicInfo {
   SResSchema   resSchema;
   int16_t      numOfParams;  // argument value of each function

include/common/tmsg.h

@@ -177,6 +177,7 @@ typedef struct SField {
   char    name[TSDB_COL_NAME_LEN];
   uint8_t type;
   int32_t bytes;
+  int8_t  flags;
 } SField;
 
 typedef struct SRetention {
@@ -286,6 +287,12 @@ typedef struct SSchema {
 #define SSCHMEA_BYTES(s) ((s)->bytes)
 #define SSCHMEA_NAME(s)  ((s)->name)
 
+typedef struct {
+  int32_t  nCols;
+  int32_t  sver;
+  SSchema* pSchema;
+} SSchemaWrapper;
+
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
 
 typedef struct {
@@ -296,13 +303,11 @@ typedef struct {
   int32_t ttl;
   int32_t numOfColumns;
   int32_t numOfTags;
-  int32_t numOfSmas;
   int32_t commentLen;
   int32_t ast1Len;
   int32_t ast2Len;
   SArray* pColumns;  // array of SField
   SArray* pTags;     // array of SField
-  SArray* pSmas;     // array of SField
   char*   comment;
   char*   pAst1;
   char*   pAst2;
@@ -1515,59 +1520,81 @@ typedef struct {
   char*   qmsg2;  // pAst2:qmsg2:SRetention2 => trigger aggr task2
 } SRSmaParam;
 
+int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
+int tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
+
+typedef struct SVCreateStbReq {
+  const char*    name;
+  tb_uid_t       suid;
+  int8_t         rollup;
+  SSchemaWrapper schema;
+  SSchemaWrapper schemaTag;
+  SRSmaParam     pRSmaParam;
+} SVCreateStbReq;
+
+int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq);
+int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq);
+
+typedef struct SVDropStbReq {
+  // data
+} SVDropStbReq;
+
+typedef struct SVCreateStbRsp {
+  int code;
+} SVCreateStbRsp;
+
 typedef struct SVCreateTbReq {
-  char*    name;
+  tb_uid_t    uid;
-  uint32_t ttl;
+  int64_t     ctime;
-  uint32_t keep;
+  const char* name;
-  union {
+  int32_t     ttl;
-    uint8_t info;
+  int8_t      type;
-    struct {
-      uint8_t rollup : 1;  // 1 means rollup sma
-      uint8_t type : 7;
-    };
-  };
   union {
     struct {
       tb_uid_t    suid;
-      col_id_t    nCols;
+      const void* pTag;
-      col_id_t    nBSmaCols;
+    } ctb;
-      SSchema*    pSchema;
-      col_id_t    nTagCols;
-      SSchema*    pTagSchema;
-      SRSmaParam* pRSmaParam;
-    } stbCfg;
     struct {
-      tb_uid_t suid;
+      SSchemaWrapper schema;
-      SKVRow   pTag;
+    } ntb;
-    } ctbCfg;
-    struct {
-      col_id_t    nCols;
-      col_id_t    nBSmaCols;
-      SSchema*    pSchema;
-      SRSmaParam* pRSmaParam;
-    } ntbCfg;
   };
-} SVCreateTbReq, SVUpdateTbReq;
+} SVCreateTbReq;
+
+int tEncodeSVCreateTbReq(SCoder* pCoder, const SVCreateTbReq* pReq);
+int tDecodeSVCreateTbReq(SCoder* pCoder, SVCreateTbReq* pReq);
+
+typedef struct {
+  int32_t nReqs;
+  union {
+    SVCreateTbReq* pReqs;
+    SArray*        pArray;
+  };
+} SVCreateTbBatchReq;
+
+int tEncodeSVCreateTbBatchReq(SCoder* pCoder, const SVCreateTbBatchReq* pReq);
+int tDecodeSVCreateTbBatchReq(SCoder* pCoder, SVCreateTbBatchReq* pReq);
 
 typedef struct {
   int32_t code;
 } SVCreateTbRsp, SVUpdateTbRsp;
 
+int tEncodeSVCreateTbRsp(SCoder* pCoder, const SVCreateTbRsp* pRsp);
+int tDecodeSVCreateTbRsp(SCoder* pCoder, SVCreateTbRsp* pRsp);
+
 int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
 void*   tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
 
 typedef struct {
-  int64_t ver;  // use a general definition
+  int32_t nRsps;
+  union {
+    SVCreateTbRsp* pRsps;
     SArray*        pArray;
-} SVCreateTbBatchReq;
+  };
-
-int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
-void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
-
-typedef struct {
-  SArray* rspList;  // SArray<SVCreateTbRsp>
 } SVCreateTbBatchRsp;
 
+int tEncodeSVCreateTbBatchRsp(SCoder* pCoder, const SVCreateTbBatchRsp* pRsp);
+int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp);
+
 int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
 int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
 
@@ -1622,103 +1649,11 @@ typedef struct {
   char    data[];
 } SVShowTablesFetchRsp;
 
-typedef struct SMqCMGetSubEpReq {
+typedef struct {
   int64_t consumerId;
   int32_t epoch;
   char    cgroup[TSDB_CGROUP_LEN];
-} SMqCMGetSubEpReq;
+} SMqAskEpReq;
-
-static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI32(buf, pMsg->contLen);
-  tlen += taosEncodeFixedI32(buf, pMsg->vgId);
-  return tlen;
-}
-
-typedef struct SMqHbRsp {
-  int8_t status;  // idle or not
-  int8_t vnodeChanged;
-  int8_t epChanged;  // should use new epset
-  int8_t reserved;
-  SEpSet epSet;
-} SMqHbRsp;
-
-static FORCE_INLINE int32_t taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI8(buf, pRsp->status);
-  tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
-  tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
-  tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
-  return tlen;
-}
-
-static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) {
-  buf = taosDecodeFixedI8(buf, &pRsp->status);
-  buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged);
-  buf = taosDecodeFixedI8(buf, &pRsp->epChanged);
-  buf = taosDecodeSEpSet(buf, &pRsp->epSet);
-  return buf;
-}
-
-typedef struct SMqHbOneTopicBatchRsp {
-  char    topicName[TSDB_TOPIC_FNAME_LEN];
-  SArray* rsps;  // SArray<SMqHbRsp>
-} SMqHbOneTopicBatchRsp;
-
-static FORCE_INLINE int32_t taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pBatchRsp->topicName);
-  int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i);
-    tlen += taosEncodeSMqHbRsp(buf, pRsp);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) {
-  int32_t sz;
-  buf = taosDecodeStringTo(buf, pBatchRsp->topicName);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
-  for (int32_t i = 0; i < sz; i++) {
-    SMqHbRsp rsp;
-    buf = taosDecodeSMqHbRsp(buf, &rsp);
-    buf = taosArrayPush(pBatchRsp->rsps, &rsp);
-  }
-  return buf;
-}
-
-typedef struct SMqHbBatchRsp {
-  int64_t consumerId;
-  SArray* batchRsps;  // SArray<SMqHbOneTopicBatchRsp>
-} SMqHbBatchRsp;
-
-static FORCE_INLINE int32_t taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId);
-  int32_t sz;
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*)taosArrayGet(pBatchRsp->batchRsps, i);
-    tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) {
-  buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId);
-  int32_t sz;
-  buf = taosDecodeFixedI32(buf, &sz);
-  pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp));
-  for (int32_t i = 0; i < sz; i++) {
-    SMqHbOneTopicBatchRsp rsp;
-    buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp);
-    buf = taosArrayPush(pBatchRsp->batchRsps, &rsp);
-  }
-  return buf;
-}
 
 typedef struct {
   int32_t key;
@@ -2087,15 +2022,11 @@ int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
 int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
 int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
 
-typedef struct {
-  uint32_t nCols;
-  SSchema* pSchema;
-} SSchemaWrapper;
-
 static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
   SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
   if (pSW == NULL) return pSW;
   pSW->nCols = pSchemaWrapper->nCols;
+  pSW->sver = pSchemaWrapper->sver;
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) {
     taosMemoryFree(pSW);
@@ -2132,8 +2063,8 @@ static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) {
 static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
   if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
   if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1;
-  if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1;
-  if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1;
+  if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
   return 0;
 }
@@ -2141,15 +2072,16 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch
 static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
   if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
   if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1;
-  if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1;
+  if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;
   return 0;
 }
 
 static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
   int32_t tlen = 0;
-  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  tlen += taosEncodeVariantI32(buf, pSW->nCols);
+  tlen += taosEncodeVariantI32(buf, pSW->sver);
   for (int32_t i = 0; i < pSW->nCols; i++) {
     tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]);
   }
@@ -2157,7 +2089,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
 }
 
 static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
-  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  buf = taosDecodeVariantI32(buf, &pSW->nCols);
+  buf = taosDecodeVariantI32(buf, &pSW->sver);
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) {
     return NULL;
@@ -2170,23 +2103,25 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
 }
 
 static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) {
-  if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1;
   for (int32_t i = 0; i < pSW->nCols; i++) {
     if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1;
   }
-  return pEncoder->pos;
+
+  return 0;
 }
 
 static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) {
-  if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
-  void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema));
+  if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
-  if (ptr == NULL) {
+
-    return -1;
+  pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols);
-  }
+  if (pSW->pSchema == NULL) return -1;
-  pSW->pSchema = (SSchema*)ptr;
   for (int32_t i = 0; i < pSW->nCols; i++) {
     if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
   }
+
   return 0;
 }
 
@@ -2442,22 +2377,6 @@ typedef struct {
   int64_t consumerId;
 } SMqRspHead;
 
-#if 0
-typedef struct {
-  SMsgHead head;
-
-  int64_t consumerId;
-  int64_t blockingTime;
-  int32_t epoch;
-  int8_t  withSchema;
-  char    cgroup[TSDB_CGROUP_LEN];
-
-  int64_t  currentOffset;
-  uint64_t reqId;
-  char     topic[TSDB_TOPIC_FNAME_LEN];
-} SMqPollReq;
-#endif
-
 typedef struct {
   SMsgHead head;
   char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
@@ -2481,18 +2400,6 @@ typedef struct {
   SSchemaWrapper schema;
 } SMqSubTopicEp;
 
-typedef struct {
-  SMqRspHead head;
-  int64_t    reqOffset;
-  int64_t    rspOffset;
-  int32_t    skipLogNum;
-  // TODO: replace with topic name
-  int32_t numOfTopics;
-  // TODO: remove from msg
-  SSchemaWrapper* schema;
-  SArray*         pBlockData;  // SArray<SSDataBlock>
-} SMqPollRsp;
-
 typedef struct {
   SMqRspHead head;
   int64_t    reqOffset;
@@ -2616,7 +2523,7 @@ typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
   SArray*    topics;  // SArray<SMqSubTopicEp>
-} SMqCMGetSubEpRsp;
+} SMqAskEpRsp;
 
 static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
   // taosMemoryFree(pSubTopicEp->schema.pSchema);
@@ -2638,10 +2545,6 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
   return buf;
 }
 
-static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
-  taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
-}
-
 static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
   int32_t tlen = 0;
   tlen += taosEncodeString(buf, pTopicEp->topic);
@@ -2674,7 +2577,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
   return buf;
 }
 
-static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) {
+static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) {
   int32_t tlen = 0;
   // tlen += taosEncodeString(buf, pRsp->cgroup);
   int32_t sz = taosArrayGetSize(pRsp->topics);
@@ -2686,7 +2589,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu
   return tlen;
 }
 
-static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) {
+static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) {
   // buf = taosDecodeStringTo(buf, pRsp->cgroup);
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
@@ -2702,6 +2605,10 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
   return buf;
 }
 
+static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
+  taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
+}
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -145,7 +145,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-mq-ask-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)

include/common/trow.h

@@ -23,7 +23,6 @@
 #include "tbuffer.h"
 #include "tdataformat.h"
 #include "tdef.h"
-#include "tschema.h"
 #include "ttypes.h"
 #include "tutil.h"
 
@@ -341,7 +340,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx,
   return false;
 }
 
-static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
+                                               int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);

include/common/tschema.h

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_COMMON_SCHEMA_H_
-#define _TD_COMMON_SCHEMA_H_
-
-#include "os.h"
-#include "tarray.h"
-#include "ttypes.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if 0
-typedef struct STColumn {
-  /// column name
-  char *cname;
-  union {
-    /// for encode purpose
-    uint64_t info;
-    struct {
-      uint64_t sma : 1;
-      /// column data type
-      uint64_t type : 7;
-      /// column id
-      uint64_t cid : 16;
-      /// max bytes of the column
-      uint64_t bytes : 32;
-      /// reserved
-      uint64_t reserve : 8;
-    };
-  };
-  /// comment about the column
-  char *comment;
-} STColumn;
-
-typedef struct STSchema {
-  /// schema version
-  uint16_t sver;
-  /// number of columns
-  uint16_t ncols;
-  /// sma attributes
-  struct {
-    bool    sma;
-    SArray *smaArray;
-  };
-  /// column info
-  STColumn cols[];
-} STSchema;
-
-typedef struct {
-  uint64_t  size;
-  STSchema *pSchema;
-} STShemaBuilder;
-
-#define tSchemaBuilderInit(target, capacity) \
-  { .size = (capacity), .pSchema = (target) }
-void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
-void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
-int32_t  tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
-
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_COMMON_SCHEMA_H_*/

include/dnode/mnode/sdb/sdb.h

@@ -221,7 +221,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw);
  * @param pRaw The raw data.
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw);
+int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw);
 
 /**
  * @brief Acquire a row from sdb

include/libs/function/function.h

@@ -202,7 +202,7 @@ typedef struct SqlFunctionCtx {
   SPoint1                end;
   SFuncExecFuncs         fpSet;
   SScalarFuncExecFuncs   sfp;
-  SExprInfo             *pExpr;
+  struct SExprInfo      *pExpr;
   struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   int32_t                curBufPage;

include/libs/function/functionMgt.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "function.h"
+#include "querynodes.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -123,7 +123,7 @@ struct SCatalog;
 
 typedef struct SFmGetFuncInfoParam {
   struct SCatalog* pCtg;
-  void *pRpc;
+  void*            pRpc;
   const SEpSet*    pMgmtEps;
   char*            pErrBuf;
   int32_t          errBufLen;
@@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
+bool fmIsScanPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);

include/libs/nodes/plannodes.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 #include "tname.h"
 
 typedef struct SLogicNode {
@@ -33,16 +33,12 @@ typedef struct SLogicNode {
   int32_t            optimizedFlag;
 } SLogicNode;
 
-typedef enum EScanType {
+typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
-  SCAN_TYPE_TAG,
-  SCAN_TYPE_TABLE,
-  SCAN_TYPE_SYSTEM_TABLE,
-  SCAN_TYPE_STREAM
-} EScanType;
 
 typedef struct SScanLogicNode {
   SLogicNode         node;
   SNodeList*         pScanCols;
+  SNodeList*         pScanPseudoCols;
   struct STableMeta* pMeta;
   SVgroupsInfo*      pVgroupList;
   EScanType          scanType;
@@ -95,11 +91,7 @@ typedef struct SExchangeLogicNode {
   uint8_t    precision;
 } SExchangeLogicNode;
 
-typedef enum EWindowType {
+typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
-  WINDOW_TYPE_INTERVAL = 1,
-  WINDOW_TYPE_SESSION,
-  WINDOW_TYPE_STATE
-} EWindowType;
 
 typedef struct SWindowLogicNode {
   SLogicNode  node;
@@ -187,6 +179,7 @@ typedef struct SPhysiNode {
 typedef struct SScanPhysiNode {
   SPhysiNode node;
   SNodeList* pScanCols;
+  SNodeList* pScanPseudoCols;
   uint64_t   uid;  // unique id of the table
   int8_t     tableType;
   SName      tableName;
@@ -315,7 +308,7 @@ typedef struct SDataInserterNode {
   SDataSinkNode sink;
   int32_t       numOfTables;
   uint32_t      size;
-  char     *pData;
+  char*         pData;
 } SDataInserterNode;
 
 typedef struct SSubplan {
@@ -333,11 +326,7 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;     // data of the subplan flow into the datasink
 } SSubplan;
 
-typedef enum EExplainMode {
+typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
-  EXPLAIN_MODE_DISABLE = 1,
-  EXPLAIN_MODE_STATIC,
-  EXPLAIN_MODE_ANALYZE
-} EExplainMode;
 
 typedef struct SExplainInfo {
   EExplainMode mode;

include/libs/nodes/querynodes.h

@@ -293,7 +293,10 @@ typedef struct SExplainStmt {
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
+typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
+
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
+                            SNodeList** pCols);
 
 typedef bool (*FFuncClassifier)(int32_t funcId);
 int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);

include/libs/parser/parser.h

@@ -88,11 +88,11 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
 void    qFreeStmtDataBlock(void* pDataBlock);
 int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
 void    qDestroyStmtDataBlock(void* pBlock);
-int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
-int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
+int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
 int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
 int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
-int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
 void destroyBoundColumnInfo(void* pBoundInfo);
 int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
 

include/libs/planner/planner.h

@@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams);
+int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx);
 
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

include/libs/sync/sync.h

@@ -158,6 +158,8 @@ typedef enum {
 
 int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
+bool syncEnvIsStart();
+
 extern int32_t sDebugFlag;
 
 //-----------------------------------------

include/util/taoserror.h

@@ -139,16 +139,9 @@ int32_t* taosGetErrno();
 // mnode-common
 #define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
 #define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0301)
-#define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0302)
+#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0302)
-#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0303)
+#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0303)
-#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED   TAOS_DEF_ERROR_CODE(0, 0x0304)
+#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0304)
-#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0305)
-#define TSDB_CODE_MND_INVALID_OPTIONS           TAOS_DEF_ERROR_CODE(0, 0x0306)
-#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0307)
-#define TSDB_CODE_MND_INVALID_MSG_VERSION       TAOS_DEF_ERROR_CODE(0, 0x0308)
-#define TSDB_CODE_MND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0309)
-#define TSDB_CODE_MND_INVALID_MSG_TYPE          TAOS_DEF_ERROR_CODE(0, 0x030A)
-#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS      TAOS_DEF_ERROR_CODE(0, 0x030B)
 
 // mnode-show
 #define TSDB_CODE_MND_INVALID_SHOWOBJ           TAOS_DEF_ERROR_CODE(0, 0x0310)

include/util/tdef.h

@@ -130,7 +130,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_PERFS_TABLE_QUERIES       "queries"
 #define TSDB_PERFS_TABLE_TOPICS        "topics"
 #define TSDB_PERFS_TABLE_CONSUMERS     "consumers"
-#define TSDB_PERFS_TABLE_SUBSCRIBES  "subscribes"
+#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
 
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
@@ -195,6 +195,8 @@ typedef enum EOperatorType {
   OP_TYPE_JSON_CONTAINS
 } EOperatorType;
 
+#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
+
 typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_AND = 1,
   LOGIC_COND_TYPE_OR,
@@ -284,8 +286,9 @@ typedef enum ELogicConditionType {
 #define TSDB_IPv4ADDR_LEN       16
 #define TSDB_FILENAME_LEN       128
 #define TSDB_SHOW_SQL_LEN       512
-#define TSDB_SHOW_SUBQUERY_LEN  1000
 #define TSDB_SLOW_QUERY_SQL_LEN 512
+#define TSDB_SHOW_SUBQUERY_LEN  1000
+#define TSDB_SHOW_LIST_LEN      1000
 
 #define TSDB_TRANS_STAGE_LEN 12
 #define TSDB_TRANS_TYPE_LEN  16

include/util/tencode.h

@@ -17,7 +17,8 @@
 #define _TD_UTIL_ENCODE_H_
 
 #include "tcoding.h"
-#include "tfreelist.h"
+#include "tlist.h"
+// #include "tfreelist.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -62,10 +63,14 @@ struct SCoderNode {
   CODER_NODE_FIELDS
 };
 
+typedef struct SCoderMem {
+  struct SCoderMem* next;
+} SCoderMem;
+
 typedef struct {
   td_coder_t  type;
   td_endian_t endian;
-  SFreeList   fl;
+  SCoderMem*  mList;
   CODER_NODE_FIELDS
   TD_SLIST(SCoderNode) stack;
 } SCoder;
@@ -74,7 +79,31 @@ typedef struct {
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
-#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER)          TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl))
+#define TCODER_MALLOC(PCODER, SIZE)                                         \
+  ({                                                                        \
+    void*      ptr = NULL;                                                  \
+    SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
+    if (pMem) {                                                             \
+      pMem->next = (PCODER)->mList;                                         \
+      (PCODER)->mList = pMem;                                               \
+      ptr = (void*)&pMem[1];                                                \
+    }                                                                       \
+    ptr;                                                                    \
+  })
+
+#define tEncodeSize(E, S, SIZE)                                \
+  ({                                                           \
+    SCoder coder = {0};                                        \
+    int    ret = 0;                                            \
+    tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
+    if ((E)(&coder, S) == 0) {                                 \
+      SIZE = coder.pos;                                        \
+    } else {                                                   \
+      ret = -1;                                                \
+    }                                                          \
+    tCoderClear(&coder);                                       \
+    ret;                                                       \
+  })
 
 void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
 void tCoderClear(SCoder* pCoder);
@@ -377,7 +406,9 @@ static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, ui
   if (tDecodeU64v(pDecoder, len) < 0) return -1;
 
   if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
+  if (val) {
     *val = (void*)TD_CODER_CURRENT(pDecoder);
+  }
 
   TD_CODER_MOVE_POS(pDecoder, *len);
   return 0;

include/util/tlog.h

@@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
+extern int32_t metaDebugFlag;
 extern int32_t fnDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
@@ -79,6 +80,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #endif
     ;
 
+// clang-format off
 #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
 #define uWarn(...)  { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@@ -88,6 +90,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
+// clang-format on
 
 #ifdef __cplusplus
 }

source/client/inc/clientInt.h

@@ -310,6 +310,9 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 void hbMgrInitMqHbRspHandle();
 
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
+int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
+int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
+
 
 #ifdef __cplusplus
 }

source/client/inc/clientStmt.h

@@ -34,8 +34,7 @@ typedef enum {
   STMT_PREPARE,
   STMT_SETTBNAME,
   STMT_SETTAGS,
-  STMT_FETCH_TAG_FIELDS,
+  STMT_FETCH_FIELDS,
-  STMT_FETCH_COL_FIELDS,
   STMT_BIND,
   STMT_BIND_COL,
   STMT_ADD_BATCH,
@@ -75,6 +74,8 @@ typedef struct SStmtSQLInfo {
   SQuery*      pQuery;
   char*        sqlStr;
   int32_t      sqlLen;
+  SArray*      nodeList;
+  SQueryPlan*  pQueryPlan;
 } SStmtSQLInfo;
 
 typedef struct STscStmt {
@@ -87,6 +88,8 @@ typedef struct STscStmt {
   SStmtBindInfo bInfo;
 } STscStmt;
 
+#define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
+#define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
 
 #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
 #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
@@ -97,14 +100,15 @@ int stmtClose(TAOS_STMT *stmt);
 int stmtExec(TAOS_STMT *stmt);
 const char *stmtErrstr(TAOS_STMT *stmt);
 int stmtAffectedRows(TAOS_STMT *stmt);
+int stmtAffectedRowsOnce(TAOS_STMT *stmt);
 int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
 int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
-int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags);
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags);
 int stmtIsInsert(TAOS_STMT *stmt, int *insert);
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums);
 int stmtAddBatch(TAOS_STMT *stmt);
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt);
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx);
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx);
 
 
 #ifdef __cplusplus

source/client/src/clientImpl.c

@@ -305,6 +305,11 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   return pRequest->code;
 }
 
+int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) {
+  *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
+  return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList);
+}
+
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     switch (pQuery->execMode) {

source/client/src/clientMain.c

@@ -603,7 +603,7 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
   return stmtPrepare(stmt, sql, length);
 }
 
-int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) {
+int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags) {
   if (stmt == NULL || name == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -636,7 +636,7 @@ int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
   return taos_stmt_set_tbname(stmt, name);
 }
 
-int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -652,7 +652,7 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
   return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -665,10 +665,18 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
     return terrno;
   }
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert && bind->num > 1) {
+    tscError("only one row data allowed for query");
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
   return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) {
+int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -681,6 +689,14 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int c
     return terrno;
   }
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert && bind->num > 1) {
+    tscError("only one row data allowed for query");
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  
   return stmtBindBatch(stmt, bind, colIdx);
 }
 
@@ -748,6 +764,16 @@ int taos_stmt_affected_rows(TAOS_STMT *stmt) {
   return stmtAffectedRows(stmt);
 }
 
+int taos_stmt_affected_rows_once(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
+    tscError("NULL parameter for %s", __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return 0;
+  }
+
+  return stmtAffectedRowsOnce(stmt);
+}
+
 int taos_stmt_close(TAOS_STMT *stmt) {
   if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);

source/client/src/clientStmt.c

@@ -5,14 +5,52 @@
 #include "tdef.h"
 
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
+  int32_t code = 0;
+  
   switch (newStatus) {
+    case STMT_PREPARE:
+      break;
     case STMT_SETTBNAME:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_SETTAGS:
+      if (STMT_STATUS_NE(SETTBNAME)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_FETCH_FIELDS:
+      if (STMT_STATUS_EQ(INIT)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_BIND:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_BIND_COL:
+      if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_ADD_BATCH:
+      if (STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL) && STMT_STATUS_NE(FETCH_FIELDS)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
+      break;
+    case STMT_EXECUTE:
+      if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) {
+        code = TSDB_CODE_TSC_STMT_API_ERROR;
+      }
       break;
     default:
+      code = TSDB_CODE_TSC_APP_ERROR;
       break;
   }
 
-  //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  STMT_ERR_RET(code);
 
   pStmt->sql.status = newStatus;
 
@@ -69,15 +107,10 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
     return TSDB_CODE_SUCCESS;
   }
 
-  uint64_t uid;
+  uint64_t uid = pStmt->bInfo.tbUid; 
-  if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) {
+  uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid;
-    uid = pStmt->bInfo.tbSuid;
-  } else {
-    ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType);
-    uid = pStmt->bInfo.tbUid;
-  }
 
-  if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) {
+  if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -91,7 +124,7 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
     .boundTags = pStmt->bInfo.boundTags,
   };
 
-  if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) {
+  if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
@@ -149,9 +182,11 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
+int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
+  if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) {
     taos_free_result(pStmt->exec.pRequest);
     pStmt->exec.pRequest = NULL;
+  }
 
   void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
   while (pIter) {
@@ -186,6 +221,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
 int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
   taosMemoryFree(pStmt->sql.sqlStr);
   qDestroyQuery(pStmt->sql.pQuery);
+  qDestroyQueryPlan(pStmt->sql.pQueryPlan);
+  taosArrayDestroy(pStmt->sql.nodeList);
   
   void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
   while (pIter) {
@@ -201,7 +238,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
 
   memset(&pStmt->sql, 0, sizeof(pStmt->sql));
 
-  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false));
+  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true));
   STMT_ERR_RET(stmtCleanBindInfo(pStmt));
 
   return TSDB_CODE_SUCCESS;
@@ -333,6 +370,13 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
 
+  int32_t insert = 0;
+  stmtIsInsert(stmt, &insert);
+  if (0 == insert) {
+    tscError("set tb name not available for none insert statement");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
   if (NULL == pStmt->exec.pRequest) {
     STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
   }
@@ -349,7 +393,7 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
@@ -370,15 +414,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
 }
 
 
-int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
-  STscStmt* pStmt = (STscStmt*)stmt;
-
-  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS));
-
-  if (pStmt->bInfo.needParse) {
-    STMT_ERR_RET(stmtParseSql(pStmt));
-  }
-
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     tscError("invalid operation to get query tag fileds");
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -395,15 +431,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
-  STscStmt* pStmt = (STscStmt*)stmt;
-
-  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS));
-
-  if (pStmt->bInfo.needParse) {
-    STMT_ERR_RET(stmtParseSql(pStmt));
-  }
-
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     tscError("invalid operation to get query column fileds");
     STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@@ -420,7 +448,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
   return TSDB_CODE_SUCCESS;  
 }
 
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
@@ -429,6 +457,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
     pStmt->bInfo.needParse = false;
   }
 
+  if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
+    taos_free_result(pStmt->exec.pRequest);
+    pStmt->exec.pRequest = NULL;
+  }
+  
   if (NULL == pStmt->exec.pRequest) {
     STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
   }
@@ -437,6 +470,16 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
     STMT_ERR_RET(stmtParseSql(pStmt));
   }
 
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    if (NULL == pStmt->sql.pQueryPlan) {
+      STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
+      pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
+      pStmt->exec.pRequest->body.pDag = NULL;
+    }
+    
+    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx));
+  }
+  
   STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
   if (NULL == pDataBlock) {
     tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
@@ -480,9 +523,12 @@ int stmtExec(TAOS_STMT *stmt) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
 
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList);
+  } else {
     STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
-
     launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
+  }
   
   STMT_ERR_JRET(pStmt->exec.pRequest->code);
 
@@ -491,7 +537,7 @@ int stmtExec(TAOS_STMT *stmt) {
 
 _return:
 
-  stmtCleanExecInfo(pStmt, (code ? false : true));
+  stmtCleanExecInfo(pStmt, (code ? false : true), false);
   
   ++pStmt->sql.runTimes;
   
@@ -523,6 +569,10 @@ int stmtAffectedRows(TAOS_STMT *stmt) {
   return ((STscStmt*)stmt)->affectedRows;
 }
 
+int stmtAffectedRowsOnce(TAOS_STMT *stmt) {
+  return ((STscStmt*)stmt)->exec.affectedRows;
+}
+
 int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -536,13 +586,38 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
 }
 
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    if (NULL == pStmt->sql.pQueryPlan) {
+      STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
+      pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
+      pStmt->exec.pRequest->body.pDag = NULL;
+    }
+
+    *nums = (pStmt->sql.pQueryPlan->pPlaceholderValues) ? pStmt->sql.pQueryPlan->pPlaceholderValues->length : 0;
+  } else {
     STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
+  }
   
   return TSDB_CODE_SUCCESS;
 }
 
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (STMT_TYPE_QUERY != pStmt->sql.type) {
+    tscError("useResult only for query statement");
     return NULL;
+  }
+
+  return pStmt->exec.pRequest;
 }
 
 

source/client/src/tmq.c

@@ -25,7 +25,14 @@
 #include "tref.h"
 #include "ttimer.h"
 
-int32_t tmqAskEp(tmq_t* tmq, bool sync);
+int32_t tmqAskEp(tmq_t* tmq, bool async);
+
+typedef struct {
+  int8_t inited;
+  tmr_h  timer;
+} SMqMgmt;
+
+static SMqMgmt tmqMgmt = {0};
 
 typedef struct {
   int8_t  tmqRspType;
@@ -35,7 +42,7 @@ typedef struct {
 typedef struct {
   int8_t      tmqRspType;
   int32_t     epoch;
-  SMqCMGetSubEpRsp msg;
+  SMqAskEpRsp msg;
 } SMqAskEpRspWrapper;
 
 struct tmq_list_t {
@@ -64,13 +71,6 @@ struct tmq_conf_t {
   tmq_commit_cb* commit_cb;
 };
 
-typedef struct {
-  int8_t inited;
-  tmr_h  timer;
-} SMqMgmt;
-
-static SMqMgmt tmqMgmt = {0};
-
 struct tmq_t {
   // conf
   char           groupId[TSDB_CGROUP_LEN];
@@ -164,7 +164,7 @@ typedef struct {
 typedef struct {
   tmq_t*  tmq;
   int32_t code;
-  int32_t sync;
+  int32_t async;
   tsem_t  rspSem;
 } SMqAskEpCbParam;
 
@@ -188,6 +188,7 @@ typedef struct {
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
   conf->autoCommit = false;
+  conf->autoCommitInterval = 5000;
   conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
   return conf;
 }
@@ -324,7 +325,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
     if (pTaskType == NULL) break;
 
     if (*pTaskType == TMQ_DELAYED_TASK__HB) {
-      tmqAskEp(tmq, false);
+      tmqAskEp(tmq, true);
       taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
       tmq_commit(tmq, NULL, true);
@@ -472,8 +473,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
-  /*pTmq->autoCommit = conf->autoCommit;*/
+  pTmq->autoCommit = conf->autoCommit;
-  pTmq->autoCommit = 0;
   pTmq->autoCommitInterval = conf->autoCommitInterval;
   pTmq->commit_cb = conf->commit_cb;
   pTmq->resetOffsetCfg = conf->resetOffset;
@@ -662,8 +662,8 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   if (code != 0) goto FAIL;
 
   // TODO: add max retry cnt
-  while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, true)) {
+  while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
-    tscDebug("not ready, retry\n");
+    tscDebug("not ready, retry");
     taosMsleep(500);
   }
 
@@ -854,7 +854,7 @@ CREATE_MSG_FAIL:
   return -1;
 }
 
-bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
+bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
@@ -936,7 +936,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tmq_t*           tmq = pParam->tmq;
   pParam->code = code;
   if (code != 0) {
-    tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
+    tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async);
     goto END;
   }
 
@@ -950,15 +950,15 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     goto END;
   }
 
-  if (pParam->sync) {
+  if (!pParam->async) {
-    SMqCMGetSubEpRsp rsp;
+    SMqAskEpRsp rsp;
-    tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
+    tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
     /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
     /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
     if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
       atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
     }
-    tDeleteSMqCMGetSubEpRsp(&rsp);
+    tDeleteSMqAskEpRsp(&rsp);
   } else {
     SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
     if (pWrapper == NULL) {
@@ -969,7 +969,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
     pWrapper->epoch = head->epoch;
     memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
-    tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
+    tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
 
     taosWriteQitem(tmq->mqueue, pWrapper);
     /*tsem_post(&tmq->rspSem);*/
@@ -978,13 +978,13 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
 
 END:
   /*atomic_store_8(&tmq->epStatus, 0);*/
-  if (pParam->sync) {
+  if (!pParam->async) {
     tsem_post(&pParam->rspSem);
   }
   return code;
 }
 
-int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+int32_t tmqAskEp(tmq_t* tmq, bool async) {
   int32_t code = 0;
 #if 0
   int8_t  epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
@@ -995,8 +995,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   }
   atomic_store_32(&tmq->epSkipCnt, 0);
 #endif
-  int32_t           tlen = sizeof(SMqCMGetSubEpReq);
+  int32_t      tlen = sizeof(SMqAskEpReq);
-  SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
+  SMqAskEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
     tscError("failed to malloc get subscribe ep buf");
     /*atomic_store_8(&tmq->epStatus, 0);*/
@@ -1014,7 +1014,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
     return -1;
   }
   pParam->tmq = tmq;
-  pParam->sync = sync;
+  pParam->async = async;
   tsem_init(&pParam->rspSem, 0, 0);
 
   SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
@@ -1036,7 +1036,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   sendInfo->requestObjRefId = 0;
   sendInfo->param = pParam;
   sendInfo->fp = tmqAskEpCb;
-  sendInfo->msgType = TDMT_MND_GET_SUB_EP;
+  sendInfo->msgType = TDMT_MND_MQ_ASK_EP;
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
@@ -1045,7 +1045,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  if (sync) {
+  if (!async) {
     tsem_wait(&pParam->rspSem);
     code = pParam->code;
     taosMemoryFree(pParam);
@@ -1209,7 +1209,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) {
       SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper;
-      SMqCMGetSubEpRsp*   rspMsg = &pEpRspWrapper->msg;
+      SMqAskEpRsp*        rspMsg = &pEpRspWrapper->msg;
       tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg);
       /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
@@ -1271,15 +1271,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   SMqRspObj* rspObj;
   int64_t    startTime = taosGetTimestampMs();
 
-  // TODO: put into delayed queue
-#if 0
-  int8_t status = atomic_load_8(&tmq->status);
-  while (0 != tmqAskEp(tmq, status != TMQ_CONSUMER_STATUS__READY)) {
-    tscDebug("not ready, retry\n");
-    taosSsleep(1);
-  }
-#endif
-
   rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
   if (rspObj) {
     return (TAOS_RES*)rspObj;

source/common/src/tdatablock.c

@@ -78,9 +78,13 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo
   ASSERT(pColumnInfoData != NULL);
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return pColumnInfoData->varmeta.length;
+  } else {
+    if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) {
+      return 0;
     } else {
       return pColumnInfoData->info.bytes * numOfRows;
     }
+  }
 }
 
 int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
@@ -307,6 +311,8 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
     memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
   }
 
+  pColumnInfoData->hasNull = pSource->hasNull;
+  pColumnInfoData->info = pSource->info;
   return 0;
 }
 

source/common/src/tmsg.c

@@ -490,181 +490,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
   return 0;
 }
 
-int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
-  int32_t tlen = 0;
-
-  tlen += taosEncodeString(buf, pReq->name);
-  tlen += taosEncodeFixedU32(buf, pReq->ttl);
-  tlen += taosEncodeFixedU32(buf, pReq->keep);
-  tlen += taosEncodeFixedU8(buf, pReq->info);
-
-  switch (pReq->type) {
-    case TD_SUPER_TABLE:
-      tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
-        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
-        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags);
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
-        tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
-        tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
-      }
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
-        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
-        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags);
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
-        tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
-        tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
-      }
-      if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
-        SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        tlen += taosEncodeFixedI32(buf, param->delay);
-        tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
-        if (param->qmsg1Len > 0) {
-          tlen += taosEncodeString(buf, param->qmsg1);
-        }
-
-        tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
-        if (param->qmsg2Len > 0) {
-          tlen += taosEncodeString(buf, param->qmsg2);
-        }
-      }
-      break;
-    case TD_CHILD_TABLE:
-      tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid);
-      tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
-      break;
-    case TD_NORMAL_TABLE:
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
-        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
-        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags);
-        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
-        tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
-        tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
-      }
-      if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
-        SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        tlen += taosEncodeFixedI32(buf, param->delay);
-      }
-      break;
-    default:
-      ASSERT(0);
-  }
-
-  return tlen;
-}
-
-void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
-  buf = taosDecodeString(buf, &(pReq->name));
-  buf = taosDecodeFixedU32(buf, &(pReq->ttl));
-  buf = taosDecodeFixedU32(buf, &(pReq->keep));
-  buf = taosDecodeFixedU8(buf, &(pReq->info));
-
-  switch (pReq->type) {
-    case TD_SUPER_TABLE:
-      buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
-      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
-        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
-        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags));
-        buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
-        buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
-        buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
-      }
-      buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
-      pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
-        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
-        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags));
-        buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
-        buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
-        buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
-      }
-      if (pReq->rollup) {
-        pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
-        SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        buf = taosDecodeFixedI32(buf, &param->delay);
-        buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
-        if (param->qmsg1Len > 0) {
-          buf = taosDecodeString(buf, &param->qmsg1);
-        }
-
-        buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
-        if (param->qmsg2Len > 0) {
-          buf = taosDecodeString(buf, &param->qmsg2);
-        }
-      } else {
-        pReq->stbCfg.pRSmaParam = NULL;
-      }
-      break;
-    case TD_CHILD_TABLE:
-      buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid);
-      buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
-      break;
-    case TD_NORMAL_TABLE:
-      buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
-      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
-      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
-        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
-        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags);
-        buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
-        buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
-        buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
-      }
-      if (pReq->rollup) {
-        pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
-        SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
-        buf = taosDecodeFixedI32(buf, &param->delay);
-      } else {
-        pReq->ntbCfg.pRSmaParam = NULL;
-      }
-      break;
-    default:
-      ASSERT(0);
-  }
-
-  return buf;
-}
-
-int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
-  int32_t tlen = 0;
-
-  tlen += taosEncodeFixedI64(buf, pReq->ver);
-  tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
-  for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
-    SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
-    tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq);
-  }
-
-  return tlen;
-}
-
-void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
-  uint32_t nsize = 0;
-
-  buf = taosDecodeFixedI64(buf, &pReq->ver);
-  buf = taosDecodeFixedU32(buf, &nsize);
-  pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
-  for (size_t i = 0; i < nsize; i++) {
-    SVCreateTbReq req = {0};
-    buf = tDeserializeSVCreateTbReq(buf, &req);
-    taosArrayPush(pReq->pArray, &req);
-  }
-
-  return buf;
-}
-
 int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pReq->ver);
@@ -692,7 +517,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
@@ -702,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
     if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+    if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
   }
 
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
@@ -709,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
     if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
-  }
+    if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
-
-  for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
-    SField *pField = taosArrayGet(pReq->pSmas, i);
-    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
   if (pReq->commentLen > 0) {
@@ -746,15 +565,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
 
   pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
   pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
-  pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField));
+  if (pReq->pColumns == NULL || pReq->pTags == NULL) {
-  if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
@@ -764,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
     if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
     if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+    if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
     if (taosArrayPush(pReq->pColumns, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -775,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
     if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
     if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+    if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
     if (taosArrayPush(pReq->pTags, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
   }
 
-  for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
-    SField field = {0};
-    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
-    if (taosArrayPush(pReq->pSmas, &field) == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-  }
-
   if (pReq->commentLen > 0) {
     pReq->comment = taosMemoryMalloc(pReq->commentLen);
     if (pReq->comment == NULL) return -1;
@@ -819,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
 void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
   taosArrayDestroy(pReq->pColumns);
   taosArrayDestroy(pReq->pTags);
-  taosArrayDestroy(pReq->pSmas);
   taosMemoryFreeClear(pReq->comment);
   taosMemoryFreeClear(pReq->pAst1);
   taosMemoryFreeClear(pReq->pAst2);
   pReq->pColumns = NULL;
   pReq->pTags = NULL;
-  pReq->pSmas = NULL;
 }
 
 int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
@@ -3420,7 +3226,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq
 int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
   if (tStartDecode(decoder) < 0) return -1;
   if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
-  TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder);
+  pReq->offsets = (SMqOffset *)TCODER_MALLOC(decoder, sizeof(SMqOffset) * pReq->num);
   if (pReq->offsets == NULL) return -1;
   for (int32_t i = 0; i < pReq->num; i++) {
     tDecodeSMqOffset(decoder, &pReq->offsets[i]);
@@ -3636,48 +3442,79 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR
 }
 
 int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
-  SCoder encoder = {0};
+  // SCoder encoder = {0};
-  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+  // tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+  // if (tStartEncode(&encoder) < 0) return -1;
-  if (pRsp->rspList) {
+  // if (pRsp->rspList) {
-    int32_t num = taosArrayGetSize(pRsp->rspList);
+  //   int32_t num = taosArrayGetSize(pRsp->rspList);
-    if (tEncodeI32(&encoder, num) < 0) return -1;
+  //   if (tEncodeI32(&encoder, num) < 0) return -1;
-    for (int32_t i = 0; i < num; ++i) {
+  //   for (int32_t i = 0; i < num; ++i) {
-      SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
+  //     SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
-      if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
+  //     if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
-    }
+  //   }
-  } else {
+  // } else {
-    if (tEncodeI32(&encoder, 0) < 0) return -1;
+  //   if (tEncodeI32(&encoder, 0) < 0) return -1;
-  }
+  // }
-  tEndEncode(&encoder);
+  // tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
+  // int32_t tlen = encoder.pos;
-  tCoderClear(&encoder);
+  // tCoderClear(&encoder);
-  return tlen;
+  // reture tlen;
+  return 0;
 }
 
 int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
-  SCoder  decoder = {0};
+  // SCoder  decoder = {0};
-  int32_t num = 0;
+  // int32_t num = 0;
-  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  // tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+  // if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &num) < 0) return -1;
+  // if (tDecodeI32(&decoder, &num) < 0) return -1;
-  if (num > 0) {
+  // if (num > 0) {
-    pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
+  //   pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
-    if (NULL == pRsp->rspList) return -1;
+  //   if (NULL == pRsp->rspList) return -1;
-    for (int32_t i = 0; i < num; ++i) {
+  //   for (int32_t i = 0; i < num; ++i) {
-      SVCreateTbRsp rsp = {0};
+  //     SVCreateTbRsp rsp = {0};
-      if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
+  //     if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
-      if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
+  //     if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
-    }
+  //   }
-  } else {
+  // } else {
-    pRsp->rspList = NULL;
+  //   pRsp->rspList = NULL;
-  }
+  // }
-  tEndDecode(&decoder);
+  // tEndDecode(&decoder);
 
-  tCoderClear(&decoder);
+  // tCoderClear(&decoder);
+  return 0;
+}
+
+int tEncodeSVCreateTbBatchRsp(SCoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
+  int32_t        nRsps = taosArrayGetSize(pRsp->pArray);
+  SVCreateTbRsp *pCreateRsp;
+
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI32v(pCoder, nRsps) < 0) return -1;
+  for (int32_t i = 0; i < nRsps; i++) {
+    pCreateRsp = taosArrayGet(pRsp->pArray, i);
+    if (tEncodeSVCreateTbRsp(pCoder, pCreateRsp) < 0) return -1;
+  }
+
+  tEndEncode(pCoder);
+
+  return 0;
+}
+
+int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1;
+  pRsp->pRsps = (SVCreateTbRsp *)TCODER_MALLOC(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps);
+  for (int32_t i = 0; i < pRsp->nRsps; i++) {
+    if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1;
+  }
+
+  tEndDecode(pCoder);
   return 0;
 }
 
@@ -3779,6 +3616,38 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
   taosMemoryFreeClear(pReq->ast);
 }
 
+int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
+  if (tEncodeI64(pCoder, pReq->suid) < 0) return -1;
+  if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1;
+  if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
+  if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
+  // if (pReq->rollup) {
+  //   if (tEncodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1;
+  // }
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int tDecodeSVCreateStbReq(SCoder *pCoder, SVCreateStbReq *pReq) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
+  if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1;
+  if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1;
+  if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
+  if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
+  // if (pReq->rollup) {
+  //   if (tDecodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1;
+  // }
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
 STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
   STSchemaBuilder schemaBuilder = {0};
   if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
@@ -3802,3 +3671,97 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
   tdDestroyTSchemaBuilder(&schemaBuilder);
   return pNSchema;
 }
+
+int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
+  if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
+
+  if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
+  if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
+  if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
+
+  if (pReq->type == TSDB_CHILD_TABLE) {
+    if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
+    if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1;
+  } else if (pReq->type == TSDB_NORMAL_TABLE) {
+    if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
+  } else {
+    ASSERT(0);
+  }
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) {
+  uint64_t len;
+
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
+  if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
+
+  if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
+  if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
+  if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
+
+  if (pReq->type == TSDB_CHILD_TABLE) {
+    if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
+    if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1;
+  } else if (pReq->type == TSDB_NORMAL_TABLE) {
+    if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
+  } else {
+    ASSERT(0);
+  }
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+int tEncodeSVCreateTbBatchReq(SCoder *pCoder, const SVCreateTbBatchReq *pReq) {
+  int32_t nReq = taosArrayGetSize(pReq->pArray);
+
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI32v(pCoder, nReq) < 0) return -1;
+  for (int iReq = 0; iReq < nReq; iReq++) {
+    if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
+  }
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1;
+  pReq->pReqs = (SVCreateTbReq *)TCODER_MALLOC(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs);
+  if (pReq->pReqs == NULL) return -1;
+  for (int iReq = 0; iReq < pReq->nReqs; iReq++) {
+    if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
+  }
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+int tEncodeSVCreateTbRsp(SCoder *pCoder, const SVCreateTbRsp *pRsp) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI32(pCoder, pRsp->code) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI32(pCoder, &pRsp->code) < 0) return -1;
+
+  tEndDecode(pCoder);
+  return 0;
+}

source/common/test/tschemaTest.cpp

@@ -1,6 +0,0 @@
-#include <gtest/gtest.h>
-#include "tschema.h"
-
-TEST(td_schema_test, build_schema_test) {
-
-}

source/dnode/mgmt/exe/dmMain.c

@@ -68,6 +68,7 @@ static void dmSetSignalHandle() {
 
 static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
   int32_t cmdEnvIndex = 0;
+  if (argc < 2) return 0;
   global.envCmd = taosMemoryMalloc(argc-1);
   memset(global.envCmd, 0, argc-1);
   for (int32_t i = 1; i < argc; ++i) {

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -211,7 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_MND_MQ_ASK_EP, mmProcessReadMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -107,14 +107,13 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
 
   pCfg->vgId = pCreate->vgId;
   strcpy(pCfg->dbname, pCreate->db);
-  pCfg->wsize = pCreate->cacheBlockSize * 1024 * 1024;
+  pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024;
-  pCfg->ssize = 1024;
-  pCfg->lsize = 1024 * 1024;
   pCfg->streamMode = pCreate->streamMode;
   pCfg->isWeak = true;
-  pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.days = 10;
-  pCfg->tsdbCfg.keep0 = pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep2 = 3650;  // pCreate->daysToKeep0;
-  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep0 = 3650;  // pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep1 = 3650;  // pCreate->daysToKeep0;
   pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->walCfg.vgId = pCreate->vgId;
@@ -212,7 +211,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
     return code;
   }
 
-
   code = vmWriteVnodesToFile(pMgmt);
   if (code != 0) {
     tFreeSCreateVnodeReq(&createReq);

source/dnode/mnode/impl/inc/mndDef.h

@@ -358,13 +358,11 @@ typedef struct {
   int32_t  ttl;
   int32_t  numOfColumns;
   int32_t  numOfTags;
-  int32_t  numOfSmas;
   int32_t  commentLen;
   int32_t  ast1Len;
   int32_t  ast2Len;
   SSchema* pColumns;
   SSchema* pTags;
-  SSchema* pSmas;
   char*    comment;
   char*    pAst1;
   char*    pAst2;
@@ -445,9 +443,7 @@ typedef struct {
   int64_t        createTime;
   int64_t        updateTime;
   int64_t        uid;
-  // TODO: use subDbUid
   int64_t        dbUid;
-  int64_t        subDbUid;
   int32_t        version;
   int8_t         subType;  // db or table
   int8_t         withTbName;

source/dnode/mnode/impl/inc/mndInt.h

@@ -45,7 +45,7 @@ extern "C" {
 typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg);
 typedef int32_t (*MndInitFp)(SMnode *pMnode);
 typedef void (*MndCleanupFp)(SMnode *pMnode);
-typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
 typedef struct SQWorkerMgmt SQHandle;
 
@@ -84,7 +84,7 @@ typedef struct {
   int64_t timeseriesAllowed;
 } SGrantInfo;
 
-struct SMnode {
+typedef struct SMnode {
   int32_t       selfId;
   int64_t       clusterId;
   int8_t        replica;
@@ -109,7 +109,7 @@ struct SMnode {
   SGrantInfo    grant;
   MndMsgFp      msgFp[TDMT_MAX];
   SMsgCb        msgCb;
-};
+} SMnode;
 
 void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
 int64_t mndGenerateUid(char *name, int32_t len);

source/dnode/mnode/impl/src/mndAcct.c

@@ -17,8 +17,8 @@
 #include "mndAcct.h"
 #include "mndShow.h"
 
-#define TSDB_ACCT_VER_NUMBER   1
+#define ACCT_VER_NUMBER   1
-#define TSDB_ACCT_RESERVE_SIZE 128
+#define ACCT_RESERVE_SIZE 128
 
 static int32_t  mndCreateDefaultAcct(SMnode *pMnode);
 static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct);
@@ -55,6 +55,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
   acctObj.createdTime = taosGetTimestampMs();
   acctObj.updateTime = acctObj.createdTime;
   acctObj.acctId = 1;
+  acctObj.status = 0;
   acctObj.cfg = (SAcctCfg){.maxUsers = INT32_MAX,
                            .maxDbs = INT32_MAX,
                            .maxStbs = INT32_MAX,
@@ -79,7 +80,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
 static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE);
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, ACCT_VER_NUMBER, sizeof(SAcctObj) + ACCT_RESERVE_SIZE);
   if (pRaw == NULL) goto _OVER;
 
   int32_t dataPos = 0;
@@ -100,7 +101,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
   SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxTopics, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState, _OVER)
-  SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER)
+  SDB_SET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER)
   SDB_SET_DATALEN(pRaw, dataPos, _OVER)
 
   terrno = 0;
@@ -122,7 +123,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
   int8_t sver = 0;
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
 
-  if (sver != TSDB_ACCT_VER_NUMBER) {
+  if (sver != ACCT_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
     goto _OVER;
   }
@@ -151,7 +152,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.maxTopics, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pAcct->cfg.maxStorage, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.accessState, _OVER)
-  SDB_GET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER)
+  SDB_GET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER)
 
   terrno = 0;
 
@@ -178,7 +179,6 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) {
 
 static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
   mTrace("acct:%s, perform update action, old row:%p new row:%p", pOld->acct, pOld, pNew);
-
   pOld->updateTime = pNew->updateTime;
   pOld->status = pNew->status;
   memcpy(&pOld->cfg, &pNew->cfg, sizeof(SAcctCfg));
@@ -186,19 +186,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
 }
 
 static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) {
-  terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
 
 static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) {
-  terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
 
 static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) {
-  terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }

source/dnode/mnode/impl/src/mndConsumer.c

@@ -59,7 +59,7 @@ int32_t mndInitConsumer(SMnode *pMnode) {
                      .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
 
   mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
   return sdbSetTable(pMnode->pSdb, table);
@@ -86,7 +86,7 @@ static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
   mndTransDrop(pTrans);
   return 0;
 FAIL:
-  // TODO delete consumer
+  tDeleteSMqConsumerObj(pConsumerNew);
   mndTransDrop(pTrans);
   return -1;
 }
@@ -198,8 +198,8 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
 
 static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   SMnode      *pMnode = pMsg->pNode;
-  SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
+  SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->rpcMsg.pCont;
-  SMqCMGetSubEpRsp  rsp = {0};
+  SMqAskEpRsp  rsp = {0};
   int64_t      consumerId = be64toh(pReq->consumerId);
   int32_t      epoch = ntohl(pReq->epoch);
 
@@ -300,7 +300,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
     taosRUnLockLatch(&pConsumer->lock);
   }
   // encode rsp
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
+  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, &rsp);
   void   *buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -311,10 +311,10 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
 
   void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
+  tEncodeSMqAskEpRsp(&abuf, &rsp);
 
   // release consumer and free memory
-  tDeleteSMqCMGetSubEpRsp(&rsp);
+  tDeleteSMqAskEpRsp(&rsp);
   mndReleaseConsumer(pMnode, pConsumer);
 
   // send rsp
@@ -322,7 +322,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   pMsg->rspLen = tlen;
   return 0;
 FAIL:
-  tDeleteSMqCMGetSubEpRsp(&rsp);
+  tDeleteSMqAskEpRsp(&rsp);
   mndReleaseConsumer(pMnode, pConsumer);
   return -1;
 }

source/dnode/mnode/impl/src/mndDef.c

@@ -215,7 +215,7 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
   if (pSubNew == NULL) return NULL;
   memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN);
   taosInitRWLatch(&pSubNew->lock);
-  pSubNew->vgNum = -1;
+  pSubNew->vgNum = 0;
   pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   // TODO set free fp
   SMqConsumerEpInSub epInSub = {

source/dnode/mnode/impl/src/mndDnode.c

@@ -363,7 +363,7 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
         pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH;
       }
       mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion);
-      terrno = TSDB_CODE_MND_INVALID_MSG_VERSION;
+      terrno = TSDB_CODE_VERSION_NOT_COMPATIBLE;
       goto PROCESS_STATUS_MSG_OVER;
     }
 

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -41,29 +41,31 @@ static const SPerfsTableSchema queriesSchema[] = {
 
 static const SPerfsTableSchema topicSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    /*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    /*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/
+    // TODO config
 };
 
 static const SPerfsTableSchema consumerSchema[] = {
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    // ep
+    {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    // up time
+    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    // topics
+    {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
 
-static const SPerfsTableSchema subscribeSchema[] = {
+static const SPerfsTableSchema subscriptionSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+    {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
 };
 
 static const SPerfsTableMeta perfsMeta[] = {
@@ -71,7 +73,7 @@ static const SPerfsTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)},
     {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
     {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
-    {TSDB_PERFS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
+    {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
 };
 
 // connection/application/

source/dnode/mnode/impl/src/mndScheduler.c

@@ -478,6 +478,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   SVgObj*     pVgroup = NULL;
   SQueryPlan* pPlan = NULL;
   SSubplan*   plan = NULL;
+
   if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
     pPlan = qStringToQueryPlan(pTopic->physicalPlan);
     if (pPlan == NULL) {
@@ -485,10 +486,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
       return -1;
     }
 
-    ASSERT(pSub->vgNum == -1);
-
-    pSub->vgNum = 0;
-
     int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
     if (levelNum != 1) {
       qDestroyQueryPlan(pPlan);
@@ -529,7 +526,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     pVgEp->vgId = pVgroup->vgId;
     taosArrayPush(pEpInSub->vgs, &pVgEp);
 
-    mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
+    mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
     if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
       int32_t msgLen;

source/dnode/mnode/impl/src/mndStb.c

@@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {}
 SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) +
+  int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen +
-                 + pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
+                 pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
   if (pRaw == NULL) goto _OVER;
 
@@ -91,7 +91,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER)
@@ -112,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
     SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
   }
 
-  for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
-    SSchema *pSchema = &pStb->pSmas[i];
-    SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER)
-    SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER)
-    SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER)
-    SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
-  }
-
   if (pStb->commentLen > 0) {
     SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
   }
@@ -178,15 +169,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER)
 
   pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
   pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema));
-  pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema));
+  if (pStb->pColumns == NULL || pStb->pTags == NULL) {
-  if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) {
     goto _OVER;
   }
 
@@ -206,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
     SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
   }
 
-  for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
-    SSchema *pSchema = &pStb->pSmas[i];
-    SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER)
-    SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER)
-    SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER)
-    SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
-  }
-
   if (pStb->commentLen > 0) {
     pStb->comment = taosMemoryCalloc(pStb->commentLen, 1);
     if (pStb->comment == NULL) goto _OVER;
@@ -291,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
     }
   }
 
-  if (pOld->numOfSmas < pNew->numOfSmas) {
-    void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema));
-    if (pSmas != NULL) {
-      taosMemoryFree(pOld->pSmas);
-      pOld->pSmas = pSmas;
-    } else {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
-      taosWUnLockLatch(&pOld->lock);
-    }
-  }
-
   if (pOld->commentLen < pNew->commentLen) {
     void *comment = taosMemoryMalloc(pNew->commentLen);
     if (comment != NULL) {
@@ -393,72 +362,49 @@ static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSch
 }
 
 static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
+  SCoder         coder = {0};
+  int32_t        contLen;
   SName          name = {0};
+  SVCreateStbReq req = {0};
+
   tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
   char dbFName[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(&name, dbFName);
 
-  SVCreateTbReq req = {0};
   req.name = (char *)tNameGetTableName(&name);
-  req.ttl = 0;
+  req.suid = pStb->uid;
-  req.keep = 0;
+  req.rollup = pStb->ast1Len > 0 ? 1 : 0;
-  req.rollup = pStb->pAst1 > 0 ? 1 : 0;
+  req.schema.nCols = pStb->numOfColumns;
-  req.type = TD_SUPER_TABLE;
+  req.schema.sver = 0;
-  req.stbCfg.suid = pStb->uid;
+  req.schema.pSchema = pStb->pColumns;
-  req.stbCfg.nCols = pStb->numOfColumns;
+  req.schemaTag.nCols = pStb->numOfTags;
-  req.stbCfg.nTagCols = pStb->numOfTags;
+  req.schemaTag.pSchema = pStb->pTags;
-  req.stbCfg.pTagSchema = pStb->pTags;
-  req.stbCfg.nBSmaCols = pStb->numOfSmas;
-  req.stbCfg.pSchema = (SSchema *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
-  if (req.stbCfg.pSchema == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
 
-  memcpy(req.stbCfg.pSchema, pStb->pColumns, sizeof(SSchema) * pStb->numOfColumns);
-  for (int i = 0; i < pStb->numOfColumns; i++) {
-    req.stbCfg.pSchema[i].flags = SCHEMA_SMA_ON;
-  }
-
-  SRSmaParam *pRSmaParam = NULL;
   if (req.rollup) {
-    pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
+    req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
-    if (pRSmaParam == NULL) {
+    req.pRSmaParam.delay = pStb->delay;
-      taosMemoryFreeClear(req.stbCfg.pSchema);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return NULL;
-    }
-
-    pRSmaParam->xFilesFactor = pStb->xFilesFactor;
-    pRSmaParam->delay = pStb->delay;
     if (pStb->ast1Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) {
+      if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len) != TSDB_CODE_SUCCESS) {
-        taosMemoryFreeClear(req.stbCfg.pRSmaParam);
-        taosMemoryFreeClear(req.stbCfg.pSchema);
         return NULL;
       }
     }
     if (pStb->ast2Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) {
+      if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len) != TSDB_CODE_SUCCESS) {
-        taosMemoryFreeClear(pRSmaParam->qmsg1);
-        taosMemoryFreeClear(req.stbCfg.pRSmaParam);
-        taosMemoryFreeClear(req.stbCfg.pSchema);
         return NULL;
       }
     }
-
+  }
-    req.stbCfg.pRSmaParam = pRSmaParam;
+  // get length
+  if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) {
+    return NULL;
   }
 
-  int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
+  contLen += sizeof(SMsgHead);
+
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
-    if (pRSmaParam) {
+    taosMemoryFreeClear(req.pRSmaParam.qmsg1);
-      taosMemoryFreeClear(pRSmaParam->qmsg1);
+    taosMemoryFreeClear(req.pRSmaParam.qmsg2);
-      taosMemoryFreeClear(pRSmaParam->qmsg2);
-      taosMemoryFreeClear(pRSmaParam);
-    }
-    taosMemoryFreeClear(req.stbCfg.pSchema);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
@@ -467,15 +413,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   pHead->vgId = htonl(pVgroup->vgId);
 
   void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
-  tSerializeSVCreateTbReq(&pBuf, &req);
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER);
+  if (tEncodeSVCreateStbReq(&coder, &req) < 0) {
+    return NULL;
+  }
+  tCoderClear(&coder);
 
   *pContLen = contLen;
-  if (pRSmaParam) {
+  taosMemoryFreeClear(req.pRSmaParam.qmsg1);
-    taosMemoryFreeClear(pRSmaParam->qmsg1);
+  taosMemoryFreeClear(req.pRSmaParam.qmsg2);
-    taosMemoryFreeClear(pRSmaParam->qmsg2);
-    taosMemoryFreeClear(pRSmaParam);
-  }
-  taosMemoryFreeClear(req.stbCfg.pSchema);
   return pHead;
 }
 
@@ -692,7 +638,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
   stbObj.ttl = pCreate->ttl;
   stbObj.numOfColumns = pCreate->numOfColumns;
   stbObj.numOfTags = pCreate->numOfTags;
-  stbObj.numOfSmas = pCreate->numOfSmas;
   stbObj.commentLen = pCreate->commentLen;
   if (stbObj.commentLen > 0) {
     stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
@@ -725,8 +670,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
 
   stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema));
   stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema));
-  stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema));
+  if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
-  if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
@@ -736,6 +680,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     SSchema *pSchema = &stbObj.pColumns[i];
     pSchema->type = pField->type;
     pSchema->bytes = pField->bytes;
+    pSchema->flags = pField->flags;
     memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
     pSchema->colId = stbObj.nextColId;
     stbObj.nextColId++;
@@ -751,18 +696,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     stbObj.nextColId++;
   }
 
-  for (int32_t i = 0; i < stbObj.numOfSmas; ++i) {
-    SField  *pField = taosArrayGet(pCreate->pSmas, i);
-    SSchema *pSchema = &stbObj.pSmas[i];
-    SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
-    if (pColSchema == NULL) {
-      mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
-      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
-      return -1;
-    }
-    memcpy(pSchema, pColSchema, sizeof(SSchema));
-  }
-
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
   if (pTrans == NULL) goto _OVER;

source/dnode/mnode/impl/src/mndSync.c

@@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) {
 
   char path[PATH_MAX] = {0};
   snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
-  SWalCfg cfg = {.vgId = 1,
+  SWalCfg cfg = {
+      .vgId = 1,
       .fsyncPeriod = 0,
       .rollPeriod = -1,
       .segSize = -1,
       .retentionPeriod = -1,
       .retentionSize = -1,
-                 .level = TAOS_WAL_FSYNC};
+      .level = TAOS_WAL_FSYNC,
+  };
   pMgmt->pWal = walOpen(path, &cfg);
   if (pMgmt->pWal == NULL) return -1;
 
@@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
 
   int64_t first = walGetFirstVer(pWal);
   int64_t last = walGetLastVer(pWal);
-  mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
+  mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
 
   first = TMAX(lastSdbVer + 1, first);
   for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) {
     if (walReadWithHandle(pHandle, ver) < 0) {
-      mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver);
+      mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr());
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     SWalHead *pHead = pHandle->pHead;
     int64_t   sdbVer = sdbUpdateVer(pSdb, 0);
     if (sdbVer + 1 != ver) {
       terrno = TSDB_CODE_SDB_INVALID_WAl_VER;
-      mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver);
+      mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer);
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
-    mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
+    mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
-    if (sdbWriteNotFree(pSdb, (void *)pHead->head.body) < 0) {
+    if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) {
-      mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver);
+      mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr());
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     sdbUpdateVer(pSdb, 1);
-    mDebug("wal:%" PRId64 ", is restored", ver);
+    mDebug("ver:%" PRId64 ", is restored", ver);
   }
 
   int64_t sdbVer = sdbUpdateVer(pSdb, 0);
-  mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer);
+  mDebug("restore wal finished, sdbver:%" PRId64, sdbVer);
 
   mndTransPullup(pMnode);
   sdbVer = sdbUpdateVer(pSdb, 0);
-  mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer);
+  mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer);
 
   if (sdbVer != lastSdbVer) {
     mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer);
     if (sdbWriteFile(pSdb) != 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walCommit(pWal, sdbVer) != 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walBeginSnapshot(pWal, sdbVer) < 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walEndSnapshot(pWal) < 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
   }
 
   code = 0;
 
-WAL_RESTORE_OVER:
+_OVER:
   walCloseReadHandle(pHandle);
   return code;
 }
@@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
   int64_t ver = sdbUpdateVer(pSdb, 1);
   if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) {
     sdbUpdateVer(pSdb, -1);
-    mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver);
+    mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr());
     return -1;
   }
 
-  mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver);
+  mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw);
   walCommit(pWal, ver);
   walFsync(pWal, true);
 

source/dnode/mnode/impl/src/mndTelem.c

@@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) {
   taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
   mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
 
-  mDebug("mnode telemetry is initialized");
   return 0;
 }
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -76,7 +76,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER);
   SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER);
   SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER);
-  SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
   SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER);
   SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);
@@ -139,7 +138,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
   SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER);
   SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
-  SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER);
   SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
   SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER);
   SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);
@@ -520,29 +518,33 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
     if (pShow->pIter == NULL) break;
 
+    SColumnInfoData *pColInfo;
+    SName            n;
     int32_t          cols = 0;
 
     char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
-
+    tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB);
-    SName n;
-    tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB);
     tNameGetDbName(&n, varDataVal(topicName));
     varDataSetLen(topicName, strlen(varDataVal(topicName)));
-
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
 
+    char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB);
+    tNameGetDbName(&n, varDataVal(dbName));
+    varDataSetLen(dbName, strlen(varDataVal(dbName)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)dbName, false);
+
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
 
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
     tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
     varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
 
-//    taosMemoryFree(sql);
-
     numOfRows++;
     sdbRelease(pSdb, pTopic);
   }

source/dnode/mnode/impl/src/mndTrans.c

@@ -844,7 +844,7 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
 
   for (int32_t i = 0; i < arraySize; ++i) {
     SSdbRaw *pRaw = taosArrayGetP(pArray, i);
-    int32_t  code = sdbWriteNotFree(pSdb, pRaw);
+    int32_t  code = sdbWriteWithoutFree(pSdb, pRaw);
     if (code != 0) {
       return code;
     }

source/dnode/mnode/impl/src/mndUser.c

@@ -21,8 +21,8 @@
 #include "mndTrans.h"
 #include "tbase64.h"
 
-#define TSDB_USER_VER_NUMBER   1
+#define USER_VER_NUMBER   1
-#define TSDB_USER_RESERVE_SIZE 64
+#define USER_RESERVE_SIZE 64
 
 static int32_t  mndCreateDefaultUsers(SMnode *pMnode);
 static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
@@ -35,7 +35,7 @@ static int32_t  mndProcessCreateUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessAlterUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessDropUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessGetUserAuthReq(SNodeMsg *pReq);
-static int32_t  mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextUser(SMnode *pMnode, void *pIter);
 
 int32_t mndInitUser(SMnode *pMnode) {
@@ -93,9 +93,9 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
 
   int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
   int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
-  int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
+  int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size);
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
   if (pRaw == NULL) goto USER_ENCODE_OVER;
 
   int32_t dataPos = 0;
@@ -120,7 +120,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
     db = taosHashIterate(pUser->writeDbs, db);
   }
 
-  SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER)
+  SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER)
   SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER)
 
   terrno = 0;
@@ -142,7 +142,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
   int8_t sver = 0;
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER;
 
-  if (sver != TSDB_USER_VER_NUMBER) {
+  if (sver != USER_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
     goto USER_DECODE_OVER;
   }
@@ -184,7 +184,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
     taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
   }
 
-  SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER)
+  SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER)
 
   terrno = 0;
 
@@ -639,7 +639,7 @@ GET_AUTH_OVER:
   return code;
 }
 
-static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode   *pMnode = pReq->pNode;
   SSdb     *pSdb = pMnode->pSdb;
   int32_t   numOfRows = 0;
@@ -652,29 +652,29 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
 
     char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]);
 
-    colDataAppend(pColInfo, numOfRows, (const char*) name, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
 
-    const char* src = pUser->superUser? "super":"normal";
+    const char *src = pUser->superUser ? "super" : "normal";
-    char b[10+VARSTR_HEADER_SIZE] = {0};
+    char        b[10 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src));
-    colDataAppend(pColInfo, numOfRows, (const char*) b, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)b, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]);
-    colDataAppend(pColInfo, numOfRows, (const char*) name, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
 
     numOfRows++;
     sdbRelease(pSdb, pUser);

source/dnode/mnode/impl/src/mnode.c

@@ -368,7 +368,7 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
   }
 
   if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) {
-    terrno = TSDB_CODE_MND_INVALID_MSG_LEN;
+    terrno = TSDB_CODE_INVALID_MSG_LEN;
     mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
     return -1;
   }

source/dnode/mnode/impl/test/acct/acct.cpp

@@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
 }
 
 TEST_F(MndTestAcct, 02_Alter_Acct) {
@@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
 }
 
 TEST_F(MndTestAcct, 03_Drop_Acct) {
@@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
 }

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -114,18 +114,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen)
   SMCreateStbReq createReq = {0};
   createReq.numOfColumns = 3;
   createReq.numOfTags = 1;
-  createReq.numOfSmas = 1;
   createReq.igExists = 0;
   createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
   createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
-  createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField));
   strcpy(createReq.name, stbname);
 
   PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts");
   PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1");
   PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2");
   PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1");
-  PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1");
 
   int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
   void*   pHead = rpcMallocCont(tlen);
@@ -190,7 +187,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3
 }
 
 TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
-  #if 0
+#if 0
   const char* dbname = "1.d1";
   const char* stbname = "1.d1.stb";
   const char* smaname = "1.d1.sma";
@@ -264,7 +261,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateBSmaStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
+    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
   }
 
@@ -280,7 +277,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildDropStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
+    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 0);
   }
 

source/dnode/mnode/sdb/src/sdb.c

@@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) {
   return 0;
 }
 
-int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) {
+int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); }
-  pSdb->curVer += val;
-  return pSdb->curVer;
-}

source/dnode/mnode/sdb/src/sdbFile.c

@@ -28,7 +28,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
     if (fp == NULL) continue;
 
     if ((*fp)(pSdb->pMnode) != 0) {
-      mError("failed to deploy sdb:%d since %s", i, terrstr());
+      mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr());
       return -1;
     }
   }
@@ -202,7 +202,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
       break;
     }
 
-    code = sdbWriteNotFree(pSdb, pRaw);
+    code = sdbWriteWithoutFree(pSdb, pRaw);
     if (code != 0) {
       mError("failed to read file:%s since %s", file, terrstr());
       goto PARSE_SDB_DATA_ERROR;
@@ -263,7 +263,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
         continue;
       }
 
-      sdbPrintOper(pSdb, pRow, "writeFile");
+      sdbPrintOper(pSdb, pRow, "write");
 
       SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
       if (pRaw != NULL) {

source/dnode/mnode/sdb/src/sdbHash.c

@@ -51,7 +51,9 @@ const char *sdbTableName(ESdbType type) {
     case SDB_TOPIC:
       return "topic";
     case SDB_VGROUP:
-      return "vgId";
+      return "vgroup";
+    case SDB_SMA:
+      return "sma";
     case SDB_STB:
       return "stb";
     case SDB_DB:
@@ -86,13 +88,13 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) {
   EKeyType keyType = pSdb->keyTypes[pRow->type];
 
   if (keyType == SDB_KEY_BINARY) {
-    mTrace("%s:%s, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount,
+    mTrace("%s:%s, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper,
-           oper, pRow->pObj, sdbStatusStr(pRow->status));
+           pRow->pObj, sdbStatusStr(pRow->status));
   } else if (keyType == SDB_KEY_INT32) {
-    mTrace("%s:%d, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj,
+    mTrace("%s:%d, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount,
-           pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status));
+           oper, pRow->pObj, sdbStatusStr(pRow->status));
   } else if (keyType == SDB_KEY_INT64) {
-    mTrace("%s:%" PRId64 ", refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj,
+    mTrace("%s:%" PRId64 ", ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj,
            pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status));
   } else {
   }
@@ -142,7 +144,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
 
   pRow->refCount = 0;
   pRow->status = pRaw->status;
-  sdbPrintOper(pSdb, pRow, "insertRow");
+  sdbPrintOper(pSdb, pRow, "insert");
 
   if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
     taosWUnLockLatch(pLock);
@@ -191,7 +193,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
 
   SSdbRow *pOldRow = *ppOldRow;
   pOldRow->status = pRaw->status;
-  sdbPrintOper(pSdb, pOldRow, "updateRow");
+  sdbPrintOper(pSdb, pOldRow, "update");
   taosRUnLockLatch(pLock);
 
   int32_t     code = 0;
@@ -220,7 +222,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow *pOldRow = *ppOldRow;
 
   pOldRow->status = pRaw->status;
-  sdbPrintOper(pSdb, pOldRow, "deleteRow");
+  sdbPrintOper(pSdb, pOldRow, "delete");
 
   taosHashRemove(hash, pOldRow->pObj, keySize);
   taosWUnLockLatch(pLock);
@@ -233,7 +235,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   return 0;
 }
 
-int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) {
+int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) {
   SHashObj *hash = sdbGetHash(pSdb, pRaw->type);
   if (hash == NULL) return terrno;
 
@@ -266,7 +268,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) {
 }
 
 int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) {
-  int32_t code = sdbWriteNotFree(pSdb, pRaw);
+  int32_t code = sdbWriteWithoutFree(pSdb, pRaw);
   sdbFreeRaw(pRaw);
   return code;
 }
@@ -296,7 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) {
     case SDB_STATUS_UPDATING:
       atomic_add_fetch_32(&pRow->refCount, 1);
       pRet = pRow->pObj;
-      sdbPrintOper(pSdb, pRow, "acquireRow");
+      sdbPrintOper(pSdb, pRow, "acquire");
       break;
     case SDB_STATUS_CREATING:
       terrno = TSDB_CODE_SDB_OBJ_CREATING;
@@ -318,7 +320,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) {
   taosRLockLatch(pLock);
 
   int32_t ref = atomic_load_32(&pRow->refCount);
-  sdbPrintOper(pSdb, pRow, "checkRow");
+  sdbPrintOper(pSdb, pRow, "check");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
     sdbFreeRow(pSdb, pRow);
   }
@@ -330,13 +332,13 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
   if (pObj == NULL) return;
 
   SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
-  if (pRow->type >= SDB_MAX ) return;
+  if (pRow->type >= SDB_MAX) return;
 
   SRWLatch *pLock = &pSdb->locks[pRow->type];
   taosRLockLatch(pLock);
 
   int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
-  sdbPrintOper(pSdb, pRow, "releaseRow");
+  sdbPrintOper(pSdb, pRow, "release");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
     sdbFreeRow(pSdb, pRow);
   }
@@ -372,7 +374,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
     }
 
     atomic_add_fetch_32(&pRow->refCount, 1);
-    sdbPrintOper(pSdb, pRow, "fetchRow");
+    sdbPrintOper(pSdb, pRow, "fetch");
     *ppObj = pRow->pObj;
     break;
   }

source/dnode/mnode/sdb/src/sdbRaw.c

@@ -27,7 +27,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
   pRaw->sver = sver;
   pRaw->dataLen = dataLen;
 
-  mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
+  mTrace("raw:%p, is created, len:%d table:%s", pRaw, dataLen, sdbTableName(type));
   return pRaw;
 }
 

source/dnode/mnode/sdb/src/sdbRow.c

@@ -43,7 +43,7 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
     (*deleteFp)(pSdb, pRow->pObj);
   }
 
-  sdbPrintOper(pSdb, pRow, "freeRow");
+  sdbPrintOper(pSdb, pRow, "free");
 
   mTrace("row:%p, is freed", pRow->pObj);
   taosMemoryFreeClear(pRow);

source/dnode/vnode/CMakeLists.txt

@@ -5,9 +5,7 @@ target_sources(
     PRIVATE
     # vnode
     "src/vnd/vnodeOpen.c"
-    "src/vnd/vnodeArenaMAImpl.c"
+    "src/vnd/vnodeBufPool.c"
-    "src/vnd/vnodeBufferPool.c"
-    # "src/vnd/vnodeBufferPool2.c"
     "src/vnd/vnodeCfg.c"
     "src/vnd/vnodeCommit.c"
     "src/vnd/vnodeInt.c"
@@ -21,18 +19,19 @@ target_sources(
     "src/meta/metaOpen.c"
     "src/meta/metaIdx.c"
     "src/meta/metaTable.c"
-    "src/meta/metaTDBImpl.c"
+    "src/meta/metaQuery.c"
-    # "src/meta/metaBDBImpl.c"
+    "src/meta/metaCommit.c"
+    "src/meta/metaEntry.c"
 
     # tsdb
     "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
+    "src/tsdb/tsdbCommit2.c"
     "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbFile.c"
     "src/tsdb/tsdbFS.c"
     "src/tsdb/tsdbMain.c"
     "src/tsdb/tsdbMemTable.c"
-    "src/tsdb/tsdbOptions.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
     "src/tsdb/tsdbScan.c"
@@ -71,6 +70,7 @@ target_link_libraries(
     PUBLIC transport
     PUBLIC stream
 )
+target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
 
 if(${BUILD_TEST})
     add_subdirectory(test)

source/dnode/vnode/inc/vnode.h

@@ -32,6 +32,8 @@
 #include "tmsg.h"
 #include "trow.h"
 
+#include "tdbInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -59,7 +61,7 @@ int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);
 int32_t vnodeCompact(SVnode *pVnode);
 int32_t vnodeSync(SVnode *pVnode);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
-int     vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
+int     vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
 
 int32_t vnodeStart(SVnode *pVnode);
 void    vnodeStop(SVnode *pVnode);
@@ -69,27 +71,35 @@ void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
-typedef struct SMTbCursor SMTbCursor;
+typedef struct SMetaReader SMetaReader;
+typedef struct SMetaEntry  SMetaEntry;
 
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void metaReaderClear(SMetaReader *pReader);
+int  metaReadNext(SMetaReader *pReader);
+
+#if 1  // refact APIs below (TODO)
 typedef SVCreateTbReq   STbCfg;
 typedef SVCreateTSmaReq SSmaCfg;
 
+typedef struct SMTbCursor SMTbCursor;
+
 SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
 void        metaCloseTbCursor(SMTbCursor *pTbCur);
-char       *metaTbCursorNext(SMTbCursor *pTbCur);
+int         metaTbCursorNext(SMTbCursor *pTbCur);
+#endif
 
 // tsdb
 typedef struct STsdb STsdb;
-typedef struct STsdbQueryCond STsdbQueryCond;
 typedef void        *tsdbReaderT;
 
 #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
 #define BLOCK_LOAD_TABLE_SEQ_ORDER  2
 #define BLOCK_LOAD_TABLE_RR_ORDER   3
 
-tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
+tsdbReaderT *tsdbQueryTables(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
                              uint64_t taskId);
-tsdbReaderT  tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId,
+tsdbReaderT  tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId,
                                 void *pMemRef);
 int32_t      tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
 bool         isTsdbCacheLastRow(tsdbReaderT *pReader);
@@ -101,6 +111,7 @@ bool         tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
 void         tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
 int32_t      tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
 SArray      *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
+void         tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond);
 void         tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
 int32_t      tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 int32_t      tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
@@ -130,6 +141,7 @@ struct STsdbCfg {
   int8_t   precision;
   int8_t   update;
   int8_t   compression;
+  int8_t   slLevel;
   int32_t  days;
   int32_t  minRows;
   int32_t  maxRows;
@@ -144,10 +156,10 @@ struct SVnodeCfg {
   int32_t  vgId;
   char     dbname[TSDB_DB_NAME_LEN];
   uint64_t dbId;
-  uint64_t wsize;
+  int32_t  szPage;
-  uint64_t ssize;
+  int32_t  szCache;
-  uint64_t lsize;
+  uint64_t szBuf;
-  bool     isHeapAllocator;
+  bool     isHeap;
   uint32_t ttl;
   uint32_t keep;
   int8_t   streamMode;
@@ -160,20 +172,53 @@ struct SVnodeCfg {
   int8_t   hashMethod;
 };
 
-struct STsdbQueryCond {
-  STimeWindow  twindow;
-  int32_t      order;  // desc|asc order to iterate the data block
-  int32_t      numOfCols;
-  SColumnInfo *colList;
-  bool         loadExternalRows;  // load external rows or not
-  int32_t      type;              // data block load type:
-};
-
 typedef struct {
   TSKEY    lastKey;
   uint64_t uid;
 } STableKeyInfo;
 
+struct SMetaEntry {
+  int64_t     version;
+  int8_t      type;
+  tb_uid_t    uid;
+  const char *name;
+  union {
+    struct {
+      SSchemaWrapper schema;
+      SSchemaWrapper schemaTag;
+    } stbEntry;
+    struct {
+      int64_t     ctime;
+      int32_t     ttlDays;
+      tb_uid_t    suid;
+      const void *pTags;
+    } ctbEntry;
+    struct {
+      int64_t        ctime;
+      int32_t        ttlDays;
+      SSchemaWrapper schema;
+    } ntbEntry;
+  };
+};
+
+struct SMetaReader {
+  int32_t    flags;
+  SMeta     *pMeta;
+  SCoder     coder;
+  SMetaEntry me;
+  void      *pBuf;
+  int        szBuf;
+};
+
+struct SMTbCursor {
+  TDBC       *pDbc;
+  void       *pKey;
+  void       *pVal;
+  int         kLen;
+  int         vLen;
+  SMetaReader mr;
+};
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/meta.h

@@ -16,13 +16,14 @@
 #ifndef _TD_VNODE_META_H_
 #define _TD_VNODE_META_H_
 
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 typedef struct SMetaIdx    SMetaIdx;
 typedef struct SMetaDB     SMetaDB;
-typedef struct SMCtbCursor SMCtbCursor;
 typedef struct SMSmaCursor SMSmaCursor;
 
 // metaDebug ==================
@@ -36,8 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor;
 // clang-format on
 
 // metaOpen ==================
-int metaOpen(SVnode* pVnode, SMeta** ppMeta);
+
-int metaClose(SMeta* pMeta);
+// metaEntry ==================
+int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME);
+int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME);
+
+// metaTable ==================
+int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
+
+// metaQuery ==================
+int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid);
 
 // metaIdx ==================
 int  metaOpenIdx(SMeta* pMeta);
@@ -45,32 +54,59 @@ void metaCloseIdx(SMeta* pMeta);
 int  metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
 int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 
+// metaCommit ==================
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
 
-#define META_SUPER_TABLE  TD_SUPER_TABLE
+struct SMeta {
-#define META_CHILD_TABLE  TD_CHILD_TABLE
+  char*     path;
-#define META_NORMAL_TABLE TD_NORMAL_TABLE
+  SVnode*   pVnode;
+  TENV*     pEnv;
+  TXN       txn;
+  TDB*      pTbDb;
+  TDB*      pSkmDb;
+  TDB*      pUidIdx;
+  TDB*      pNameIdx;
+  TDB*      pCtbIdx;
+  TDB*      pTagIdx;
+  TDB*      pTtlIdx;
+  SMetaIdx* pIdx;
+};
 
-int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
+typedef struct {
+  int64_t  version;
+  tb_uid_t uid;
+} STbDbKey;
+
+typedef struct __attribute__((__packed__)) {
+  tb_uid_t uid;
+  int32_t  sver;
+} SSkmDbKey;
+
+typedef struct {
+  tb_uid_t suid;
+  tb_uid_t uid;
+} SCtbIdxKey;
+
+typedef struct __attribute__((__packed__)) {
+  tb_uid_t suid;
+  int16_t  cid;
+  char     data[];
+} STagIdxKey;
+
+typedef struct {
+  int64_t  dtime;
+  tb_uid_t uid;
+} STtlIdxKey;
+
+#if 1
+
+// int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int          metaDropTable(SMeta* pMeta, tb_uid_t uid);
-int             metaCommit(SMeta* pMeta);
-int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
-int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
-STbCfg*         metaGetTbInfoByUid(SMeta* pMeta, tb_uid_t uid);
-STbCfg*         metaGetTbInfoByName(SMeta* pMeta, char* tbname, tb_uid_t* uid);
-SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
-STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
-void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
-STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
-SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
-int             metaGetTbNum(SMeta* pMeta);
 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
 void         metaCloseSmaCursor(SMSmaCursor* pSmaCur);
 int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
-SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
-void            metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
-tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
 
+#ifndef META_REFACT
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
@@ -78,17 +114,9 @@ int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
 int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
 int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
+#endif
 
-// SMetaIdx
+#endif
-
-tb_uid_t metaGenerateUid(SMeta* pMeta);
-
-struct SMeta {
-  char*     path;
-  SVnode*   pVnode;
-  SMetaDB*  pDB;
-  SMetaIdx* pIdx;
-};
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/tq.h

@@ -16,6 +16,8 @@
 #ifndef _TD_VNODE_TQ_H_
 #define _TD_VNODE_TQ_H_
 
+#include "vnodeInt.h"
+
 #include "executor.h"
 #include "os.h"
 #include "tcache.h"
@@ -237,17 +239,7 @@ int  tqInit();
 void tqCleanUp();
 
 // open in each vnode
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
-void tqClose(STQ*);
 // required by vnode
-int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
-int tqCommit(STQ*);
-
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
-int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
-int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
 
 int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
 int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);

source/dnode/vnode/src/inc/tsdb.h

@@ -16,6 +16,8 @@
 #ifndef _TD_VNODE_TSDB_H_
 #define _TD_VNODE_TSDB_H_
 
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -30,29 +32,38 @@ extern "C" {
 #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
+// tsdbMemTable ================
+typedef struct STbData       STbData;
+typedef struct STsdbMemTable STsdbMemTable;
+typedef struct SMergeInfo    SMergeInfo;
+typedef struct STable        STable;
+
+int  tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable);
+void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable);
+int  tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows);
+int  tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
+                           TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
+
+// tsdbCommit ================
+
+#if 1
+
 typedef struct SSmaStat SSmaStat;
 typedef struct SSmaEnv  SSmaEnv;
 typedef struct SSmaEnvs SSmaEnvs;
 
-typedef struct STable {
+struct STable {
   uint64_t  tid;
   uint64_t  uid;
   STSchema *pSchema;
-} STable;
+};
 
 #define TABLE_TID(t) (t)->tid
 #define TABLE_UID(t) (t)->uid
 
-STsdb  *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF);
-void    tsdbClose(STsdb *);
-int     tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
 int     tsdbPrepareCommit(STsdb *pTsdb);
-int     tsdbCommit(STsdb *pTsdb);
 int32_t tsdbInitSma(STsdb *pTsdb);
-int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
 int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
-int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version);
-int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
 int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
 int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
 void    tsdbCleanupReadHandle(tsdbReaderT queryHandle);
@@ -108,25 +119,28 @@ typedef struct {
   TSKEY minKey;
 } SRtn;
 
-typedef struct STbData {
+struct STbData {
   tb_uid_t   uid;
   TSKEY      keyMin;
   TSKEY      keyMax;
+  int64_t    minVer;
+  int64_t    maxVer;
   int64_t    nrows;
   SSkipList *pData;
-} STbData;
+};
 
-typedef struct STsdbMemTable {
+struct STsdbMemTable {
+  SVBufPool *pPool;
   T_REF_DECLARE()
   SRWLatch   latch;
   TSKEY      keyMin;
   TSKEY      keyMax;
-  uint64_t       nRow;
+  int64_t    minVer;
-  SMemAllocator *pMA;
+  int64_t    maxVer;
-  // Container
+  int64_t    nRow;
   SSkipList *pSlIdx;  // SSkiplist<STbData>
   SHashObj  *pHashIdx;
-} STsdbMemTable;
+};
 
 typedef struct {
   uint32_t version;       // Commit version from 0 to increase
@@ -152,16 +166,15 @@ typedef struct {
 } STsdbFS;
 
 struct STsdb {
-  int32_t               vgId;
+  char          *path;
   SVnode        *pVnode;
+  int32_t        vgId;
   bool           repoLocked;
   TdThreadMutex  mutex;
-  char                 *path;
   STsdbCfg       config;
   STsdbMemTable *mem;
   STsdbMemTable *imem;
   SRtn           rtn;
-  SMemAllocatorFactory *pmaf;
   STsdbFS       *fs;
   SSmaEnvs       smaEnvs;
 };
@@ -185,7 +198,7 @@ static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock,
 }
 
 // tsdbMemTable.h
-typedef struct {
+struct SMergeInfo {
   int   rowsInserted;
   int   rowsUpdated;
   int   rowsDeleteSucceed;
@@ -193,7 +206,7 @@ typedef struct {
   int   nOperations;
   TSKEY keyFirst;
   TSKEY keyLast;
-} SMergeInfo;
+};
 
 static void  *taosTMalloc(size_t size);
 static void  *taosTCalloc(size_t nmemb, size_t size);
@@ -202,12 +215,6 @@ static void  *taosTZfree(void *ptr);
 static size_t taosTSizeof(void *ptr);
 static void   taosTMemset(void *ptr, int c);
 
-STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
-void           tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
-int            tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
-int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
-                          TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
-
 static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
   if (pIter == NULL) return NULL;
 
@@ -224,12 +231,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
   return TD_ROW_KEY(row);
 }
 
-// tsdbOptions
-extern const STsdbCfg defautlTsdbOptions;
-
-int  tsdbValidateOptions(const STsdbCfg *);
-void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
-
 // tsdbReadImpl
 typedef struct SReadH SReadH;
 
@@ -987,6 +988,8 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void
   return len;
 }
 
+#endif
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tsdbSma.h

@@ -16,9 +16,7 @@
 #ifndef _TD_VNODE_TSDB_SMA_H_
 #define _TD_VNODE_TSDB_SMA_H_
 
-#include "os.h"
+#include "tsdb.h"
-#include "thash.h"
-#include "tmsg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -32,12 +30,6 @@ struct STbDdlH {
   __tb_ddl_fn_t fp;
 };
 
-typedef struct {
-  tb_uid_t  suid;
-  SArray   *tbUids;
-  SHashObj *uidHash;
-} STbUidStore;
-
 static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
   ASSERT(*pStore == NULL);
   *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));

source/dnode/vnode/src/inc/vnd.h

@@ -16,6 +16,10 @@
 #ifndef _TD_VND_H_
 #define _TD_VND_H_
 
+#include "sync.h"
+#include "syncTools.h"
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -31,69 +35,65 @@ extern "C" {
 // clang-format on
 
 // vnodeCfg ====================
+extern const SVnodeCfg vnodeCfgDefault;
+
+int vnodeCheckCfg(const SVnodeCfg*);
 int vnodeEncodeConfig(const void* pObj, SJson* pJson);
 int vnodeDecodeConfig(const SJson* pJson, void* pObj);
 
 // vnodeModule ====================
 int vnodeScheduleTask(int (*execute)(void*), void* arg);
 
+// vnodeBufPool ====================
+typedef struct SVBufPoolNode SVBufPoolNode;
+struct SVBufPoolNode {
+  SVBufPoolNode*  prev;
+  SVBufPoolNode** pnext;
+  int64_t         size;
+  uint8_t         data[];
+};
+
+struct SVBufPool {
+  SVBufPool*     next;
+  int64_t        nRef;
+  int64_t        size;
+  uint8_t*       ptr;
+  SVBufPoolNode* pTail;
+  SVBufPoolNode  node;
+};
+
+int  vnodeOpenBufPool(SVnode* pVnode, int64_t size);
+int  vnodeCloseBufPool(SVnode* pVnode);
+void vnodeBufPoolReset(SVBufPool* pPool);
+
 // vnodeQuery ====================
 int  vnodeQueryOpen(SVnode* pVnode);
 void vnodeQueryClose(SVnode* pVnode);
 int  vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg);
 
 // vnodeCommit ====================
+int vnodeBegin(SVnode* pVnode);
+int vnodeShouldCommit(SVnode* pVnode);
+int vnodeCommit(SVnode* pVnode);
 int vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
 int vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);
 int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
-int vnodeBegin(SVnode* pVnode, int option);
 int vnodeSyncCommit(SVnode* pVnode);
 int vnodeAsyncCommit(SVnode* pVnode);
 
-#define vnodeShouldCommit vnodeBufPoolIsFull
+// vnodeCommit ====================
-
+int32_t   vnodeSyncOpen(SVnode* pVnode, char* path);
-#if 1
+int32_t   vnodeSyncStart(SVnode* pVnode);
-// SVBufPool
+void      vnodeSyncClose(SVnode* pVnode);
-int   vnodeOpenBufPool(SVnode* pVnode);
+void      vnodeSyncSetQ(SVnode* pVnode, void* qHandle);
-void  vnodeCloseBufPool(SVnode* pVnode);
+void      vnodeSyncSetRpc(SVnode* pVnode, void* rpcHandle);
-int   vnodeBufPoolSwitch(SVnode* pVnode);
+int32_t   vnodeSyncEqMsg(void* qHandle, SRpcMsg* pMsg);
-int   vnodeBufPoolRecycle(SVnode* pVnode);
+int32_t   vnodeSendMsg(void* rpcHandle, const SEpSet* pEpSet, SRpcMsg* pMsg);
-void* vnodeMalloc(SVnode* pVnode, uint64_t size);
+void      vnodeSyncCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-bool  vnodeBufPoolIsFull(SVnode* pVnode);
+void      vnodeSyncPreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-
+void      vnodeSyncRollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
-SMemAllocatorFactory* vBufPoolGetMAF(SVnode* pVnode);
+int32_t   vnodeSyncGetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
-
+SSyncFSM* syncVnodeMakeFsm();
-// SVMemAllocator
-typedef struct SVArenaNode {
-  TD_SLIST_NODE(SVArenaNode);
-  uint64_t size;  // current node size
-  void*    ptr;
-  char     data[];
-} SVArenaNode;
-
-typedef struct SVMemAllocator {
-  T_REF_DECLARE()
-  TD_DLIST_NODE(SVMemAllocator);
-  uint64_t     capacity;
-  uint64_t     ssize;
-  uint64_t     lsize;
-  SVArenaNode* pNode;
-  TD_SLIST(SVArenaNode) nlist;
-} SVMemAllocator;
-
-SVMemAllocator* vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
-void            vmaDestroy(SVMemAllocator* pVMA);
-void            vmaReset(SVMemAllocator* pVMA);
-void*           vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
-void            vmaFree(SVMemAllocator* pVMA, void* ptr);
-bool            vmaIsFull(SVMemAllocator* pVMA);
-
-// vnodeCfg.h
-extern const SVnodeCfg vnodeCfgDefault;
-
-int vnodeCheckCfg(const SVnodeCfg*);
-
-#endif
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -22,9 +22,11 @@
 #include "sync.h"
 #include "tchecksum.h"
 #include "tcoding.h"
+#include "tcompare.h"
 #include "tcompression.h"
 #include "tdatablock.h"
 #include "tdbInt.h"
+#include "tencode.h"
 #include "tfs.h"
 #include "tglobal.h"
 #include "tjson.h"
@@ -57,6 +59,55 @@ typedef struct SQWorkerMgmt SQHandle;
 #define VNODE_TQ_DIR   "tq"
 #define VNODE_WAL_DIR  "wal"
 
+// vnd.h
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
+
+// meta
+typedef struct SMCtbCursor SMCtbCursor;
+typedef struct STbUidStore STbUidStore;
+
+int             metaOpen(SVnode* pVnode, SMeta** ppMeta);
+int             metaClose(SMeta* pMeta);
+int             metaBegin(SMeta* pMeta);
+int             metaCommit(SMeta* pMeta);
+int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
+int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
+SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
+STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
+int             metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid);
+int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
+int             metaGetTbNum(SMeta* pMeta);
+SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
+void            metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
+tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
+SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
+void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
+STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
+int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
+int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
+
+// tsdb
+int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb);
+int     tsdbClose(STsdb* pTsdb);
+int     tsdbBegin(STsdb* pTsdb);
+int     tsdbCommit(STsdb* pTsdb);
+int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
+int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
+int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
+int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
+
+// tq
+STQ*    tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
+void    tqClose(STQ*);
+int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
+int     tqCommit(STQ*);
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
+int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
+
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
@@ -68,8 +119,8 @@ typedef struct {
 } SStreamSinkInfo;
 
 typedef struct {
-  SVnode   *pVnode;
+  SVnode*   pVnode;
-  SHashObj *pHash;  // streamId -> SStreamSinkInfo
+  SHashObj* pHash;  // streamId -> SStreamSinkInfo
 } SSink;
 
 // SVState
@@ -85,20 +136,29 @@ struct SVnodeInfo {
 };
 
 struct SVnode {
-  char      *path;
+  char*      path;
   SVnodeCfg  config;
   SVState    state;
-  STfs      *pTfs;
+  STfs*      pTfs;
   SMsgCb     msgCb;
-  SVBufPool *pBufPool;
+  SVBufPool* pPool;
-  SMeta     *pMeta;
+  SVBufPool* inUse;
-  STsdb     *pTsdb;
+  SVBufPool* onCommit;
-  SWal      *pWal;
+  SVBufPool* onRecycle;
-  STQ       *pTq;
+  SMeta*     pMeta;
-  SSink     *pSink;
+  STsdb*     pTsdb;
-  int64_t    sync;  // sync integration
+  SWal*      pWal;
+  STQ*       pTq;
+  SSink*     pSink;
+  int64_t    sync;
   tsem_t     canCommit;
-  SQHandle  *pQuery;
+  SQHandle*  pQuery;
+};
+
+struct STbUidStore {
+  tb_uid_t  suid;
+  SArray*   tbUids;
+  SHashObj* uidHash;
 };
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
@@ -106,19 +166,7 @@ struct SVnode {
 typedef struct STbDdlH STbDdlH;
 
 // sma
-void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
+void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
-
-#include "vnd.h"
-
-#include "meta.h"
-
-#include "tsdb.h"
-
-#include "tq.h"
-
-#include "vnodeSync.h"
-
-#include "tsdbSma.h"
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeSync.h

@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_SYNC_H_
-#define _TD_VNODE_SYNC_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
-int32_t vnodeSyncStart(SVnode *pVnode);
-void    vnodeSyncClose(SVnode *pVnode);
-
-void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
-void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
-
-int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
-int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
-
-void    vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-void    vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-void    vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
-
-SSyncFSM *syncVnodeMakeFsm();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_SYNC_H_*/

source/dnode/vnode/src/meta/metaCommit.c

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); }
+static FORCE_INLINE void  metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
+
+int metaBegin(SMeta *pMeta) {
+  tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); }

source/dnode/vnode/src/meta/metaEntry.c

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI64(pCoder, pME->version) < 0) return -1;
+  if (tEncodeI8(pCoder, pME->type) < 0) return -1;
+  if (tEncodeI64(pCoder, pME->uid) < 0) return -1;
+  if (tEncodeCStr(pCoder, pME->name) < 0) return -1;
+
+  if (pME->type == TSDB_SUPER_TABLE) {
+    if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
+    if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+  } else if (pME->type == TSDB_CHILD_TABLE) {
+    if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
+    if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
+    if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
+    if (tEncodeBinary(pCoder, pME->ctbEntry.pTags, kvRowLen(pME->ctbEntry.pTags)) < 0) return -1;
+  } else if (pME->type == TSDB_NORMAL_TABLE) {
+    if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
+    if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+  } else {
+    ASSERT(0);
+  }
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+int metaDecodeEntry(SCoder *pCoder, SMetaEntry *pME) {
+  uint64_t len;
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI64(pCoder, &pME->version) < 0) return -1;
+  if (tDecodeI8(pCoder, &pME->type) < 0) return -1;
+  if (tDecodeI64(pCoder, &pME->uid) < 0) return -1;
+  if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
+
+  if (pME->type == TSDB_SUPER_TABLE) {
+    if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
+    if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+  } else if (pME->type == TSDB_CHILD_TABLE) {
+    if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
+    if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
+    if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
+    if (tDecodeBinary(pCoder, &pME->ctbEntry.pTags, &len) < 0) return -1;  // (TODO)
+  } else if (pME->type == TSDB_NORMAL_TABLE) {
+    if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
+    if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
+    if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+  } else {
+    ASSERT(0);
+  }
+
+  tEndDecode(pCoder);
+  return 0;
+}

source/dnode/vnode/src/meta/metaIdx.c

@@ -16,7 +16,7 @@
 #ifdef USE_INVERTED_INDEX
 #include "index.h"
 #endif
-#include "vnodeInt.h"
+#include "meta.h"
 
 struct SMetaIdx {
 #ifdef USE_INVERTED_INDEX
@@ -54,7 +54,9 @@ int metaOpenIdx(SMeta *pMeta) {
 
 #ifdef USE_INVERTED_INDEX
   SIndexOpts opts;
-  if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { return -1; }
+  if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
+    return -1;
+  }
 
 #endif
   return 0;
@@ -70,7 +72,9 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */
 
 #ifdef USE_INVERTED_INDEX
   SIndexOpts opts;
-  if (indexClose(pMeta->pIdx->pIdx) != 0) { return -1; }
+  if (indexClose(pMeta->pIdx->pIdx) != 0) {
+    return -1;
+  }
 
 #endif
 }
@@ -87,7 +91,7 @@ int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
     tb_uid_t         suid = pTbCfg->ctbCfg.suid;  // super id
     tb_uid_t         tuid = 0;                    // child table uid
     SIndexMultiTerm *terms = indexMultiTermCreate();
-    SIndexTerm *     term =
+    SIndexTerm      *term =
         indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
     indexMultiTermAdd(terms, term);
 
@@ -117,10 +121,13 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
 
   // TODO: add atomicity
 
+#ifdef META_REFACT
+#else
   if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
     // TODO: handle error
     return -1;
   }
+#endif
   return TSDB_CODE_SUCCESS;
 }
 
@@ -128,9 +135,12 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
   // TODO: Validate the cfg
   // TODO: add atomicity
 
+#ifdef META_REFACT
+#else
   if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
     // TODO: handle error
     return -1;
   }
+#endif
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/meta/metaOpen.c

@@ -13,10 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
+
+static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
 int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   SMeta *pMeta = NULL;
+  int    ret;
   int    slen;
 
   *ppMeta = NULL;
@@ -36,31 +44,240 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   // create path if not created yet
   taosMkDir(pMeta->path);
 
-  // open meta
+  // open env
-  if (metaOpenDB(pMeta) < 0) {
+  ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
 
-  if (metaOpenIdx(pMeta) < 0) {
+  // open pTbDb
+  ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
 
+  // open pSkmDb
+  ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open pUidIdx
+  ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open pNameIdx
+  ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open pCtbIdx
+  ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open pTagIdx
+  ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open pTtlIdx
+  ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
+  if (ret < 0) {
+    metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  // open index
+  if (metaOpenIdx(pMeta) < 0) {
+    metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
+  }
+
+  metaDebug("vgId: %d meta is opened", TD_VID(pVnode));
+
   *ppMeta = pMeta;
   return 0;
 
 _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
-  if (pMeta->pDB) metaCloseDB(pMeta);
+  if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
+  if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
+  if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
+  if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
+  if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+  if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
+  if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
+  if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
   taosMemoryFree(pMeta);
   return -1;
 }
 
 int metaClose(SMeta *pMeta) {
   if (pMeta) {
-    metaCloseIdx(pMeta);
+    if (pMeta->pIdx) metaCloseIdx(pMeta);
-    metaCloseDB(pMeta);
+    if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
+    if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
+    if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
+    if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
+    if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+    if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
+    if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
+    if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
     taosMemoryFree(pMeta);
   }
 
   return 0;
 }
+
+static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1;
+  STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2;
+
+  if (pTbDbKey1->version > pTbDbKey2->version) {
+    return 1;
+  } else if (pTbDbKey1->version < pTbDbKey2->version) {
+    return -1;
+  }
+
+  if (pTbDbKey1->uid > pTbDbKey2->uid) {
+    return 1;
+  } else if (pTbDbKey1->uid < pTbDbKey2->uid) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  SSkmDbKey *pSkmDbKey1 = (SSkmDbKey *)pKey1;
+  SSkmDbKey *pSkmDbKey2 = (SSkmDbKey *)pKey2;
+
+  if (pSkmDbKey1->uid > pSkmDbKey2->uid) {
+    return 1;
+  } else if (pSkmDbKey1->uid < pSkmDbKey2->uid) {
+    return -1;
+  }
+
+  if (pSkmDbKey1->sver > pSkmDbKey2->sver) {
+    return 1;
+  } else if (pSkmDbKey1->sver < pSkmDbKey2->sver) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  tb_uid_t uid1 = *(tb_uid_t *)pKey1;
+  tb_uid_t uid2 = *(tb_uid_t *)pKey2;
+
+  if (uid1 > uid2) {
+    return 1;
+  } else if (uid1 < uid2) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  SCtbIdxKey *pCtbIdxKey1 = (SCtbIdxKey *)pKey1;
+  SCtbIdxKey *pCtbIdxKey2 = (SCtbIdxKey *)pKey2;
+
+  if (pCtbIdxKey1->suid > pCtbIdxKey2->suid) {
+    return 1;
+  } else if (pCtbIdxKey1->suid < pCtbIdxKey2->suid) {
+    return -1;
+  }
+
+  if (pCtbIdxKey1->uid > pCtbIdxKey2->uid) {
+    return 1;
+  } else if (pCtbIdxKey1->uid < pCtbIdxKey2->uid) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1;
+  STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2;
+  int8_t     *p1, *p2;
+  int8_t      type;
+  int         c;
+
+  // compare suid
+  if (pTagIdxKey1->suid > pTagIdxKey2->suid) {
+    return 1;
+  } else if (pTagIdxKey1->suid < pTagIdxKey2->suid) {
+    return -1;
+  }
+
+  // compare column id
+  if (pTagIdxKey1->cid > pTagIdxKey2->cid) {
+    return 1;
+  } else if (pTagIdxKey1->cid < pTagIdxKey2->cid) {
+    return -1;
+  }
+
+  // compare value
+  p1 = pTagIdxKey1->data;
+  p2 = pTagIdxKey2->data;
+  ASSERT(p1[0] == p2[0]);
+  type = p1[0];
+
+  p1++;
+  p2++;
+
+  c = doCompare(p1, p2, type, 0);
+  if (c) return c;
+
+  if (IS_VAR_DATA_TYPE(type)) {
+    p1 = p1 + varDataTLen(p1);
+    p2 = p2 + varDataTLen(p2);
+  } else {
+    p1 = p1 + tDataTypes[type].bytes;
+    p2 = p2 + tDataTypes[type].bytes;
+  }
+
+  // compare suid
+  if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) {
+    return 1;
+  } else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
+  STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
+
+  if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) {
+    return 1;
+  } else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) {
+    return -1;
+  }
+
+  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
+    return 1;
+  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/src/meta/metaQuery.c

@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "meta.h"
+
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
+  memset(pReader, 0, sizeof(*pReader));
+  pReader->flags = flags;
+  pReader->pMeta = pMeta;
+}
+
+void metaReaderClear(SMetaReader *pReader) {
+  tCoderClear(&pReader->coder);
+  TDB_FREE(pReader->pBuf);
+}
+
+int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) {
+  SMeta   *pMeta = pReader->pMeta;
+  STbDbKey tbDbKey = {.version = version, .uid = uid};
+
+  // query table.db
+  if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
+    goto _err;
+  }
+
+  // decode the entry
+  tCoderInit(&pReader->coder, TD_LITTLE_ENDIAN, pReader->pBuf, pReader->szBuf, TD_DECODER);
+
+  if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) {
+    goto _err;
+  }
+
+  return 0;
+
+_err:
+  return -1;
+}
+
+int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
+  SMeta  *pMeta = pReader->pMeta;
+  int64_t version;
+
+  // query uid.idx
+  if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
+    return -1;
+  }
+
+  version = *(int64_t *)pReader->pBuf;
+  return metaGetTableEntryByVersion(pReader, version, uid);
+}
+
+int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
+  SMeta   *pMeta = pReader->pMeta;
+  tb_uid_t uid;
+
+  // query name.idx
+  if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
+    return -1;
+  }
+
+  uid = *(tb_uid_t *)pReader->pBuf;
+  return metaGetTableEntryByUid(pReader, uid);
+}
+
+int metaReadNext(SMetaReader *pReader) {
+  SMeta *pMeta = pReader->pMeta;
+
+  // TODO
+
+  return 0;
+}
+
+#if 1  // ===================================================
+SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
+  SMTbCursor *pTbCur = NULL;
+
+  pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
+  if (pTbCur == NULL) {
+    return NULL;
+  }
+
+  metaReaderInit(&pTbCur->mr, pMeta, 0);
+
+  tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc);
+
+  return pTbCur;
+}
+
+void metaCloseTbCursor(SMTbCursor *pTbCur) {
+  if (pTbCur) {
+    TDB_FREE(pTbCur->pKey);
+    TDB_FREE(pTbCur->pVal);
+    metaReaderClear(&pTbCur->mr);
+    if (pTbCur->pDbc) {
+      tdbDbcClose(pTbCur->pDbc);
+    }
+    taosMemoryFree(pTbCur);
+  }
+}
+
+int metaTbCursorNext(SMTbCursor *pTbCur) {
+  int    ret;
+  void  *pBuf;
+  STbCfg tbCfg;
+
+  for (;;) {
+    ret = tdbDbNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+    if (ret < 0) {
+      return -1;
+    }
+
+    metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
+    if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
+      continue;
+    }
+
+    break;
+  }
+
+  return 0;
+}
+
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  void           *pKey = NULL;
+  void           *pVal = NULL;
+  int             kLen = 0;
+  int             vLen = 0;
+  int             ret;
+  SSkmDbKey       skmDbKey;
+  SSchemaWrapper *pSW = NULL;
+  SSchema        *pSchema = NULL;
+  void           *pBuf;
+  SCoder          coder = {0};
+
+  // fetch
+  skmDbKey.uid = uid;
+  skmDbKey.sver = sver;
+  pKey = &skmDbKey;
+  kLen = sizeof(skmDbKey);
+  ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen);
+  if (ret < 0) {
+    return NULL;
+  }
+
+  // decode
+  pBuf = pVal;
+  pSW = taosMemoryMalloc(sizeof(pSW));
+
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_DECODER);
+  tDecodeSSchemaWrapper(&coder, pSW);
+  pSchema = taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+  memcpy(pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
+  tCoderClear(&coder);
+
+  pSW->pSchema = pSchema;
+
+  TDB_FREE(pVal);
+
+  return pSW;
+}
+
+struct SMCtbCursor {
+  TDBC    *pCur;
+  tb_uid_t suid;
+  void    *pKey;
+  void    *pVal;
+  int      kLen;
+  int      vLen;
+};
+
+SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
+  SMCtbCursor *pCtbCur = NULL;
+  int          ret;
+
+  pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
+  if (pCtbCur == NULL) {
+    return NULL;
+  }
+
+  pCtbCur->suid = uid;
+  ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur);
+  if (ret < 0) {
+    taosMemoryFree(pCtbCur);
+    return NULL;
+  }
+
+  return pCtbCur;
+}
+
+void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
+  if (pCtbCur) {
+    if (pCtbCur->pCur) {
+      tdbDbcClose(pCtbCur->pCur);
+
+      TDB_FREE(pCtbCur->pKey);
+      TDB_FREE(pCtbCur->pVal);
+    }
+
+    taosMemoryFree(pCtbCur);
+  }
+}
+
+tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
+  int         ret;
+  SCtbIdxKey *pCtbIdxKey;
+
+  ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
+  if (ret < 0) {
+    return 0;
+  }
+
+  pCtbIdxKey = pCtbCur->pKey;
+
+  return pCtbIdxKey->uid;
+}
+
+STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
+  tb_uid_t        quid;
+  SMetaReader     mr = {0};
+  STSchema       *pTSchema = NULL;
+  SSchemaWrapper *pSW = NULL;
+  STSchemaBuilder sb = {0};
+  SSchema        *pSchema;
+
+  metaReaderInit(&mr, pMeta, 0);
+  metaGetTableEntryByUid(&mr, uid);
+
+  if (mr.me.type == TSDB_CHILD_TABLE) {
+    quid = mr.me.ctbEntry.suid;
+  } else {
+    quid = uid;
+  }
+
+  metaReaderClear(&mr);
+
+  pSW = metaGetTableSchema(pMeta, quid, sver, 0);
+  tdInitTSchemaBuilder(&sb, 0);
+  for (int i = 0; i < pSW->nCols; i++) {
+    pSchema = pSW->pSchema + i;
+    tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
+  }
+  pTSchema = tdGetSchemaFromBuilder(&sb);
+  tdDestroyTSchemaBuilder(&sb);
+
+  taosMemoryFree(pSW->pSchema);
+  taosMemoryFree(pSW);
+  return pTSchema;
+}
+
+STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
+#if 0
+#ifdef META_TDB_SMA_TEST
+  STSmaWrapper *pSW = NULL;
+
+  pSW = taosMemoryCalloc(1, sizeof(*pSW));
+  if (pSW == NULL) {
+    return NULL;
+  }
+
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
+  if (pCur == NULL) {
+    taosMemoryFree(pSW);
+    return NULL;
+  }
+
+  void       *pBuf = NULL;
+  SSmaIdxKey *pSmaIdxKey = NULL;
+
+  while (true) {
+    // TODO: lock during iterate?
+    if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
+      pSmaIdxKey = pCur->pKey;
+      ASSERT(pSmaIdxKey != NULL);
+
+      void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
+
+      if (pSmaVal == NULL) {
+        tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
+        continue;
+      }
+
+      ++pSW->number;
+      STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
+      if (tptr == NULL) {
+        TDB_FREE(pSmaVal);
+        metaCloseSmaCursor(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        taosMemoryFreeClear(pSW);
+        return NULL;
+      }
+      pSW->tSma = tptr;
+      pBuf = pSmaVal;
+      if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
+        TDB_FREE(pSmaVal);
+        metaCloseSmaCursor(pCur);
+        tdDestroyTSmaWrapper(pSW);
+        taosMemoryFreeClear(pSW);
+        return NULL;
+      }
+      TDB_FREE(pSmaVal);
+      continue;
+    }
+    break;
+  }
+
+  metaCloseSmaCursor(pCur);
+
+  return pSW;
+
+#endif
+#endif
+  return NULL;
+}
+
+int metaGetTbNum(SMeta *pMeta) {
+  // TODO
+  // ASSERT(0);
+  return 0;
+}
+
+SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
+#if 0
+  // TODO
+  // ASSERT(0); // comment this line to pass CI
+  // return NULL:
+#ifdef META_TDB_SMA_TEST
+  SArray  *pUids = NULL;
+  SMetaDB *pDB = pMeta->pDB;
+  void    *pKey;
+
+  // TODO: lock?
+  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
+  if (pCur == NULL) {
+    return NULL;
+  }
+  // TODO: lock?
+
+  SSmaIdxKey *pSmaIdxKey = NULL;
+  tb_uid_t    uid = 0;
+  while (true) {
+    // TODO: lock during iterate?
+    if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
+      ASSERT(pSmaIdxKey != NULL);
+      pSmaIdxKey = pCur->pKey;
+
+      if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
+        continue;
+      }
+      uid = pSmaIdxKey->uid;
+
+      if (!pUids) {
+        pUids = taosArrayInit(16, sizeof(tb_uid_t));
+        if (!pUids) {
+          metaCloseSmaCursor(pCur);
+          return NULL;
+        }
+      }
+
+      taosArrayPush(pUids, &uid);
+
+      continue;
+    }
+    break;
+  }
+
+  metaCloseSmaCursor(pCur);
+
+  return pUids;
+#endif
+#endif
+  return NULL;
+}
+
+void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
+#if 0
+  // TODO
+  // ASSERT(0);
+  // return NULL;
+#ifdef META_TDB_SMA_TEST
+  SMetaDB *pDB = pMeta->pDB;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
+  int      kLen = 0;
+  int      vLen = 0;
+  int      ret = -1;
+
+  // Set key
+  pKey = (void *)&indexUid;
+  kLen = sizeof(indexUid);
+
+  // Query
+  ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
+  if (ret != 0 || !pVal) {
+    return NULL;
+  }
+
+  if (!isDecode) {
+    // return raw value
+    return pVal;
+  }
+
+  // Decode
+  STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
+  if (pCfg == NULL) {
+    taosMemoryFree(pVal);
+    return NULL;
+  }
+
+  void *pBuf = pVal;
+  if (tDecodeTSma(pBuf, pCfg) == NULL) {
+    tdDestroyTSma(pCfg);
+    taosMemoryFree(pCfg);
+    TDB_FREE(pVal);
+    return NULL;
+  }
+
+  TDB_FREE(pVal);
+  return pCfg;
+#endif
+#endif
+  return NULL;
+}
+
+#endif

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -15,6 +15,8 @@
 
 #include "vnodeInt.h"
 
+#ifndef META_REFACT
+
 typedef struct SPoolMem {
   int64_t          size;
   struct SPoolMem *prev;
@@ -45,7 +47,7 @@ struct SMetaDB {
 #endif
 };
 
-#pragma pack(push,1)
+#pragma pack(push, 1)
 typedef struct {
   tb_uid_t uid;
   int32_t  sver;
@@ -379,64 +381,6 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
   return 0;
 }
 
-STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
-  int      ret;
-  SMetaDB *pMetaDb = pMeta->pDB;
-  void    *pKey;
-  void    *pVal;
-  int      kLen;
-  int      vLen;
-  STbCfg  *pTbCfg;
-
-  // Fetch
-  pKey = &uid;
-  kLen = sizeof(uid);
-  pVal = NULL;
-  ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen);
-  if (ret < 0) {
-    return NULL;
-  }
-
-  // Decode
-  pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg));
-  metaDecodeTbInfo(pVal, pTbCfg);
-
-  TDB_FREE(pVal);
-
-  return pTbCfg;
-}
-
-STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
-  void *pKey;
-  void *pVal;
-  void *ppKey;
-  int   pkLen;
-  int   kLen;
-  int   vLen;
-  int   ret;
-
-  pKey = tbname;
-  kLen = strlen(tbname) + 1;
-  pVal = NULL;
-  ppKey = NULL;
-  ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen);
-  if (ret < 0) {
-    return NULL;
-  }
-
-  ASSERT(pkLen == kLen + sizeof(uid));
-
-  *uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen);
-  TDB_FREE(ppKey);
-  TDB_FREE(pVal);
-
-  return metaGetTbInfoByUid(pMeta, *uid);
-}
-
-SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
-  return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
-}
-
 static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
   void           *pKey;
   void           *pVal;
@@ -468,156 +412,6 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_
   return pSchemaWrapper;
 }
 
-STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
-  tb_uid_t        quid;
-  SSchemaWrapper *pSW;
-  STSchemaBuilder sb;
-  SSchema        *pSchema;
-  STSchema       *pTSchema;
-  STbCfg         *pTbCfg;
-
-  pTbCfg = metaGetTbInfoByUid(pMeta, uid);
-  if (pTbCfg->type == META_CHILD_TABLE) {
-    quid = pTbCfg->ctbCfg.suid;
-  } else {
-    quid = uid;
-  }
-
-  pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
-  if (pSW == NULL) {
-    return NULL;
-  }
-
-  tdInitTSchemaBuilder(&sb, 0);
-  for (int i = 0; i < pSW->nCols; i++) {
-    pSchema = pSW->pSchema + i;
-    tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
-  }
-  pTSchema = tdGetSchemaFromBuilder(&sb);
-  tdDestroyTSchemaBuilder(&sb);
-
-  return pTSchema;
-}
-
-struct SMTbCursor {
-  TDBC *pDbc;
-};
-
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
-  SMTbCursor *pTbCur = NULL;
-  SMetaDB    *pDB = pMeta->pDB;
-
-  pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
-  if (pTbCur == NULL) {
-    return NULL;
-  }
-
-  tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc);
-
-  return pTbCur;
-}
-
-void metaCloseTbCursor(SMTbCursor *pTbCur) {
-  if (pTbCur) {
-    if (pTbCur->pDbc) {
-      tdbDbcClose(pTbCur->pDbc);
-    }
-    taosMemoryFree(pTbCur);
-  }
-}
-
-char *metaTbCursorNext(SMTbCursor *pTbCur) {
-  void  *pKey = NULL;
-  void  *pVal = NULL;
-  int    kLen;
-  int    vLen;
-  int    ret;
-  void  *pBuf;
-  STbCfg tbCfg;
-
-  for (;;) {
-    ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen);
-    if (ret < 0) break;
-    pBuf = pVal;
-    metaDecodeTbInfo(pBuf, &tbCfg);
-    if (tbCfg.type == META_SUPER_TABLE) {
-      taosMemoryFree(tbCfg.name);
-      taosMemoryFree(tbCfg.stbCfg.pTagSchema);
-      continue;
-    } else if (tbCfg.type == META_CHILD_TABLE) {
-      kvRowFree(tbCfg.ctbCfg.pTag);
-    }
-
-    return tbCfg.name;
-  }
-
-  return NULL;
-}
-
-struct SMCtbCursor {
-  TDBC    *pCur;
-  tb_uid_t suid;
-  void    *pKey;
-  void    *pVal;
-  int      kLen;
-  int      vLen;
-};
-
-SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
-  SMCtbCursor *pCtbCur = NULL;
-  SMetaDB     *pDB = pMeta->pDB;
-  int          ret;
-
-  pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
-  if (pCtbCur == NULL) {
-    return NULL;
-  }
-
-  pCtbCur->suid = uid;
-  ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur);
-  if (ret < 0) {
-    taosMemoryFree(pCtbCur);
-    return NULL;
-  }
-
-  // TODO: move the cursor to the suid there
-
-  return pCtbCur;
-}
-
-void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
-  if (pCtbCur) {
-    if (pCtbCur->pCur) {
-      tdbDbcClose(pCtbCur->pCur);
-
-      TDB_FREE(pCtbCur->pKey);
-      TDB_FREE(pCtbCur->pVal);
-    }
-
-    taosMemoryFree(pCtbCur);
-  }
-}
-
-tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
-  int         ret;
-  SCtbIdxKey *pCtbIdxKey;
-
-  ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
-  if (ret < 0) {
-    return 0;
-  }
-
-  pCtbIdxKey = pCtbCur->pKey;
-
-  return pCtbIdxKey->uid;
-}
-
-int metaGetTbNum(SMeta *pMeta) {
-  // TODO
-  // ASSERT(0);
-  return 0;
-}
-
 struct SMSmaCursor {
   TDBC    *pCur;
   tb_uid_t uid;
@@ -768,53 +562,6 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   return 0;
 }
 
-void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
-  // TODO
-  // ASSERT(0);
-  // return NULL;
-#ifdef META_TDB_SMA_TEST
-  SMetaDB *pDB = pMeta->pDB;
-  void    *pKey = NULL;
-  void    *pVal = NULL;
-  int      kLen = 0;
-  int      vLen = 0;
-  int      ret = -1;
-
-  // Set key
-  pKey = (void *)&indexUid;
-  kLen = sizeof(indexUid);
-
-  // Query
-  ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
-  if (ret != 0 || !pVal) {
-    return NULL;
-  }
-
-  if (!isDecode) {
-    // return raw value
-    return pVal;
-  }
-
-  // Decode
-  STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
-  if (pCfg == NULL) {
-    taosMemoryFree(pVal);
-    return NULL;
-  }
-
-  void *pBuf = pVal;
-  if (tDecodeTSma(pBuf, pCfg) == NULL) {
-    tdDestroyTSma(pCfg);
-    taosMemoryFree(pCfg);
-    TDB_FREE(pVal);
-    return NULL;
-  }
-
-  TDB_FREE(pVal);
-  return pCfg;
-#endif
-}
-
 /**
  * @brief
  *
@@ -889,56 +636,6 @@ void metaCloseSmaCursor(SMSmaCursor *pCur) {
 #endif
 }
 
-SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
-  // TODO
-  // ASSERT(0); // comment this line to pass CI
-  // return NULL:
-#ifdef META_TDB_SMA_TEST
-  SArray  *pUids = NULL;
-  SMetaDB *pDB = pMeta->pDB;
-  void    *pKey;
-
-  // TODO: lock?
-  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
-  if (pCur == NULL) {
-    return NULL;
-  }
-  // TODO: lock?
-
-  SSmaIdxKey *pSmaIdxKey = NULL;
-  tb_uid_t    uid = 0;
-  while (true) {
-    // TODO: lock during iterate?
-    if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
-      ASSERT(pSmaIdxKey != NULL);
-      pSmaIdxKey = pCur->pKey;
-
-      if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
-        continue;
-      }
-      uid = pSmaIdxKey->uid;
-
-      if (!pUids) {
-        pUids = taosArrayInit(16, sizeof(tb_uid_t));
-        if (!pUids) {
-          metaCloseSmaCursor(pCur);
-          return NULL;
-        }
-      }
-
-      taosArrayPush(pUids, &uid);
-
-      continue;
-    }
-    break;
-  }
-
-  metaCloseSmaCursor(pCur);
-
-  return pUids;
-#endif
-}
-
 static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
   int      tlen = 0;
   SSchema *pSchema;
@@ -1142,3 +839,5 @@ static void poolFree(void *arg, void *ptr) {
 
   tdbOsFree(pMem);
 }
+
+#endif

source/dnode/vnode/src/meta/metaTable.c

@@ -13,31 +13,122 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
-int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
+static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
-  // Validate the tbOptions
+static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
-  // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
+static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
-  //   // TODO: handle error
+static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
-  //   return -1;
+static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME);
-  // }
+static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
+static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME);
 
-  // TODO: add atomicity
+int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
+  SMetaEntry  me = {0};
+  int         kLen = 0;
+  int         vLen = 0;
+  const void *pKey = NULL;
+  const void *pVal = NULL;
+  void       *pBuf = NULL;
+  int32_t     szBuf = 0;
+  void       *p = NULL;
+  SCoder      coder = {0};
+  SMetaReader mr = {0};
 
-  if (metaSaveTableToDB(pMeta, pTbCfg, pHandle) < 0) {
+  // validate req
-    // TODO: handle error
+  metaReaderInit(&mr, pMeta, 0);
+  if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
+// TODO: just for pass case
+#if 0
+    terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+    metaReaderClear(&mr);
     return -1;
+#else
+    metaReaderClear(&mr);
+    return 0;
+#endif
   }
+  metaReaderClear(&mr);
 
-  if (metaSaveTableToIdx(pMeta, pTbCfg) < 0) {
+  // set structs
-    // TODO: handle error
+  me.version = version;
+  me.type = TSDB_SUPER_TABLE;
+  me.uid = pReq->suid;
+  me.name = pReq->name;
+  me.stbEntry.schema = pReq->schema;
+  me.stbEntry.schemaTag = pReq->schemaTag;
+
+  if (metaHandleEntry(pMeta, &me) < 0) goto _err;
+
+  metaDebug("vgId: %d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
+
+  return 0;
+
+_err:
+  metaError("vgId: %d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name,
+            pReq->suid, tstrerror(terrno));
   return -1;
-  }
+}
 
+int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
+  // TODO
   return 0;
 }
 
+int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
+  SMetaEntry  me = {0};
+  SMetaReader mr = {0};
+
+  // validate message
+  if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _err;
+  }
+
+  // preprocess req
+  pReq->uid = tGenIdPI64();
+  pReq->ctime = taosGetTimestampSec();
+
+  // validate req
+  metaReaderInit(&mr, pMeta, 0);
+  if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
+    terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
+    metaReaderClear(&mr);
+    return -1;
+  }
+  metaReaderClear(&mr);
+
+  // build SMetaEntry
+  me.version = version;
+  me.type = pReq->type;
+  me.uid = pReq->uid;
+  me.name = pReq->name;
+  if (me.type == TSDB_CHILD_TABLE) {
+    me.ctbEntry.ctime = pReq->ctime;
+    me.ctbEntry.ttlDays = pReq->ttl;
+    me.ctbEntry.suid = pReq->ctb.suid;
+    me.ctbEntry.pTags = pReq->ctb.pTag;
+  } else {
+    me.ntbEntry.ctime = pReq->ctime;
+    me.ntbEntry.ttlDays = pReq->ttl;
+    me.ntbEntry.schema = pReq->ntb.schema;
+  }
+
+  if (metaHandleEntry(pMeta, &me) < 0) goto _err;
+
+  metaDebug("vgId:%d table %s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
+            pReq->type);
+  return 0;
+
+_err:
+  metaError("vgId:%d failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name,
+            pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno));
+  return -1;
+}
+
 int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
+#if 0
   if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
     // TODO: handle error
     return -1;
@@ -47,7 +138,164 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
     // TODO
     return -1;
   }
+#endif
 
   return 0;
 }
 
+static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
+  STbDbKey tbDbKey;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
+  int      kLen = 0;
+  int      vLen = 0;
+  SCoder   coder = {0};
+
+  // set key and value
+  tbDbKey.version = pME->version;
+  tbDbKey.uid = pME->uid;
+
+  pKey = &tbDbKey;
+  kLen = sizeof(tbDbKey);
+
+  if (tEncodeSize(metaEncodeEntry, pME, vLen) < 0) {
+    goto _err;
+  }
+
+  pVal = taosMemoryMalloc(vLen);
+  if (pVal == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
+
+  if (metaEncodeEntry(&coder, pME) < 0) {
+    goto _err;
+  }
+
+  tCoderClear(&coder);
+
+  // write to table.db
+  if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+    goto _err;
+  }
+
+  taosMemoryFree(pVal);
+  return 0;
+
+_err:
+  taosMemoryFree(pVal);
+  return -1;
+}
+
+static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
+}
+
+static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+}
+
+static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  int32_t    ttlDays;
+  int64_t    ctime;
+  STtlIdxKey ttlKey;
+
+  if (pME->type == TSDB_CHILD_TABLE) {
+    ctime = pME->ctbEntry.ctime;
+    ttlDays = pME->ctbEntry.ttlDays;
+  } else if (pME->type == TSDB_NORMAL_TABLE) {
+    ctime = pME->ntbEntry.ctime;
+    ttlDays = pME->ntbEntry.ttlDays;
+  } else {
+    ASSERT(0);
+  }
+
+  if (ttlDays <= 0) return 0;
+
+  ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
+  ttlKey.uid = pME->uid;
+
+  return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
+}
+
+static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
+  return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
+}
+
+static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  // TODO
+  return 0;
+}
+
+static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
+  SCoder                coder = {0};
+  void                 *pVal = NULL;
+  int                   vLen = 0;
+  int                   rcode = 0;
+  SSkmDbKey             skmDbKey = {0};
+  const SSchemaWrapper *pSW;
+
+  if (pME->type == TSDB_SUPER_TABLE) {
+    pSW = &pME->stbEntry.schema;
+  } else if (pME->type == TSDB_NORMAL_TABLE) {
+    pSW = &pME->ntbEntry.schema;
+  } else {
+    ASSERT(0);
+  }
+
+  skmDbKey.uid = pME->uid;
+  skmDbKey.sver = pSW->sver;
+
+  // encode schema
+  if (tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen) < 0) return -1;
+  pVal = taosMemoryMalloc(vLen);
+  if (pVal == NULL) {
+    rcode = -1;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
+  tEncodeSSchemaWrapper(&coder, pSW);
+
+  if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
+    rcode = -1;
+    goto _exit;
+  }
+
+_exit:
+  taosMemoryFree(pVal);
+  tCoderClear(&coder);
+  return rcode;
+}
+
+static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+  // save to table.db
+  if (metaSaveToTbDb(pMeta, pME) < 0) return -1;
+
+  // update uid.idx
+  if (metaUpdateUidIdx(pMeta, pME) < 0) return -1;
+
+  // update name.idx
+  if (metaUpdateNameIdx(pMeta, pME) < 0) return -1;
+
+  if (pME->type == TSDB_CHILD_TABLE) {
+    // update ctb.idx
+    if (metaUpdateCtbIdx(pMeta, pME) < 0) return -1;
+
+    // update tag.idx
+    if (metaUpdateTagIdx(pMeta, pME) < 0) return -1;
+  } else {
+    // update schema.db
+    if (metaSaveToSkmDb(pMeta, pME) < 0) return -1;
+  }
+
+  if (pME->type != TSDB_SUPER_TABLE) {
+    if (metaUpdateTtlIdx(pMeta, pME) < 0) return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/src/tq/tq.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 int32_t tqInit() {
   //
@@ -176,7 +176,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
     atomic_store_ptr(&pExec->pushHandle.handle, NULL);
     taosWUnLockLatch(&pExec->pushHandle.lock);
 
-    vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
             TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
             rsp.reqOffset, rsp.rspOffset);
 
@@ -390,7 +390,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
           TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey));
@@ -418,7 +418,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
     if (consumerEpoch > reqEpoch) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
               consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
@@ -426,7 +426,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     taosThreadMutexLock(&pExec->pWalReader->mutex);
 
     if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
               TD_VID(pTq->pVnode), fetchOffset);
       taosThreadMutexUnlock(&pExec->pWalReader->mutex);
       break;
@@ -448,7 +448,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
 
 #if 0
@@ -476,7 +476,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 #endif
 
-    vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
             TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
 
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -591,7 +591,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
 
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+  tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
           TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
 
   // TODO destroy
@@ -618,7 +618,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   SMqPollRspV2 rspV2 = {0};
@@ -660,7 +660,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     return 0;
   }
 
-  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
+  tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
          TD_VID(pTq->pVnode));
 
   rspV2.reqOffset = pReq->currentOffset;
@@ -671,7 +671,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     // TODO
     consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
@@ -680,11 +680,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
-    vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
@@ -709,7 +709,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       }
 
       if (taosArrayGetSize(pRes) == 0) {
-        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
+        tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
                pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
         fetchOffset++;
         rspV2.skipLogNum++;
@@ -770,7 +770,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = msgLen;
       pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
+      tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
              pHead->msgType, consumerId, pReq->epoch);
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
@@ -804,7 +804,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
+  tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
          pReq->epoch);
   /*}*/
 

source/dnode/vnode/src/tq/tqCommit.c

@@ -13,4 +13,4 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "vnodeInt.h"
+#include "tq.h"
 // #include <fcntl.h>
 // #include <string.h>
 // #include <unistd.h>
@@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   }
   strcpy(pMeta->dirPath, path);
 
-  char *name = taosMemoryMalloc(pathLen + 10) ;
+  char* name = taosMemoryMalloc(pathLen + 10);
 
   strcpy(name, path);
   if (!taosDirExist(name) && taosMkDir(name) != 0) {

source/dnode/vnode/src/tq/tqOffset.c

@@ -14,7 +14,7 @@
  */
 #define _DEFAULT_SOURCE
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 enum ETqOffsetPersist {
   TQ_OFFSET_PERSIST__LAZY = 1,

source/dnode/vnode/src/tq/tqRead.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
   STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle));

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 #define TSDB_MAX_SUBBLOCKS 8
 
@@ -142,12 +142,13 @@ int tsdbPrepareCommit(STsdb *pTsdb) {
 }
 
 int tsdbCommit(STsdb *pRepo) {
-  STsdbMemTable *pMem = pRepo->imem;
   SCommitH   commith = {0};
   SDFileSet *pSet = NULL;
   int        fid;
 
-  if (pRepo->imem == NULL) return 0;
+  // if (pRepo->imem == NULL) return 0;
+  pRepo->imem = pRepo->mem;
+  pRepo->mem = NULL;
 
   tsdbStartCommit(pRepo);
   // Resource initialization
@@ -239,7 +240,7 @@ static void tsdbStartCommit(STsdb *pRepo) {
 
 static void tsdbEndCommit(STsdb *pTsdb, int eno) {
   tsdbEndFSTxn(pTsdb);
-  tsdbFreeMemTable(pTsdb, pTsdb->imem);
+  tsdbMemTableDestroy(pTsdb, pTsdb->imem);
   pTsdb->imem = NULL;
   tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
 }

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+
+int tsdbBegin(STsdb *pTsdb) {
+  STsdbMemTable *pMem;
+
+  if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/vnode/src/tsdb/tsdbCompact.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #if 0
-#include "tsdbint.h"
+#include "tsdb.h"
 
 typedef struct {
   STable *    pTable;
@@ -531,4 +531,3 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
 }
 
 #endif
-

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
 static const char *tsdbTxnFname[] = {"current.t", "current"};

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 static const char *TSDB_FNAME_SUFFIX[] = {
     "head",  // TSDB_FILE_HEAD

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -13,96 +13,62 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
-static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF);
+int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) {
-static void   tsdbFree(STsdb *pTsdb);
-static int    tsdbOpenImpl(STsdb *pTsdb);
-static void   tsdbCloseImpl(STsdb *pTsdb);
-
-STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) {
   STsdb *pTsdb = NULL;
+  int    slen = 0;
 
-  // Set default TSDB Options
+  *ppTsdb = NULL;
-  // if (pTsdbCfg == NULL) {
+  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3;
-  pTsdbCfg = &defautlTsdbOptions;
-  // }
 
-  // Validate the options
+  // create handle
-  if (tsdbValidateOptions(pTsdbCfg) < 0) {
+  pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
-    // TODO: handle error
-    return NULL;
-  }
-
-  // Create the handle
-  pTsdb = tsdbNew(path, pVnode, pTsdbCfg, pMAF);
   if (pTsdb == NULL) {
-    // TODO: handle error
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
+    return -1;
   }
 
-  taosMkDir(path);
+  pTsdb->path = (char *)&pTsdb[1];
-
+  sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
-  // Open the TSDB
+          VNODE_TSDB_DIR);
-  if (tsdbOpenImpl(pTsdb) < 0) {
-    // TODO: handle error
-    return NULL;
-  }
-
-  return pTsdb;
-}
-
-void tsdbClose(STsdb *pTsdb) {
-  if (pTsdb) {
-    tsdbCloseImpl(pTsdb);
-    tsdbFree(pTsdb);
-  }
-}
-
-/* ------------------------ STATIC METHODS ------------------------ */
-static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) {
-  STsdb *pTsdb = NULL;
-
-  pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb));
-  if (pTsdb == NULL) {
-    // TODO: handle error
-    return NULL;
-  }
-
-  pTsdb->path = strdup(path);
-  pTsdb->vgId = TD_VID(pVnode);
   pTsdb->pVnode = pVnode;
-  tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg);
+  pTsdb->vgId = TD_VID(pVnode);
-  pTsdb->pmaf = pMAF;
+  pTsdb->repoLocked = false;
-  pTsdb->fs = tsdbNewFS(pTsdbCfg);
+  tdbMutexInit(&pTsdb->mutex, NULL);
+  pTsdb->config = pVnode->config.tsdbCfg;
+  pTsdb->fs = tsdbNewFS(&pTsdb->config);
 
-  return pTsdb;
+  // create dir (TODO: use tfsMkdir)
+  taosMkDir(pTsdb->path);
+
+  // open tsdb
+  if (tsdbOpenFS(pTsdb) < 0) {
+    goto _err;
+  }
+
+  tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode));
+
+  *ppTsdb = pTsdb;
+  return 0;
+
+_err:
+  taosMemoryFree(pTsdb);
+  return -1;
 }
 
-static void tsdbFree(STsdb *pTsdb) {
+int tsdbClose(STsdb *pTsdb) {
   if (pTsdb) {
-    tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
+    tsdbCloseFS(pTsdb);
-    tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
+    // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
+    // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
     tsdbFreeFS(pTsdb->fs);
-    taosMemoryFreeClear(pTsdb->path);
+    // taosMemoryFreeClear(pTsdb->path);
     taosMemoryFree(pTsdb);
   }
-}
-
-static int tsdbOpenImpl(STsdb *pTsdb) {
-  tsdbOpenFS(pTsdb);
-
-  // tsdbInitSma(pTsdb);
-  // TODO
-
   return 0;
 }
 
-static void tsdbCloseImpl(STsdb *pTsdb) {
-  tsdbCloseFS(pTsdb);
-  // TODO
-}
-
 int tsdbLockRepo(STsdb *pTsdb) {
   int code = taosThreadMutexLock(&pTsdb->mutex);
   if (code != 0) {

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -13,10 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
-static int      tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
-static int      tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows);
 static STbData *tsdbNewTbData(tb_uid_t uid);
 static void     tsdbFreeTbData(STbData *pTbData);
 static char    *tsdbGetTsTupleKey(const void *data);
@@ -24,86 +22,51 @@ static int      tsdbTbDataComp(const void *arg1, const void *arg2);
 static char    *tsdbTbDataGetUid(const void *arg);
 static int      tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
 
-STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) {
+int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
-  STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
+  STsdbMemTable *pMemTable;
+  SVnode        *pVnode;
+
+  *ppMemTable = NULL;
+  pVnode = pTsdb->pVnode;
+
+  // alloc handle
+  pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
   if (pMemTable == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
-    return NULL;
   }
 
+  pMemTable->pPool = pTsdb->pVnode->inUse;
   T_REF_INIT_VAL(pMemTable, 1);
-  taosInitRWLatch(&(pMemTable->latch));
+  taosInitRWLatch(&pMemTable->latch);
-  pMemTable->keyMax = TSKEY_MIN;
   pMemTable->keyMin = TSKEY_MAX;
+  pMemTable->keyMax = TSKEY_MIN;
   pMemTable->nRow = 0;
-  pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf);
+  pMemTable->pSlIdx = tSkipListCreate(pVnode->config.tsdbCfg.slLevel, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t),
-  if (pMemTable->pMA == NULL) {
+                                      tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
-    taosMemoryFree(pMemTable);
-    return NULL;
-  }
-
-  // Initialize the container
-  pMemTable->pSlIdx =
-      tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
   if (pMemTable->pSlIdx == NULL) {
-    pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
     taosMemoryFree(pMemTable);
-    return NULL;
+    return -1;
   }
 
   pMemTable->pHashIdx = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   if (pMemTable->pHashIdx == NULL) {
-    pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
     tSkipListDestroy(pMemTable->pSlIdx);
     taosMemoryFree(pMemTable);
-    return NULL;
+    return -1;
   }
 
-  return pMemTable;
+  *ppMemTable = pMemTable;
+  return 0;
 }
 
-void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) {
+void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) {
   if (pMemTable) {
     taosHashCleanup(pMemTable->pHashIdx);
     tSkipListDestroy(pMemTable->pSlIdx);
-    if (pMemTable->pMA) {
-      pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
-    }
     taosMemoryFree(pMemTable);
   }
 }
 
-int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
-  SSubmitBlk    *pBlock = NULL;
-  SSubmitMsgIter msgIter = {0};
-  int32_t        affectedrows = 0, numOfRows = 0;
-
-  if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
-    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
-      tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
-    }
-    return -1;
-  }
-
-  tInitSubmitMsgIter(pMsg, &msgIter);
-  while (true) {
-    tGetSubmitMsgNext(&msgIter, &pBlock);
-    if (pBlock == NULL) break;
-    if (tsdbMemTableInsertTbData(pTsdb, pBlock, &affectedrows) < 0) {
-      return -1;
-    }
-
-    numOfRows += pBlock->numOfRows;
-  }
-
-  if (pRsp != NULL) {
-    pRsp->affectedRows = affectedrows;
-    pRsp->numOfRows = numOfRows;
-  }
-
-  return 0;
-}
-
 /**
  * This is an important function to load data or try to load data from memory skiplist iterator.
  *
@@ -255,78 +218,7 @@ int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) {
   return 0;
 }
 
-static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
+int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) {
-  ASSERT(pMsg != NULL);
-  // STsdbMeta *    pMeta = pTsdb->tsdbMeta;
-  SSubmitMsgIter msgIter = {0};
-  SSubmitBlk    *pBlock = NULL;
-  SSubmitBlkIter blkIter = {0};
-  STSRow        *row = NULL;
-  TSKEY          now = taosGetTimestamp(pTsdb->config.precision);
-  TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2;
-  TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days;
-
-  terrno = TSDB_CODE_SUCCESS;
-  pMsg->length = htonl(pMsg->length);
-  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
-
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
-  while (true) {
-    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
-    if (pBlock == NULL) break;
-
-    pBlock->uid = htobe64(pBlock->uid);
-    pBlock->suid = htobe64(pBlock->suid);
-    pBlock->sversion = htonl(pBlock->sversion);
-    pBlock->dataLen = htonl(pBlock->dataLen);
-    pBlock->schemaLen = htonl(pBlock->schemaLen);
-    pBlock->numOfRows = htons(pBlock->numOfRows);
-
-#if 0
-    if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
-      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
-                pBlock->tid);
-      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
-      return -1;
-    }
-
-    STable *pTable = pMeta->tables[pBlock->tid];
-    if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
-      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
-                pBlock->tid);
-      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
-      return -1;
-    }
-
-    if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
-      tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable));
-      terrno = TSDB_CODE_TDB_INVALID_ACTION;
-      return -1;
-    }
-
-    // Check schema version and update schema if needed
-    if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) {
-      if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
-        continue;
-      } else {
-        return -1;
-      }
-    }
-
-    tsdbInitSubmitBlkIter(pBlock, &blkIter);
-    while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
-      if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) {
-        return -1;
-      }
-    }
-#endif
-  }
-
-  if (terrno != TSDB_CODE_SUCCESS) return -1;
-  return 0;
-}
-
-static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) {
   // STsdbMeta       *pMeta = pRepo->tsdbMeta;
   // int32_t          points = 0;
   // STable          *pTable = NULL;
@@ -337,11 +229,9 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
   STSRow        *row;
   TSKEY          keyMin;
   TSKEY          keyMax;
+  SSubmitBlk    *pBlkCopy;
 
-  // SMemTable       *pMemTable = NULL;
+  // create container is nedd
-  // STableData      *pTableData = NULL;
-  // STsdbCfg        *pCfg = &(pRepo->config);
-
   tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid));
   if (tptr == NULL) {
     pTbData = tsdbNewTbData(pBlock->uid);
@@ -358,7 +248,11 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
     pTbData = *(STbData **)tptr;
   }
 
-  tInitSubmitBlkIter(pBlock, &blkIter);
+  // copy data to buffer pool
+  pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock));
+  memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock));
+
+  tInitSubmitBlkIter(pBlkCopy, &blkIter);
   if (blkIter.row == NULL) return 0;
   keyMin = TD_ROW_KEY(blkIter.row);
 
@@ -377,31 +271,6 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
 
   (*pAffectedRows) += pBlock->numOfRows;
 
-  // STSRow* lastRow = NULL;
-  // int64_t osize = SL_SIZE(pTableData->pData);
-  // tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
-  // tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
-  // int64_t dsize = SL_SIZE(pTableData->pData) - osize;
-  // (*pAffectedRows) += points;
-
-  // if(lastRow != NULL) {
-  //   TSKEY lastRowKey = TD_ROW_KEY(lastRow);
-  //   if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey;
-  //   pMemTable->numOfRows += dsize;
-
-  //   if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey;
-  //   pTableData->numOfRows += dsize;
-  //   if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey;
-  //   if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey;
-  //   if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) {
-  //     return -1;
-  //   }
-  // }
-
-  // STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion, -1);
-  // pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
-  // pRepo->stat.totalStorage += points * schemaVLen(pSchema);
-
   return 0;
 }
 
@@ -526,7 +395,6 @@ static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
 static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row);
 static int          tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
 static STSRow*      tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
-static int          tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitReq *pMsg);
 static int          tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
 static int          tsdbInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter);
 static int          tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);

source/dnode/vnode/src/tsdb/tsdbOptions.c

@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeInt.h"
-
-const STsdbCfg defautlTsdbOptions = {.precision = 0,
-                                     .lruCacheSize = 0,
-                                     .days = 10,
-                                     .minRows = 100,
-                                     .maxRows = 4096,
-                                     .keep2 = 3650,
-                                     .keep0 = 3650,
-                                     .keep1 = 3650,
-                                     .update = 0,
-                                     .compression = TWO_STAGE_COMP};
-
-int tsdbValidateOptions(const STsdbCfg *pTsdbOptions) {
-  // TODO
-  return 0;
-}
-
-void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc) { memcpy(pDest, pSrc, sizeof(STsdbCfg)); }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 #define EXTRA_BYTES                2
 #define ASCENDING_TRAVERSE(o)      (o == TSDB_ORDER_ASC)
@@ -254,7 +254,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
 
         assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey);
       } else {
-        assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey);
+        info.lastKey = pTsdbReadHandle->window.skey;
       }
 
       taosArrayPush(pTableCheckInfo, &info);
@@ -317,7 +317,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
   return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
 }
 
-static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) {
+static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) {
   pTsdbReadHandle->window = pCond->twindow;
 
   bool    updateTs = false;
@@ -343,7 +343,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond*
   }
 }
 
-static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) {
+static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) {
   STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle));
   if (pReadHandle == NULL) {
     goto _end;
@@ -422,7 +422,7 @@ _end:
   return NULL;
 }
 
-tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT* tsdbQueryTables(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                              uint64_t taskId) {
   STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId);
   if (pTsdbReadHandle == NULL) {
@@ -448,7 +448,7 @@ tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo
   return (tsdbReaderT)pTsdbReadHandle;
 }
 
-void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) {
+void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) {
   STsdbReadHandle* pTsdbReadHandle = queryHandle;
 
   if (emptyQueryTimewindow(pTsdbReadHandle)) {
@@ -485,7 +485,7 @@ void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) {
   resetCheckInfo(pTsdbReadHandle);
 }
 
-void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
+void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, STableGroupInfo* groupList) {
   STsdbReadHandle* pTsdbReadHandle = queryHandle;
 
   pTsdbReadHandle->order = pCond->order;
@@ -526,7 +526,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pC
   //  pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
 }
 
-tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                              uint64_t taskId) {
   pCond->twindow = updateLastrowForEachGroup(groupList);
 
@@ -555,7 +555,7 @@ tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo
 }
 
 #if 0
-tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
+tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
   STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef);
   if (pTsdbReadHandle == NULL) {
     return NULL;
@@ -618,8 +618,8 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
   return pNew;
 }
 
-tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList,
-                                          uint64_t taskId) {
+                                          uint64_t qId, uint64_t taskId) {
   STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
 
   if (pNew->numOfTables == 0) {
@@ -1185,10 +1185,12 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     tsdbDebug("%p no data in mem, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
   }
 
-  if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
+  bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
-      (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
+
-    if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
+  if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
-        (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
+      (!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
+    if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
+        (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
 
@@ -1225,8 +1227,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     assert(pTsdbReadHandle->outputCapacity >= binfo.rows);
     int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo);
 
-    if ((cur->pos == 0 && endPos == binfo.rows - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
+    if ((cur->pos == 0 && endPos == binfo.rows - 1 && ascScan) ||
-        (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) {
+        (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ascScan))) {
       pTsdbReadHandle->realNumOfRows = binfo.rows;
 
       cur->rows = binfo.rows;
@@ -1234,7 +1236,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
       cur->mixBlock = false;
       cur->blockCompleted = true;
 
-      if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
+      if (ascScan) {
         cur->lastKey = binfo.window.ekey + 1;
         cur->pos = binfo.rows;
       } else {
@@ -1382,8 +1384,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 
 static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
                                        int32_t start, int32_t end) {
-  int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
-
   SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
   TSKEY*     tsArray = pCols->cols[0].pData;
 
@@ -1394,6 +1394,11 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
     return numOfRows;
   }
 
+  bool    ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+  int32_t trueStart = ascScan ? start : end;
+  int32_t trueEnd = ascScan ? end : start;
+  int32_t step = ascScan ? 1 : -1;
+
   int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
 
   // data in buffer has greater timestamp, copy data in file block
@@ -1411,7 +1416,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
       if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) {  // todo opt performance
         //        memmove(pData, (char*)src->pData + bytes * start, bytes * num);
         int32_t rowIndex = numOfRows;
-        for (int32_t k = start; k <= end; ++k, ++rowIndex) {
+        for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) {
           SCellVal sVal = {0};
           if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) {
             TASSERT(0);
@@ -1427,7 +1432,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
         int32_t rowIndex = numOfRows;
 
         // todo refactor, only copy one-by-one
-        for (int32_t k = start; k < num + start; ++k, ++rowIndex) {
+        for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) {
           SCellVal sVal = {0};
           if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) {
             TASSERT(0);
@@ -1444,26 +1449,19 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
       j++;
       i++;
     } else {  // pColInfo->info.colId < src->colId, it is a NULL data
-      int32_t rowIndex = numOfRows;
+      colDataAppendNNULL(pColInfo, numOfRows, num);
-      for (int32_t k = start; k < num + start; ++k, ++rowIndex) {  // TODO opt performance
-        colDataAppend(pColInfo, rowIndex, NULL, true);
-      }
       i++;
     }
   }
 
   while (i < requiredNumOfCols) {  // the remain columns are all null data
     SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-    int32_t rowIndex = numOfRows;
+    colDataAppendNNULL(pColInfo, numOfRows, num);
-
-    for (int32_t k = start; k < num + start; ++k, ++rowIndex) {
-      colDataAppend(pColInfo, rowIndex, NULL, true);  // TODO add a fast version to set a number of consecutive NULL value.
-    }
     i++;
   }
 
-  pTsdbReadHandle->cur.win.ekey = tsArray[end];
+  pTsdbReadHandle->cur.win.ekey = tsArray[trueEnd];
-  pTsdbReadHandle->cur.lastKey = tsArray[end] + step;
+  pTsdbReadHandle->cur.lastKey = tsArray[trueEnd] + step;
 
   return numOfRows + num;
 }
@@ -1777,7 +1775,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
   STable* pTable = NULL;
   int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
 
-  tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 " rows:%d, start:%d, end:%d, %s",
+  tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64
+            " rows:%d, start:%d, end:%d, %s",
             pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
             cur->pos, endPos, pTsdbReadHandle->idStr);
 
@@ -2966,7 +2965,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
 //  }
 //
 //  // load the previous row
-//  STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
+//  SQueryTableDataCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
 //  if (type == TSDB_PREV_ROW) {
 //    cond.order = TSDB_ORDER_DESC;
 //    cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN};
@@ -3330,21 +3329,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
           return NULL;
         }
 
-        // todo refactor
         int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-
-        // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-        if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
-          int32_t emptySize = pHandle->outputCapacity - numOfRows;
-          int32_t reqNumOfCols = (int32_t)taosArrayGetSize(pHandle->pColumns);
-
-          for (int32_t i = 0; i < reqNumOfCols; ++i) {
-            SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
-            memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes,
-                    numOfRows * pColInfo->info.bytes);
-          }
-        }
-
         return pHandle->pColumns;
       }
     }
@@ -3631,8 +3616,11 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd
 int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
                                  int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
                                  SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) {
-  STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
+  SMetaReader mr = {0};
-  if (pTbCfg == NULL) {
+
+  metaReaderInit(&mr, (SMeta*)pMeta, 0);
+
+  if (metaGetTableEntryByUid(&mr, uid) < 0) {
     tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     goto _error;
@@ -3640,13 +3628,15 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
     tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
   }
 
-  if (pTbCfg->type != META_SUPER_TABLE) {
+  if (mr.me.type != TSDB_SUPER_TABLE) {
     tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId,
               reqId);
     terrno = TSDB_CODE_OPS_NOT_SUPPORT;  // basically, this error is caused by invalid sql issued by client
     goto _error;
   }
 
+  metaReaderClear(&mr);
+
   // NOTE: not add ref count for super table
   SArray*         res = taosArrayInit(8, sizeof(STableKeyInfo));
   SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true);
@@ -3697,12 +3687,17 @@ int32_t tsdbQueryTableList(void* pMeta, SArray* pRes, void* filterInfo) {
   return TSDB_CODE_SUCCESS;
 }
 int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
-  STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
+  SMetaReader mr = {0};
-  if (pTbCfg == NULL) {
+
+  metaReaderInit(&mr, (SMeta*)pMeta, 0);
+
+  if (metaGetTableEntryByUid(&mr, uid) < 0) {
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     goto _error;
   }
 
+  metaReaderClear(&mr);
+
   pGroupInfo->numOfTables = 1;
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
 
@@ -3715,6 +3710,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr
   return TSDB_CODE_SUCCESS;
 
 _error:
+  metaReaderClear(&mr);
   return terrno;
 }
 

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 #define TSDB_KEY_COL_OFFSET 0
 

source/dnode/vnode/src/tsdb/tsdbScan.c

@@ -13,9 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #if 0
-#include "tsdbint.h"
+#include "tsdb.h"
 #ifndef _TSDB_PLUGINS
 
 int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -13,7 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdbSma.h"
+#include "tsdb.h"
 
 static const char *TSDB_SMA_DNAME[] = {
     "",      // TSDB_SMA_TYPE_BLOCK
@@ -678,7 +679,6 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
     return TSDB_CODE_FAILED;
   }
 
-
   if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_TDB_INIT_FAILED;
     return TSDB_CODE_FAILED;
@@ -1693,6 +1693,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
  * @return int32_t
  */
 int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
+#if 0
   SRSmaParam *param = pReq->stbCfg.pRSmaParam;
 
   if (!param) {
@@ -1764,6 +1765,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
    tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->stbCfg.suid);
   }
 
+#endif
   return TSDB_CODE_SUCCESS;
 }
 

source/dnode/vnode/src/tsdb/tsdbTDBImpl.c

@@ -15,7 +15,7 @@
 
 #define ALLOW_FORBID_FUNC
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) {
   int ret = 0;

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -13,23 +13,113 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
-/**
+static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
- * @brief insert TS data
+
- *
+int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
- * @param pTsdb
+  SSubmitMsgIter msgIter = {0};
- * @param pMsg
+  SSubmitBlk    *pBlock = NULL;
- * @param pRsp
+  int32_t        affectedrows = 0;
- * @return int
+  int32_t        numOfRows = 0;
- */
+
-int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
+  ASSERT(pTsdb->mem != NULL);
-  // Check if mem is there. If not, create one.
+
-  if (pTsdb->mem == NULL) {
+  // scan and convert
-    pTsdb->mem = tsdbNewMemTable(pTsdb);
+  if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
-    if (pTsdb->mem == NULL) {
+    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+      tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    }
+    return -1;
+  }
+
+  // loop to insert
+  tInitSubmitMsgIter(pMsg, &msgIter);
+  while (true) {
+    tGetSubmitMsgNext(&msgIter, &pBlock);
+    if (pBlock == NULL) break;
+    if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) {
+      return -1;
+    }
+
+    numOfRows += pBlock->numOfRows;
+  }
+
+  if (pRsp != NULL) {
+    pRsp->affectedRows = affectedrows;
+    pRsp->numOfRows = numOfRows;
+  }
+
+  return 0;
+}
+
+static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
+  ASSERT(pMsg != NULL);
+  // STsdbMeta *    pMeta = pTsdb->tsdbMeta;
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlk    *pBlock = NULL;
+  SSubmitBlkIter blkIter = {0};
+  STSRow        *row = NULL;
+  TSKEY          now = taosGetTimestamp(pTsdb->config.precision);
+  TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2;
+  TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days;
+
+  terrno = TSDB_CODE_SUCCESS;
+  pMsg->length = htonl(pMsg->length);
+  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+
+  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  while (true) {
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (pBlock == NULL) break;
+
+    pBlock->uid = htobe64(pBlock->uid);
+    pBlock->suid = htobe64(pBlock->suid);
+    pBlock->sversion = htonl(pBlock->sversion);
+    pBlock->dataLen = htonl(pBlock->dataLen);
+    pBlock->schemaLen = htonl(pBlock->schemaLen);
+    pBlock->numOfRows = htons(pBlock->numOfRows);
+
+#if 0
+    if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+      return -1;
+    }
+
+    STable *pTable = pMeta->tables[pBlock->tid];
+    if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+      return -1;
+    }
+
+    if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+      tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable));
+      terrno = TSDB_CODE_TDB_INVALID_ACTION;
+      return -1;
+    }
+
+    // Check schema version and update schema if needed
+    if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) {
+      if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+        continue;
+      } else {
         return -1;
       }
     }
-  return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, pRsp);
+
+    tsdbInitSubmitBlkIter(pBlock, &blkIter);
+    while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+      if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) {
+        return -1;
+      }
+    }
+#endif
+  }
+
+  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  return 0;
 }

source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c

@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeInt.h"
-
-static SVArenaNode *vArenaNodeNew(uint64_t capacity);
-static void         vArenaNodeFree(SVArenaNode *pNode);
-
-SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) {
-  SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA));
-  if (pVMA == NULL) {
-    return NULL;
-  }
-
-  pVMA->capacity = capacity;
-  pVMA->ssize = ssize;
-  pVMA->lsize = lsize;
-  TD_SLIST_INIT(&(pVMA->nlist));
-
-  pVMA->pNode = vArenaNodeNew(capacity);
-  if (pVMA->pNode == NULL) {
-    taosMemoryFree(pVMA);
-    return NULL;
-  }
-
-  TD_SLIST_PUSH(&(pVMA->nlist), pVMA->pNode);
-
-  return pVMA;
-}
-
-void vmaDestroy(SVMemAllocator *pVMA) {
-  if (pVMA) {
-    while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
-      SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
-      TD_SLIST_POP(&(pVMA->nlist));
-      vArenaNodeFree(pNode);
-    }
-
-    taosMemoryFree(pVMA);
-  }
-}
-
-void vmaReset(SVMemAllocator *pVMA) {
-  while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
-    SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
-    TD_SLIST_POP(&(pVMA->nlist));
-    vArenaNodeFree(pNode);
-  }
-
-  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
-  pNode->ptr = pNode->data;
-}
-
-void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
-  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
-  void *       ptr;
-
-  if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) {
-    uint64_t capacity = TMAX(pVMA->ssize, size);
-    pNode = vArenaNodeNew(capacity);
-    if (pNode == NULL) {
-      // TODO: handle error
-      return NULL;
-    }
-
-    TD_SLIST_PUSH(&(pVMA->nlist), pNode);
-  }
-
-  ptr = pNode->ptr;
-  pNode->ptr = POINTER_SHIFT(ptr, size);
-
-  return ptr;
-}
-
-void vmaFree(SVMemAllocator *pVMA, void *ptr) {
-  // TODO
-}
-
-bool vmaIsFull(SVMemAllocator *pVMA) {
-  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
-
-  return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) ||
-         (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize);
-}
-
-/* ------------------------ STATIC METHODS ------------------------ */
-static SVArenaNode *vArenaNodeNew(uint64_t capacity) {
-  SVArenaNode *pNode = NULL;
-
-  pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity);
-  if (pNode == NULL) {
-    return NULL;
-  }
-
-  pNode->size = capacity;
-  pNode->ptr = pNode->data;
-
-  return pNode;
-}
-
-static void vArenaNodeFree(SVArenaNode *pNode) {
-  if (pNode) {
-    taosMemoryFree(pNode);
-  }
-}

source/dnode/vnode/src/vnd/vnodeBufPool.c

@@ -13,11 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
 /* ------------------------ STRUCTURES ------------------------ */
 
-static int vnodeBufPoolCreate(int size, SVBufPool **ppPool);
+static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool);
 static int vnodeBufPoolDestroy(SVBufPool *pPool);
 
 int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
@@ -30,17 +30,17 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
     // create pool
     ret = vnodeBufPoolCreate(size, &pPool);
     if (ret < 0) {
-      vError("vgId:%d failed to open vnode buffer pool since %s", TD_VNODE_ID(pVnode), tstrerror(terrno));
+      vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
       vnodeCloseBufPool(pVnode);
       return -1;
     }
 
-    // add pool to queue
+    // add pool to vnode
     pPool->next = pVnode->pPool;
     pVnode->pPool = pPool;
   }
 
-  vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VNODE_ID(pVnode), size);
+  vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size);
 
   return 0;
 }
@@ -53,7 +53,7 @@ int vnodeCloseBufPool(SVnode *pVnode) {
     vnodeBufPoolDestroy(pPool);
   }
 
-  vDebug("vgId:%d vnode buffer pool is closed", TD_VNODE_ID(pVnode));
+  vDebug("vgId:%d vnode buffer pool is closed", TD_VID(pVnode));
 
   return 0;
 }
@@ -75,7 +75,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
   pPool->ptr = pPool->node.data;
 }
 
-void *vnodeBufPoolMalloc(SVBufPool *pPool, size_t size) {
+void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
   SVBufPoolNode *pNode;
   void          *p;
 
@@ -120,7 +120,7 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) {
 }
 
 // STATIC METHODS -------------------
-static int vnodeBufPoolCreate(int size, SVBufPool **ppPool) {
+static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) {
   SVBufPool *pPool;
 
   pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);

source/dnode/vnode/src/vnd/vnodeBufferPool.c

@@ -1,191 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeInt.h"
-
-/* ------------------------ STRUCTURES ------------------------ */
-#define VNODE_BUF_POOL_SHARDS 3
-
-struct SVBufPool {
-  TdThreadMutex mutex;
-  TdThreadCond  hasFree;
-  TD_DLIST(SVMemAllocator) free;
-  TD_DLIST(SVMemAllocator) incycle;
-  SVMemAllocator *inuse;
-  // MAF for submodules to use
-  SMemAllocatorFactory *pMAF;
-};
-
-static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF);
-static void           vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA);
-
-int vnodeOpenBufPool(SVnode *pVnode) {
-  uint64_t capacity;
-
-  if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) {
-    /* TODO */
-    return -1;
-  }
-
-  TD_DLIST_INIT(&(pVnode->pBufPool->free));
-  TD_DLIST_INIT(&(pVnode->pBufPool->incycle));
-
-  pVnode->pBufPool->inuse = NULL;
-
-  // TODO
-  capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
-
-  for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
-    SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize);
-    if (pVMA == NULL) {
-      // TODO: handle error
-      return -1;
-    }
-
-    TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA);
-  }
-
-  pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory));
-  if (pVnode->pBufPool->pMAF == NULL) {
-    // TODO: handle error
-    return -1;
-  }
-  pVnode->pBufPool->pMAF->impl = pVnode;
-  pVnode->pBufPool->pMAF->create = vBufPoolCreateMA;
-  pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA;
-
-  return 0;
-}
-
-void vnodeCloseBufPool(SVnode *pVnode) {
-  if (pVnode->pBufPool) {
-    taosMemoryFreeClear(pVnode->pBufPool->pMAF);
-    vmaDestroy(pVnode->pBufPool->inuse);
-
-    while (true) {
-      SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle));
-      if (pVMA == NULL) break;
-      TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA);
-      vmaDestroy(pVMA);
-    }
-
-    while (true) {
-      SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free));
-      if (pVMA == NULL) break;
-      TD_DLIST_POP(&(pVnode->pBufPool->free), pVMA);
-      vmaDestroy(pVMA);
-    }
-
-    taosMemoryFree(pVnode->pBufPool);
-    pVnode->pBufPool = NULL;
-  }
-}
-
-int vnodeBufPoolSwitch(SVnode *pVnode) {
-  SVMemAllocator *pvma = pVnode->pBufPool->inuse;
-
-  pVnode->pBufPool->inuse = NULL;
-
-  if (pvma) {
-    TD_DLIST_APPEND(&(pVnode->pBufPool->incycle), pvma);
-  }
-  return 0;
-}
-
-int vnodeBufPoolRecycle(SVnode *pVnode) {
-  SVBufPool *     pBufPool = pVnode->pBufPool;
-  SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle));
-  if (pvma == NULL) return 0;
-  // ASSERT(pvma != NULL);
-
-  TD_DLIST_POP(&(pBufPool->incycle), pvma);
-  vmaReset(pvma);
-  TD_DLIST_APPEND(&(pBufPool->free), pvma);
-
-  return 0;
-}
-
-void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
-  SVBufPool *pBufPool = pVnode->pBufPool;
-
-  if (pBufPool->inuse == NULL) {
-    while (true) {
-      // TODO: add sem_wait and sem_post
-      pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free));
-      if (pBufPool->inuse) {
-        TD_DLIST_POP(&(pBufPool->free), pBufPool->inuse);
-        break;
-      } else {
-        // tsem_wait(&(pBufPool->hasFree));
-      }
-    }
-  }
-
-  return vmaMalloc(pBufPool->inuse, size);
-}
-
-bool vnodeBufPoolIsFull(SVnode *pVnode) {
-  if (pVnode->pBufPool->inuse == NULL) return false;
-  return vmaIsFull(pVnode->pBufPool->inuse);
-}
-
-SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; }
-
-/* ------------------------ STATIC METHODS ------------------------ */
-typedef struct {
-  SVnode *        pVnode;
-  SVMemAllocator *pVMA;
-} SVMAWrapper;
-
-static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) {
-  SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl);
-
-  return vmaMalloc(pWrapper->pVMA, size);
-}
-
-// TODO: Add atomic operations here
-static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) {
-  SMemAllocator *pMA;
-  SVnode *       pVnode = (SVnode *)(pMAF->impl);
-  SVMAWrapper *  pWrapper;
-
-  pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper));
-  if (pMA == NULL) {
-    return NULL;
-  }
-
-  pVnode->pBufPool->inuse->_ref.val++;
-  pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA));
-  pWrapper->pVnode = pVnode;
-  pWrapper->pVMA = pVnode->pBufPool->inuse;
-
-  pMA->impl = pWrapper;
-  TD_MA_MALLOC_FUNC(pMA) = vmaMaloocCb;
-
-  return pMA;
-}
-
-static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) {
-  SVMAWrapper *   pWrapper = (SVMAWrapper *)(pMA->impl);
-  SVnode *        pVnode = pWrapper->pVnode;
-  SVMemAllocator *pVMA = pWrapper->pVMA;
-
-  taosMemoryFree(pMA);
-  if (--pVMA->_ref.val == 0) {
-    TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA);
-    vmaReset(pVMA);
-    TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA);
-  }
-}

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -13,23 +13,24 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
 const SVnodeCfg vnodeCfgDefault = {
     .vgId = -1,
     .dbname = "",
     .dbId = 0,
-    .wsize = 96 * 1024 * 1024,
+    .szPage = 4096,
-    .ssize = 1 * 1024 * 1024,
+    .szCache = 256,
-    .lsize = 1024,
+    .szBuf = 96 * 1024 * 1024,
-    .isHeapAllocator = false,
+    .isHeap = false,
     .ttl = 0,
     .keep = 0,
     .streamMode = 0,
     .isWeak = 0,
-    .tsdbCfg = {.precision = TWO_STAGE_COMP,
+    .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
                 .update = 0,
                 .compression = 2,
+                .slLevel = 5,
                 .days = 10,
                 .minRows = 100,
                 .maxRows = 4096,
@@ -53,10 +54,10 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
   if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
@@ -64,6 +65,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
@@ -103,10 +105,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1;
   if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "szPage", pCfg->szPage) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
@@ -114,6 +116,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
@@ -150,10 +153,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   return 0;
 }
 
-int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
+int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) {
   uint32_t hashValue = 0;
 
-  switch (pVnodeOptions->hashMethod) {
+  switch (pVnode->config.hashMethod) {
     default:
       hashValue = MurmurHash3_32(tableFName, strlen(tableFName));
       break;
@@ -167,5 +170,5 @@ int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
   }
 #endif
 
-  return TSDB_CODE_SUCCESS;
+  return 0;
 }

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
 #define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
@@ -22,9 +22,41 @@ static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
 static int  vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
 static int  vnodeStartCommit(SVnode *pVnode);
 static int  vnodeEndCommit(SVnode *pVnode);
-static int  vnodeCommit(void *arg);
+static int  vnodeCommitImpl(void *arg);
 static void vnodeWaitCommit(SVnode *pVnode);
 
+int vnodeBegin(SVnode *pVnode) {
+  // alloc buffer pool
+  /* pthread_mutex_lock(); */
+
+  while (pVnode->pPool == NULL) {
+    /* pthread_cond_wait(); */
+  }
+
+  pVnode->inUse = pVnode->pPool;
+  pVnode->pPool = pVnode->inUse->next;
+  pVnode->inUse->next = NULL;
+  /* ref pVnode->inUse buffer pool */
+
+  /* pthread_mutex_unlock(); */
+
+  // begin meta
+  if (metaBegin(pVnode->pMeta) < 0) {
+    vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno));
+    return -1;
+  }
+
+  // begin tsdb
+  if (tsdbBegin(pVnode->pTsdb) < 0) {
+    vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; }
+
 int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   char      fname[TSDB_FILENAME_LEN];
   TdFilePtr pFile;
@@ -142,10 +174,10 @@ _err:
 int vnodeAsyncCommit(SVnode *pVnode) {
   vnodeWaitCommit(pVnode);
 
-  vnodeBufPoolSwitch(pVnode);
+  // vnodeBufPoolSwitch(pVnode);
-  tsdbPrepareCommit(pVnode->pTsdb);
+  // tsdbPrepareCommit(pVnode->pTsdb);
 
-  vnodeScheduleTask(vnodeCommit, pVnode);
+  vnodeScheduleTask(vnodeCommitImpl, pVnode);
 
   return 0;
 }
@@ -157,25 +189,64 @@ int vnodeSyncCommit(SVnode *pVnode) {
   return 0;
 }
 
-static int vnodeCommit(void *arg) {
+int vnodeCommit(SVnode *pVnode) {
-  SVnode    *pVnode = (SVnode *)arg;
+  SVnodeInfo info;
   char       dir[TSDB_FILENAME_LEN];
-  SVnodeInfo info = {0};
 
-  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
+  vInfo("vgId:%d start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied);
+
+  pVnode->onCommit = pVnode->inUse;
+  pVnode->inUse = NULL;
+
+  // save info
   info.config = pVnode->config;
   info.state.committed = pVnode->state.applied;
-  info.state.applied = pVnode->state.applied;
+  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
+  if (vnodeSaveInfo(dir, &info) < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
-  vnodeSaveInfo(dir, &info);
+  // commit each sub-system
+  if (metaCommit(pVnode->pMeta) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+  if (tsdbCommit(pVnode->pTsdb) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+  if (tqCommit(pVnode->pTq) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+  // walCommit (TODO)
+
+  // commit info
+  if (vnodeCommitInfo(dir, &info) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // apply the commit (TODO)
+  vnodeBufPoolReset(pVnode->onCommit);
+  pVnode->onCommit->next = pVnode->pPool;
+  pVnode->pPool = pVnode->onCommit;
+  pVnode->onCommit = NULL;
+
+  vInfo("vgId:%d commit over", TD_VID(pVnode));
+
+  return 0;
+}
+
+static int vnodeCommitImpl(void *arg) {
+  SVnode *pVnode = (SVnode *)arg;
 
   // metaCommit(pVnode->pMeta);
   tqCommit(pVnode->pTq);
   tsdbCommit(pVnode->pTsdb);
 
-  vnodeCommitInfo(dir, &info);
+  // vnodeBufPoolRecycle(pVnode);
-
-  vnodeBufPoolRecycle(pVnode);
   tsem_post(&(pVnode->canCommit));
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeInt.c

@@ -14,7 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "vnodeInt.h"
+#include "vnd.h"
 // #include "vnodeInt.h"
 
 int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; }

source/dnode/vnode/src/vnd/vnodeModule.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
 typedef struct SVnodeTask SVnodeTask;
 struct SVnodeTask {

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
-#include "vnodeSync.h"
 
 int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
   SVnodeInfo info = {0};
@@ -85,7 +84,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   tsem_init(&(pVnode->canCommit), 0, 1);
 
   // open buffer pool
-  if (vnodeOpenBufPool(pVnode) < 0) {
+  if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) {
     vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
@@ -97,9 +96,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   }
 
   // open tsdb
-  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TSDB_DIR);
+  if (tsdbOpen(pVnode, &pVnode->pTsdb) < 0) {
-  pVnode->pTsdb = tsdbOpen(tdir, pVnode, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode));
-  if (pVnode->pTsdb == NULL) {
     vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
@@ -126,17 +123,17 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
     goto _err;
   }
 
-  // sync integration
+  // vnode begin
-  // open sync
+  if (vnodeBegin(pVnode) < 0) {
-  if (vnodeSyncOpen(pVnode, dir)) {
+    vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
 
-#if 0
+  // open sync
-  if (vnodeBegin() < 0) {
+  if (vnodeSyncOpen(pVnode, dir)) {
+    vError("vgId: %d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
-#endif
 
   return pVnode;
 
@@ -153,14 +150,9 @@ _err:
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    // commit (TODO: use option to control)
+    vnodeCommit(pVnode);
-    vnodeSyncCommit(pVnode);
-    // close vnode
-    vnodeQueryClose(pVnode);
-
-    // sync integration
     vnodeSyncClose(pVnode);
-
+    vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
     tsdbClose(pVnode->pTsdb);

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
 int vnodeQueryOpen(SVnode *pVnode) {
   return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), NULL, (void **)&pVnode->pQuery, &pVnode->msgCb);
@@ -22,94 +22,76 @@ int vnodeQueryOpen(SVnode *pVnode) {
 void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); }
 
 int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
-  STbCfg *        pTbCfg = NULL;
-  STbCfg *        pStbCfg = NULL;
-  tb_uid_t        uid;
-  int32_t         nCols;
-  int32_t         nTagCols;
-  SSchemaWrapper *pSW = NULL;
-  STableMetaRsp * pTbMetaMsg = NULL;
-  STableMetaRsp   metaRsp = {0};
-  SSchema *       pTagSchema;
-  SRpcMsg         rpcMsg;
-  int             msgLen = 0;
-  int32_t         code = 0;
-  char            tableFName[TSDB_TABLE_FNAME_LEN];
-  int32_t         rspLen = 0;
-  void *          pRsp = NULL;
-
   STableInfoReq  infoReq = {0};
+  STableMetaRsp  metaRsp = {0};
+  SMetaReader    mer1 = {0};
+  SMetaReader    mer2 = {0};
+  char           tableFName[TSDB_TABLE_FNAME_LEN];
+  SRpcMsg        rpcMsg;
+  int32_t        code = 0;
+  int32_t        rspLen = 0;
+  void          *pRsp = NULL;
+  SSchemaWrapper schema = {0};
+  SSchemaWrapper schemaTag = {0};
+
+  // decode req
   if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
     code = TSDB_CODE_INVALID_MSG;
     goto _exit;
   }
 
   metaRsp.dbId = pVnode->config.dbId;
-  memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
   strcpy(metaRsp.tbName, infoReq.tbName);
+  memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
 
   sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
-  code = vnodeValidateTableHash(&pVnode->config, tableFName);
+  code = vnodeValidateTableHash(pVnode, tableFName);
   if (code) {
     goto _exit;
   }
 
-  pTbCfg = metaGetTbInfoByName(pVnode->pMeta, infoReq.tbName, &uid);
+  // query meta
-  if (pTbCfg == NULL) {
+  metaReaderInit(&mer1, pVnode->pMeta, 0);
-    code = TSDB_CODE_VND_TB_NOT_EXIST;
+
+  if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) {
     goto _exit;
   }
 
-  if (pTbCfg->type == META_CHILD_TABLE) {
+  metaRsp.tableType = mer1.me.type;
-    pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid);
-    if (pStbCfg == NULL) {
-      code = TSDB_CODE_VND_TB_NOT_EXIST;
-      goto _exit;
-    }
-
-    pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true);
-  } else {
-    pSW = metaGetTableSchema(pVnode->pMeta, uid, 0, true);
-  }
-
-  nCols = pSW->nCols;
-  if (pTbCfg->type == META_SUPER_TABLE) {
-    nTagCols = pTbCfg->stbCfg.nTagCols;
-    pTagSchema = pTbCfg->stbCfg.pTagSchema;
-  } else if (pTbCfg->type == META_CHILD_TABLE) {
-    nTagCols = pStbCfg->stbCfg.nTagCols;
-    pTagSchema = pStbCfg->stbCfg.pTagSchema;
-  } else {
-    nTagCols = 0;
-    pTagSchema = NULL;
-  }
-
-  metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema));
-  if (metaRsp.pSchemas == NULL) {
-    code = TSDB_CODE_VND_OUT_OF_MEMORY;
-    goto _exit;
-  }
-
-  if (pTbCfg->type == META_CHILD_TABLE) {
-    strcpy(metaRsp.stbName, pStbCfg->name);
-    metaRsp.suid = pTbCfg->ctbCfg.suid;
-  } else if (pTbCfg->type == META_SUPER_TABLE) {
-    strcpy(metaRsp.stbName, pTbCfg->name);
-    metaRsp.suid = uid;
-  }
-  metaRsp.numOfTags = nTagCols;
-  metaRsp.numOfColumns = nCols;
-  metaRsp.tableType = pTbCfg->type;
-  metaRsp.tuid = uid;
   metaRsp.vgId = TD_VID(pVnode);
+  metaRsp.tuid = mer1.me.uid;
 
-  memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
+  if (mer1.me.type == TSDB_SUPER_TABLE) {
-  if (nTagCols) {
+    strcpy(metaRsp.stbName, mer1.me.name);
-    memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols);
+    schema = mer1.me.stbEntry.schema;
+    schemaTag = mer1.me.stbEntry.schemaTag;
+    metaRsp.suid = mer1.me.uid;
+  } else if (mer1.me.type == TSDB_CHILD_TABLE) {
+    metaReaderInit(&mer2, pVnode->pMeta, 0);
+    if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+
+    strcpy(metaRsp.stbName, mer2.me.name);
+    metaRsp.suid = mer2.me.uid;
+    schema = mer2.me.stbEntry.schema;
+    schemaTag = mer2.me.stbEntry.schemaTag;
+  } else if (mer1.me.type == TSDB_NORMAL_TABLE) {
+    schema = mer1.me.ntbEntry.schema;
+  } else {
+    ASSERT(0);
   }
 
-_exit:
+  metaRsp.numOfTags = schemaTag.nCols;
+  metaRsp.numOfColumns = schema.nCols;
+  metaRsp.precision = pVnode->config.tsdbCfg.precision;
+  metaRsp.sversion = schema.sver;
+  metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags));
 
+  memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
+  if (schemaTag.nCols) {
+    memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
+  }
+
+  // encode and send response
   rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
   if (rspLen < 0) {
     code = TSDB_CODE_INVALID_MSG;
@@ -123,23 +105,6 @@ _exit:
   }
   tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
 
-  tFreeSTableMetaRsp(&metaRsp);
-  if (pSW != NULL) {
-    taosMemoryFreeClear(pSW->pSchema);
-    taosMemoryFreeClear(pSW);
-  }
-
-  if (pTbCfg) {
-    taosMemoryFreeClear(pTbCfg->name);
-    if (pTbCfg->type == META_SUPER_TABLE) {
-      taosMemoryFree(pTbCfg->stbCfg.pTagSchema);
-    } else if (pTbCfg->type == META_SUPER_TABLE) {
-      kvRowFree(pTbCfg->ctbCfg.pTag);
-    }
-
-    taosMemoryFreeClear(pTbCfg);
-  }
-
   rpcMsg.handle = pMsg->handle;
   rpcMsg.ahandle = pMsg->ahandle;
   rpcMsg.refId = pMsg->refId;
@@ -148,12 +113,18 @@ _exit:
   rpcMsg.code = code;
 
   tmsgSendRsp(&rpcMsg);
-  return TSDB_CODE_SUCCESS;
+
+_exit:
+  taosMemoryFree(metaRsp.pSchemas);
+  metaReaderClear(&mer2);
+  metaReaderClear(&mer1);
+  return code;
 }
 
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   pLoad->vgId = TD_VID(pVnode);
-  pLoad->syncState = TAOS_SYNC_STATE_LEADER;
+  //pLoad->syncState = TAOS_SYNC_STATE_LEADER;
+  pLoad->syncState = syncGetMyRole(pVnode->sync); // sync integration
   pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
   pLoad->numOfTimeSeries = 400;
   pLoad->totalStorage = 300;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -13,14 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "sync.h"
+#include "vnd.h"
-#include "syncTools.h"
-#include "vnodeInt.h"
 
-static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
+static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
-static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp);
+static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
+static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
+static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
+static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
 #if 0
@@ -47,44 +48,55 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
 
 int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) {
   void *ptr = NULL;
+  void *pReq;
+  int   len;
   int   ret;
 
-  if (pVnode->config.streamMode == 0) {
+  vTrace("vgId: %d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
-    ptr = vnodeMalloc(pVnode, pMsg->contLen);
+         version);
-    if (ptr == NULL) {
-      // TODO: handle error
-    }
 
-    // TODO: copy here need to be extended
+  pVnode->state.applied = version;
-    memcpy(ptr, pMsg->pCont, pMsg->contLen);
+
-  }
+  // skip header
+  pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
+  len = pMsg->contLen - sizeof(SMsgHead);
 
   // todo: change the interface here
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
-    // TODO: handle error
+    vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
+    return -1;
   }
 
   switch (pMsg->msgType) {
+    /* META */
     case TDMT_VND_CREATE_STB:
-      ret = vnodeProcessCreateStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)));
+      if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
-      break;
-    case TDMT_VND_CREATE_TABLE:
-      pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
-      vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp);
       break;
     case TDMT_VND_ALTER_STB:
-      vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)));
+      if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_STB:
-      vTrace("vgId:%d, process drop stb req", TD_VID(pVnode));
+      if (vnodeProcessDropStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
+      break;
+    case TDMT_VND_CREATE_TABLE:
+      if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      break;
+    case TDMT_VND_ALTER_TABLE:
+      if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TABLE:
+      if (vnodeProcessDropTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
       break;
+    case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
+      if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
+        // TODO
+      }
+    } break;
+    /* TSDB */
     case TDMT_VND_SUBMIT:
-      tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+      if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
-      pRsp->msgType = TDMT_VND_SUBMIT_RSP;
-      vnodeProcessSubmitReq(pVnode, ptr, pRsp);
       break;
+    /* TQ */
     case TDMT_VND_MQ_VG_CHANGE:
       if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                                pMsg->contLen - sizeof(SMsgHead)) < 0) {
@@ -101,19 +113,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
                             0) < 0) {
       }
     } break;
-    case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
-      if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-        // TODO
-      }
-      // } break;
-      // case TDMT_VND_CANCEL_SMA: {  // timeRangeSMA
-      // } break;
-      // case TDMT_VND_DROP_SMA: {  // timeRangeSMA
-      //   if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
-      //     // TODO
-      //   }
-
-    } break;
     case TDMT_VND_ALTER_VNODE:
       break;
     default:
@@ -121,17 +120,24 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       break;
   }
 
-  pVnode->state.applied = version;
+  vDebug("vgId: %d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
 
-  // Check if it needs to commit
+  // commit if need
   if (vnodeShouldCommit(pVnode)) {
-    // tsem_wait(&(pVnode->canCommit));
+    vInfo("vgId:%d commit at version %" PRId64, TD_VID(pVnode), version);
-    if (vnodeAsyncCommit(pVnode) < 0) {
+    // commit current change
-      // TODO: handle error
+    vnodeCommit(pVnode);
-    }
+
+    // start a new one
+    vnodeBegin(pVnode);
   }
 
   return 0;
+
+_err:
+  vDebug("vgId: %d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+         tstrerror(terrno), version);
+  return -1;
 }
 
 int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
@@ -193,6 +199,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
 
 // sync integration
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
+  if (syncEnvIsStart()) {
     SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
     assert(pSyncNode != NULL);
 
@@ -270,99 +277,117 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
     }
 
     syncNodeRelease(pSyncNode);
-
-  return 0;
-}
-
-static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) {
-  SVCreateTbReq vCreateTbReq = {0};
-  tDeserializeSVCreateTbReq(pReq, &vCreateTbReq);
-  if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq), NULL) < 0) {
-    // TODO
-    return -1;
-  }
-
-  tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
-
-  taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
-  taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
-  if (vCreateTbReq.stbCfg.pRSmaParam) {
-    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1);
-    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2);
-    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
-  }
-  taosMemoryFree(vCreateTbReq.name);
-
-  return 0;
-}
-
-static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp) {
-  SVCreateTbBatchReq vCreateTbBatchReq = {0};
-  SVCreateTbBatchRsp vCreateTbBatchRsp = {0};
-  tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq);
-  int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray);
-
-  STbDdlH ddlHandle = {
-      .ahandle = pVnode->pTsdb,
-      .result = NULL,
-      .fp = tsdbFetchTbUidList,
-  };
-
-  for (int i = 0; i < reqNum; i++) {
-    SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
-
-    char      tableFName[TSDB_TABLE_FNAME_LEN];
-    SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
-    sprintf(tableFName, "%s.%s", pVnode->config.dbname, pCreateTbReq->name);
-
-    int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName);
-    if (code) {
-      SVCreateTbRsp rsp;
-      rsp.code = code;
-
-      taosArrayPush(vCreateTbBatchRsp.rspList, &rsp);
-    }
-
-    if (metaCreateTable(pVnode->pMeta, pCreateTbReq, &ddlHandle) < 0) {
-      // TODO: handle error
-      vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name);
-    }
-    // TODO: to encapsule a free API
-    taosMemoryFree(pCreateTbReq->name);
-    if (pCreateTbReq->type == TD_SUPER_TABLE) {
-      taosMemoryFree(pCreateTbReq->stbCfg.pSchema);
-      taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema);
-      if (pCreateTbReq->stbCfg.pRSmaParam) {
-        taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
-      }
-    } else if (pCreateTbReq->type == TD_CHILD_TABLE) {
-      taosMemoryFree(pCreateTbReq->ctbCfg.pTag);
   } else {
-      taosMemoryFree(pCreateTbReq->ntbCfg.pSchema);
+    vError("==vnodeProcessSyncReq== error syncEnv stop");
-      if (pCreateTbReq->ntbCfg.pRSmaParam) {
-        taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
   }
-    }
-  }
-
-  tsdbUpdateTbUidList(pVnode->pTsdb, ddlHandle.result);
-
-  vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray));
-  taosArrayDestroy(vCreateTbBatchReq.pArray);
-  if (vCreateTbBatchRsp.rspList) {
-    int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp);
-    void   *msg = rpcMallocCont(contLen);
-    tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp);
-    taosArrayDestroy(vCreateTbBatchRsp.rspList);
-
-    pRsp->pCont = msg;
-    pRsp->contLen = contLen;
-  }
-
   return 0;
 }
 
-static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
+static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
+  SVCreateStbReq req = {0};
+  SCoder         coder;
+
+  pRsp->msgType = TDMT_VND_CREATE_STB_RSP;
+  pRsp->code = TSDB_CODE_SUCCESS;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+
+  // decode and process req
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
+
+  if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
+    pRsp->code = terrno;
+    goto _err;
+  }
+
+  // tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
+
+  if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) {
+    pRsp->code = terrno;
+    goto _err;
+  }
+
+  tCoderClear(&coder);
+  return 0;
+
+_err:
+  tCoderClear(&coder);
+  return -1;
+}
+
+static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
+  SCoder             coder = {0};
+  int                rcode = 0;
+  SVCreateTbBatchReq req = {0};
+  SVCreateTbReq     *pCreateReq;
+  SVCreateTbBatchRsp rsp = {0};
+  SVCreateTbRsp      cRsp = {0};
+  char               tbName[TSDB_TABLE_FNAME_LEN];
+
+  pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
+  pRsp->code = TSDB_CODE_SUCCESS;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+
+  // decode
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
+  if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) {
+    rcode = -1;
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _exit;
+  }
+
+  rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs);
+  if (rsp.pArray == NULL) {
+    rcode = -1;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  // loop to create table
+  for (int iReq = 0; iReq < req.nReqs; iReq++) {
+    pCreateReq = req.pReqs + iReq;
+
+    // validate hash
+    sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name);
+    if (vnodeValidateTableHash(pVnode, tbName) < 0) {
+      cRsp.code = TSDB_CODE_VND_HASH_MISMATCH;
+      taosArrayPush(rsp.pArray, &cRsp);
+      continue;
+    }
+
+    // do create table
+    if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) {
+      cRsp.code = terrno;
+    } else {
+      cRsp.code = TSDB_CODE_SUCCESS;
+    }
+
+    taosArrayPush(rsp.pArray, &cRsp);
+  }
+
+  tCoderClear(&coder);
+
+  // prepare rsp
+  tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen);
+  pRsp->pCont = rpcMallocCont(pRsp->contLen);
+  if (pRsp->pCont == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    rcode = -1;
+    goto _exit;
+  }
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pRsp->pCont, pRsp->contLen, TD_ENCODER);
+  tEncodeSVCreateTbBatchRsp(&coder, &rsp);
+
+_exit:
+  taosArrayClear(rsp.pArray);
+  tCoderClear(&coder);
+  return rcode;
+}
+
+static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  // ASSERT(0);
+#if 0
   SVCreateTbReq vAlterTbReq = {0};
   vTrace("vgId:%d, process alter stb req", TD_VID(pVnode));
   tDeserializeSVCreateTbReq(pReq, &vAlterTbReq);
@@ -373,20 +398,44 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
     taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
   }
   taosMemoryFree(vAlterTbReq.name);
+#endif
   return 0;
 }
 
-static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp) {
+static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  // TODO
+  // ASSERT(0);
+  return 0;
+}
+
+static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  // TODO
+  ASSERT(0);
+  return 0;
+}
+
+static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  // TODO
+  ASSERT(0);
+  return 0;
+}
+
+static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+  SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
   SSubmitRsp  rsp = {0};
 
   pRsp->code = 0;
 
   // handle the request
-  if (tsdbInsertData(pVnode->pTsdb, pSubmitReq, &rsp) < 0) {
+  if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) {
     pRsp->code = terrno;
     return -1;
   }
 
+  // pRsp->msgType = TDMT_VND_SUBMIT_RSP;
+  // vnodeProcessSubmitReq(pVnode, ptr, pRsp);
+  // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+
   // encode the response (TODO)
   pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp));
   memcpy(pRsp->pCont, &rsp, sizeof(rsp));