Merge branch 'feature/dnode3' of https://github.com/taosdata/TDengine into feature/dnode3

.devcontainer/devcontainer.json

@@ -19,7 +19,9 @@
 		"ms-vscode.cmake-tools",
 		"austin.code-gnu-global",
 		"visualstudioexptteam.vscodeintel",
-		"eamodio.gitlens"
+		"eamodio.gitlens",
+		"matepek.vscode-catch2-test-adapter",
+		"spmeesseman.vscode-taskexplorer"
 	],
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.

include/common/tmsg.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "encode.h"
 #include "taosdef.h"
 #include "taoserror.h"
+#include "tarray.h"
 #include "tcoding.h"
 #include "tdataformat.h"
 #include "tlist.h"
@@ -56,46 +57,6 @@ extern int   tMsgDict[];
 
 typedef uint16_t tmsg_t;
 
-/* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */
-struct SMEListNode {
-  TD_SLIST_NODE(SMEListNode);
-  SEncoder coder;
-};
-
-typedef struct SMsgEncoder {
-  SEncoder coder;
-  TD_SLIST(SMEListNode) eStack;  // encode stack
-} SMsgEncoder;
-
-struct SMDFreeListNode {
-  TD_SLIST_NODE(SMDFreeListNode);
-  char payload[];
-};
-
-struct SMDListNode {
-  TD_SLIST_NODE(SMDListNode);
-  SDecoder coder;
-};
-
-typedef struct SMsgDecoder {
-  SDecoder coder;
-  TD_SLIST(SMDListNode) dStack;
-  TD_SLIST(SMDFreeListNode) freeList;
-} SMsgDecoder;
-
-#define TMSG_MALLOC(SIZE, DECODER)                                         \
-  ({                                                                       \
-    void* ptr = malloc((SIZE) + sizeof(struct SMDFreeListNode));           \
-    if (ptr) {                                                             \
-      TD_SLIST_PUSH(&((DECODER)->freeList), (struct SMDFreeListNode*)ptr); \
-      ptr = POINTER_SHIFT(ptr, sizeof(struct SMDFreeListNode*));           \
-    }                                                                      \
-    ptr;                                                                   \
-  })
-
-void tmsgInitMsgDecoder(SMsgDecoder* pMD, td_endian_t endian, uint8_t* data, int64_t size);
-void tmsgClearMsgDecoder(SMsgDecoder* pMD);
-
 /* ------------------------ OTHER DEFINITIONS ------------------------ */
 // IE type
 #define TSDB_IE_TYPE_SEC 1
@@ -173,6 +134,7 @@ typedef enum _mgmt_table {
 
 typedef struct SBuildTableMetaInput {
   int32_t vgId;
+  char*   dbName;
   char*   tableFullName;
 } SBuildTableMetaInput;
 
@@ -354,9 +316,9 @@ typedef struct SEpSet {
 } SEpSet;
 
 static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) {
-  if(buf == NULL) return sizeof(SEpSet);
+  if (buf == NULL) return sizeof(SEpSet);
   memcpy(buf, pEp, sizeof(SEpSet));
-  //TODO: endian conversion
+  // TODO: endian conversion
   return sizeof(SEpSet);
 }
 
@@ -370,7 +332,7 @@ typedef struct {
   int64_t clusterId;
   int32_t connId;
   int8_t  superUser;
-  int8_t  reserved[5];
+  int8_t  align[3];
   SEpSet  epSet;
 } SConnectRsp;
 
@@ -383,20 +345,17 @@ typedef struct {
   int32_t maxStreams;
   int32_t accessState;  // Configured only by command
   int64_t maxStorage;   // In unit of GB
-  int32_t reserve[8];
 } SCreateAcctMsg, SAlterAcctMsg;
 
 typedef struct {
-  char    user[TSDB_USER_LEN];
+  char user[TSDB_USER_LEN];
-  int32_t reserve[8];
 } SDropUserMsg, SDropAcctMsg;
 
 typedef struct {
-  int8_t  type;
+  int8_t type;
-  char    user[TSDB_USER_LEN];
+  char   user[TSDB_USER_LEN];
-  char    pass[TSDB_PASSWORD_LEN];
+  char   pass[TSDB_PASSWORD_LEN];
-  int8_t  superUser;  // denote if it is a super user or not
+  int8_t superUser;  // denote if it is a super user or not
-  int32_t reserve[8];
 } SCreateUserMsg, SAlterUserMsg;
 
 typedef struct {
@@ -585,7 +544,6 @@ typedef struct {
   int8_t  update;
   int8_t  cacheLastRow;
   int8_t  ignoreExist;
-  int32_t reserve[8];
 } SCreateDbMsg;
 
 typedef struct {
@@ -598,29 +556,24 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
-  int32_t reserve[8];
 } SAlterDbMsg;
 
 typedef struct {
-  char    db[TSDB_TABLE_FNAME_LEN];
+  char   db[TSDB_TABLE_FNAME_LEN];
-  int8_t  ignoreNotExists;
+  int8_t ignoreNotExists;
-  int32_t reserve[8];
 } SDropDbMsg;
 
 typedef struct {
   char    db[TSDB_TABLE_FNAME_LEN];
   int32_t vgVersion;
-  int32_t reserve[8];
 } SUseDbMsg;
 
 typedef struct {
-  char    db[TSDB_TABLE_FNAME_LEN];
+  char db[TSDB_TABLE_FNAME_LEN];
-  int32_t reserve[8];
 } SSyncDbMsg;
 
 typedef struct {
-  char    db[TSDB_TABLE_FNAME_LEN];
+  char db[TSDB_TABLE_FNAME_LEN];
-  int32_t reserve[8];
 } SCompactDbMsg;
 
 typedef struct {
@@ -676,7 +629,7 @@ typedef struct {
 typedef struct {
   int32_t vgId;
   int8_t  role;
-  int8_t  reserved[3];
+  int8_t  align[3];
   int64_t totalStorage;
   int64_t compStorage;
   int64_t pointsWritten;
@@ -713,7 +666,7 @@ typedef struct {
 typedef struct {
   int32_t  id;
   int8_t   isMnode;
-  int8_t   reserved;
+  int8_t   align;
   uint16_t port;
   char     fqdn[TSDB_FQDN_LEN];
 } SDnodeEp;
@@ -774,8 +727,9 @@ typedef struct {
 } SAuthVnodeMsg;
 
 typedef struct {
-  int32_t vgId;
+  SMsgHead header;
-  char    tableFname[TSDB_TABLE_FNAME_LEN];
+  char     dbFname[TSDB_DB_FNAME_LEN];
+  char     tableFname[TSDB_TABLE_FNAME_LEN];
 } STableInfoMsg;
 
 typedef struct {
@@ -809,6 +763,7 @@ typedef struct {
 typedef struct {
   char     tbFname[TSDB_TABLE_FNAME_LEN];  // table full name
   char     stbFname[TSDB_TABLE_FNAME_LEN];
+  char     dbFname[TSDB_DB_FNAME_LEN];
   int32_t  numOfTags;
   int32_t  numOfColumns;
   int8_t   precision;
@@ -947,7 +902,7 @@ typedef struct {
   int32_t totalDnodes;
   int32_t onlineDnodes;
   int8_t  killConnection;
-  int8_t  reserved[3];
+  int8_t  align[3];
   SEpSet  epSet;
 } SHeartBeatRsp;
 
@@ -975,7 +930,7 @@ typedef struct {
 
 typedef struct {
   int8_t finished;
-  int8_t reserved1[7];
+  int8_t align[7];
   char   name[TSDB_STEP_NAME_LEN];
   char   desc[TSDB_STEP_DESC_LEN];
 } SStartupMsg;
@@ -1059,6 +1014,7 @@ typedef struct {
 } SUpdateTagValRsp;
 
 typedef struct SSubQueryMsg {
+  SMsgHead header;
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
@@ -1067,6 +1023,7 @@ typedef struct SSubQueryMsg {
 } SSubQueryMsg;
 
 typedef struct SResReadyMsg {
+  SMsgHead header;
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
@@ -1077,6 +1034,7 @@ typedef struct SResReadyRsp {
 } SResReadyRsp;
 
 typedef struct SResFetchMsg {
+  SMsgHead header;
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
@@ -1118,10 +1076,10 @@ typedef struct STaskDropRsp {
 } STaskDropRsp;
 
 typedef struct {
-  int8_t  igExists;
+  int8_t igExists;
-  char*   name;
+  char*  name;
-  char*   physicalPlan;
+  char*  physicalPlan;
-  char*   logicalPlan;
+  char*  logicalPlan;
 } SCMCreateTopicReq;
 
 static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) {
@@ -1157,8 +1115,8 @@ static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopi
 }
 
 typedef struct {
-  char* topicName;
+  char*   topicName;
-  char* consumerGroup;
+  char*   consumerGroup;
   int64_t consumerId;
 } SCMSubscribeReq;
 
@@ -1179,7 +1137,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
 
 typedef struct {
   int32_t vgId;
-  SEpSet pEpSet;
+  SEpSet  pEpSet;
 } SCMSubscribeRsp;
 
 static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) {
@@ -1248,9 +1206,9 @@ typedef struct SVCreateTbReq {
   char*    name;
   uint32_t ttl;
   uint32_t keep;
-#define TD_SUPER_TABLE 0
+#define TD_SUPER_TABLE TSDB_SUPER_TABLE
-#define TD_CHILD_TABLE 1
+#define TD_CHILD_TABLE TSDB_CHILD_TABLE
-#define TD_NORMAL_TABLE 2
+#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
   uint8_t type;
   union {
     struct {
@@ -1271,10 +1229,15 @@ typedef struct SVCreateTbReq {
   };
 } SVCreateTbReq;
 
-int   tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq);
+typedef struct {
-int   tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq);
+  uint64_t ver;  // use a general definition
-int   tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq);
+  SArray*  pArray;
+} SVCreateTbBatchReq;
+
+int   tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
 void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
+int   tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq);
+void* tSVCreateTbBatchReqDeserialize(void* buf, SVCreateTbBatchReq* pReq);
 
 typedef struct SVCreateTbRsp {
 } SVCreateTbRsp;

include/common/tmsgtype.h

@@ -40,8 +40,9 @@ enum {
   // the SQL below is for mgmt node
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_STABLE, "create-stable" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
-  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" )  
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" )

include/dnode/vnode/meta/meta.h

@@ -25,20 +25,20 @@
 extern "C" {
 #endif
 
+#define META_SUPER_TABLE TD_SUPER_TABLE
+#define META_CHILD_TABLE TD_CHILD_TABLE
+#define META_NORMAL_TABLE TD_NORMAL_TABLE
+
 // Types exported
 typedef struct SMeta SMeta;
 
-#define META_SUPER_TABLE 0
-#define META_CHILD_TABLE 1
-#define META_NORMAL_TABLE 2
-
 typedef struct SMetaCfg {
   /// LRU cache size
   uint64_t lruSize;
 } SMetaCfg;
 
 typedef struct {
-  int32_t  nCols;
+  uint32_t nCols;
   SSchema *pSchema;
 } SSchemaWrapper;
 
@@ -59,9 +59,9 @@ STbCfg *        metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
 STbCfg *        metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 
-SMTbCursor *   metaOpenTbCursor(SMeta *pMeta);
+SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
-void           metaCloseTbCursor(SMTbCursor *pTbCur);
+void        metaCloseTbCursor(SMTbCursor *pTbCur);
-char *metaTbCursorNext(SMTbCursor *pTbCur);
+char *      metaTbCursorNext(SMTbCursor *pTbCur);
 
 // Options
 void metaOptionsInit(SMetaCfg *pMetaCfg);

include/libs/parser/parsenodes.h

@@ -27,7 +27,7 @@ extern "C" {
 #include "tname.h"
 #include "tvariant.h"
 
-/*
+/**
  * The first field of a node of any type is guaranteed to be the int16_t.
  * Hence the type of any node can be gotten by casting it to SQueryNode. 
  */
@@ -157,7 +157,7 @@ typedef struct SVgDataBlocks {
 typedef struct SInsertStmtInfo {
   int16_t     nodeType;
   SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
-  int8_t      schemaAttache;        // denote if submit block is built with table schema or not
+  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
   uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
   uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
   const char* sql;                 // current sql statement position

include/libs/planner/planner.h

@@ -119,11 +119,12 @@ typedef struct SSubplanId {
 } SSubplanId;
 
 typedef struct SSubplan {
-  SSubplanId id;          // unique id of the subplan
+  SSubplanId id;           // unique id of the subplan
   int32_t    type;         // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN|QUERY_TYPE_MODIFY
-  int32_t    level;        // the execution level of current subplan, starting from 0.
+  int32_t    msgType;      // message type for subplan, used to denote the send message type to vnode.
+  int32_t    level;        // the execution level of current subplan, starting from 0 in a top-down manner.
   SEpSet     execEpSet;    // for the scan/modify subplan, the optional execution node
-  SArray    *pChildern;    // the datasource subplan,from which to fetch the result
+  SArray    *pChildren;    // the datasource subplan,from which to fetch the result
   SArray    *pParents;     // the data destination subplan, get data from current subplan
   SPhyNode  *pNode;        // physical plan of current subplan
   SDataSink *pDataSink;    // data of the subplan flow into the datasink
@@ -140,7 +141,7 @@ struct SQueryNode;
 /**
  * Create the physical plan for the query, according to the AST.
  */
-int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag);
+int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId);
 
 // Set datasource of this subplan, multiple calls may be made to a subplan.
 // @subplan subplan to be schedule

include/libs/qcom/query.h

@@ -75,7 +75,7 @@ typedef struct STableMeta {
 } STableMeta;
 
 typedef struct SDBVgroupInfo {
-  int32_t   lock;
+  SRWLatch  lock;
   int32_t   vgVersion;  
   int8_t    hashMethod;
   SHashObj *vgInfo;  //key:vgId, value:SVgroupInfo

include/libs/scheduler/scheduler.h

@@ -50,23 +50,37 @@ typedef struct SQueryProfileSummary {
   uint64_t resultSize;   // generated result size in Kb.
 } SQueryProfileSummary;
 
+typedef struct SQueryNodeAddr{
+  int32_t    nodeId; //vgId or qnodeId
+  int8_t     inUse;
+  int8_t     numOfEps;
+  SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
+} SQueryNodeAddr;
+
+typedef struct SQueryResult {
+  int32_t  code;
+  uint64_t numOfRows;
+  int32_t  msgSize;
+  char    *msg;
+} SQueryResult;
+
 int32_t schedulerInit(SSchedulerCfg *cfg);
 
 /**
  * Process the query job, generated according to the query physical plan.
  * This is a synchronized API, and is also thread-safety.
- * @param qnodeList  Qnode address list, element is SEpAddr
+ * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows);
+int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes);
 
 /**
  * Process the query job, generated according to the query physical plan.
  * This is a asynchronized API, and is also thread-safety.
- * @param qnodeList  Qnode address list, element is SEpAddr
+ * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob);
+int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob);
 
 int32_t scheduleFetchRows(void *pJob, void **data);
 

include/os/osSysinfo.h

@@ -68,8 +68,6 @@ typedef struct {
 
 SysNameInfo taosGetSysNameInfo();
 
-int64_t taosGetPid();
-
 #ifdef __cplusplus
 }
 #endif

include/util/encode.h

@@ -16,6 +16,7 @@
 #ifndef _TD_UTIL_ENCODE_H_
 #define _TD_UTIL_ENCODE_H_
 
+#include "freelist.h"
 #include "tcoding.h"
 #include "tmacro.h"
 
@@ -23,13 +24,6 @@
 extern "C" {
 #endif
 
-typedef struct {
-  td_endian_t endian;
-  uint8_t*    data;
-  int32_t     size;
-  int32_t     pos;
-} SEncoder, SDecoder;
-
 #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
 #define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0]
 
@@ -57,31 +51,157 @@ typedef struct {
 #define tRGet32 tRPut32
 #define tRGet64 tRPut64
 
+typedef enum { TD_ENCODER, TD_DECODER } td_coder_t;
+
+#define CODER_NODE_FIELDS \
+  uint8_t* data;          \
+  int32_t  size;          \
+  int32_t  pos;
+
+struct SCoderNode {
+  TD_SLIST_NODE(SCoderNode);
+  CODER_NODE_FIELDS
+};
+
+typedef struct {
+  td_coder_t  type;
+  td_endian_t endian;
+  SFreeList   fl;
+  CODER_NODE_FIELDS
+  TD_SLIST(SCoderNode) stack;
+} SCoder;
+
+#define TD_CODER_POS(CODER) ((CODER)->pos)
 #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
-#define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
+#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
+#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl))
 
-/* ------------------------ FOR ENCODER ------------------------ */
+void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
-static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int32_t size) {
+void tCoderClear(SCoder* pCoder);
-  pEncoder->endian = endian;
+
-  pEncoder->data = data;
+/* ------------------------ ENCODE ------------------------ */
-  pEncoder->size = (data) ? size : 0;
+int        tStartEncode(SCoder* pEncoder);
-  pEncoder->pos = 0;
+void       tEndEncode(SCoder* pEncoder);
-}
+static int tEncodeU8(SCoder* pEncoder, uint8_t val);
+static int tEncodeI8(SCoder* pEncoder, int8_t val);
+static int tEncodeU16(SCoder* pEncoder, uint16_t val);
+static int tEncodeI16(SCoder* pEncoder, int16_t val);
+static int tEncodeU32(SCoder* pEncoder, uint32_t val);
+static int tEncodeI32(SCoder* pEncoder, int32_t val);
+static int tEncodeU64(SCoder* pEncoder, uint64_t val);
+static int tEncodeI64(SCoder* pEncoder, int64_t val);
+static int tEncodeU16v(SCoder* pEncoder, uint16_t val);
+static int tEncodeI16v(SCoder* pEncoder, int16_t val);
+static int tEncodeU32v(SCoder* pEncoder, uint32_t val);
+static int tEncodeI32v(SCoder* pEncoder, int32_t val);
+static int tEncodeU64v(SCoder* pEncoder, uint64_t val);
+static int tEncodeI64v(SCoder* pEncoder, int64_t val);
+static int tEncodeFloat(SCoder* pEncoder, float val);
+static int tEncodeDouble(SCoder* pEncoder, double val);
+static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len);
+static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len);
+static int tEncodeCStr(SCoder* pEncoder, const char* val);
+
+/* ------------------------ DECODE ------------------------ */
+int         tStartDecode(SCoder* pDecoder);
+void        tEndDecode(SCoder* pDecoder);
+static bool tDecodeIsEnd(SCoder* pCoder);
+static int  tDecodeU8(SCoder* pDecoder, uint8_t* val);
+static int  tDecodeI8(SCoder* pDecoder, int8_t* val);
+static int  tDecodeU16(SCoder* pDecoder, uint16_t* val);
+static int  tDecodeI16(SCoder* pDecoder, int16_t* val);
+static int  tDecodeU32(SCoder* pDecoder, uint32_t* val);
+static int  tDecodeI32(SCoder* pDecoder, int32_t* val);
+static int  tDecodeU64(SCoder* pDecoder, uint64_t* val);
+static int  tDecodeI64(SCoder* pDecoder, int64_t* val);
+static int  tDecodeU16v(SCoder* pDecoder, uint16_t* val);
+static int  tDecodeI16v(SCoder* pDecoder, int16_t* val);
+static int  tDecodeU32v(SCoder* pDecoder, uint32_t* val);
+static int  tDecodeI32v(SCoder* pDecoder, int32_t* val);
+static int  tDecodeU64v(SCoder* pDecoder, uint64_t* val);
+static int  tDecodeI64v(SCoder* pDecoder, int64_t* val);
+static int  tDecodeFloat(SCoder* pDecoder, float* val);
+static int  tDecodeDouble(SCoder* pDecoder, double* val);
+static int  tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len);
+static int  tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len);
+static int  tDecodeCStr(SCoder* pDecoder, const char** val);
+static int  tDecodeCStrTo(SCoder* pDecoder, char* val);
+
+/* ------------------------ IMPL ------------------------ */
+#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS)                        \
+  if ((CODER)->data) {                                                 \
+    if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
+    if (TD_RT_ENDIAN() == (CODER)->endian) {                           \
+      tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL));                      \
+    } else {                                                           \
+      tRPut##BITS(TD_CODER_CURRENT(CODER), &(VAL));                    \
+    }                                                                  \
+  }                                                                    \
+  TD_CODER_MOVE_POS(CODER, sizeof(VAL));                               \
+  return 0;
+
+#define TD_ENCODE_VARIANT_MACRO(CODER, VAL)                       \
+  while ((VAL) >= ENCODE_LIMIT) {                                 \
+    if ((CODER)->data) {                                          \
+      if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1;    \
+      TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \
+    }                                                             \
+                                                                  \
+    (VAL) >>= 7;                                                  \
+    TD_CODER_MOVE_POS(CODER, 1);                                  \
+  }                                                               \
+                                                                  \
+  if ((CODER)->data) {                                            \
+    if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1;      \
+    TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL);                  \
+  }                                                               \
+  TD_CODER_MOVE_POS(CODER, 1);                                    \
+  return 0;
+
+#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS)                         \
+  if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
+  if (TD_RT_ENDIAN() == (CODER)->endian) {                               \
+    tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL));                        \
+  } else {                                                               \
+    tRGet##BITS(PVAL, TD_CODER_CURRENT(CODER));                          \
+  }                                                                      \
+                                                                         \
+  TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL)));                             \
+  return 0;
+
+#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE)           \
+  int32_t i = 0;                                             \
+  *(PVAL) = 0;                                               \
+  for (;;) {                                                 \
+    if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
+    TYPE tval = TD_CODER_CURRENT(CODER)[0];                  \
+    if (tval < ENCODE_LIMIT) {                               \
+      *(PVAL) |= (tval << (7 * i));                          \
+      TD_CODER_MOVE_POS(pDecoder, 1);                        \
+      break;                                                 \
+    } else {                                                 \
+      *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
+      i++;                                                   \
+      TD_CODER_MOVE_POS(pDecoder, 1);                        \
+    }                                                        \
+  }                                                          \
+                                                             \
+  return 0;
 
 // 8
-static FORCE_INLINE int tEncodeU8(SEncoder* pEncoder, uint8_t val) {
+static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) {
   if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
     tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val);
   }
   TD_CODER_MOVE_POS(pEncoder, sizeof(val));
   return 0;
 }
 
-static FORCE_INLINE int tEncodeI8(SEncoder* pEncoder, int8_t val) {
+static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) {
   if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
     tPut(int8_t, TD_CODER_CURRENT(pEncoder), val);
   }
   TD_CODER_MOVE_POS(pEncoder, sizeof(val));
@@ -89,303 +209,99 @@ static FORCE_INLINE int tEncodeI8(SEncoder* pEncoder, int8_t val) {
 }
 
 // 16
-static FORCE_INLINE int tEncodeU16(SEncoder* pEncoder, uint16_t val) {
+static FORCE_INLINE int tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); }
-  if (pEncoder->data) {
+static FORCE_INLINE int tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); }
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(uint16_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut16(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI16(SEncoder* pEncoder, int16_t val) {
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(int16_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut16(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
 // 32
-static FORCE_INLINE int tEncodeU32(SEncoder* pEncoder, uint32_t val) {
+static FORCE_INLINE int tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); }
-  if (pEncoder->data) {
+static FORCE_INLINE int tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); }
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(uint32_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut32(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI32(SEncoder* pEncoder, int32_t val) {
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(int32_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut32(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
 // 64
-static FORCE_INLINE int tEncodeU64(SEncoder* pEncoder, uint64_t val) {
+static FORCE_INLINE int tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); }
-  if (pEncoder->data) {
+static FORCE_INLINE int tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); }
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(uint64_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut64(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI64(SEncoder* pEncoder, int64_t val) {
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1;
-    if (TD_RT_ENDIAN() == pEncoder->endian) {
-      tPut(int64_t, TD_CODER_CURRENT(pEncoder), val);
-    } else {
-      tRPut64(TD_CODER_CURRENT(pEncoder), &val);
-    }
-  }
-  TD_CODER_MOVE_POS(pEncoder, sizeof(val));
-  return 0;
-}
-
 // 16v
-static FORCE_INLINE int tEncodeU16v(SEncoder* pEncoder, uint16_t val) {
+static FORCE_INLINE int tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
-  int64_t i = 0;
+static FORCE_INLINE int tEncodeI16v(SCoder* pEncoder, int16_t val) {
-  while (val >= ENCODE_LIMIT) {
-    if (pEncoder->data) {
-      if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-      TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
-    }
-
-    val >>= 7;
-    i++;
-  }
-
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-    TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
-  }
-
-  TD_CODER_MOVE_POS(pEncoder, i + 1);
-
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI16v(SEncoder* pEncoder, int16_t val) {
   return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val));
 }
-
 // 32v
-static FORCE_INLINE int tEncodeU32v(SEncoder* pEncoder, uint32_t val) {
+static FORCE_INLINE int tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
-  int64_t i = 0;
+static FORCE_INLINE int tEncodeI32v(SCoder* pEncoder, int32_t val) {
-  while (val >= ENCODE_LIMIT) {
-    if (pEncoder->data) {
-      if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-      TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
-    }
-
-    val >>= 7;
-    i++;
-  }
-
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-    TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
-  }
-
-  TD_CODER_MOVE_POS(pEncoder, i + 1);
-
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI32v(SEncoder* pEncoder, int32_t val) {
   return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val));
 }
-
 // 64v
-static FORCE_INLINE int tEncodeU64v(SEncoder* pEncoder, uint64_t val) {
+static FORCE_INLINE int tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); }
-  int64_t i = 0;
+static FORCE_INLINE int tEncodeI64v(SCoder* pEncoder, int64_t val) {
-  while (val >= ENCODE_LIMIT) {
-    if (pEncoder->data) {
-      if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-      TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff;
-    }
-
-    val >>= 7;
-    i++;
-  }
-
-  if (pEncoder->data) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1;
-    TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val;
-  }
-
-  TD_CODER_MOVE_POS(pEncoder, i + 1);
-
-  return 0;
-}
-
-static FORCE_INLINE int tEncodeI64v(SEncoder* pEncoder, int64_t val) {
   return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val));
 }
 
-static FORCE_INLINE int tEncodeFloat(SEncoder* pEncoder, float val) {
+static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) {
-  // TODO
+  union {
+    uint32_t ui;
+    float    f;
+  } v = {.f = val};
+
+  return tEncodeU32(pEncoder, v.ui);
+}
+
+static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) {
+  union {
+    uint64_t ui;
+    double   d;
+  } v = {.d = val};
+
+  return tEncodeU64(pEncoder, v.ui);
+}
+
+static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) {
+  if (tEncodeU64v(pEncoder, len) < 0) return -1;
+  if (pEncoder->data) {
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1;
+    memcpy(TD_CODER_CURRENT(pEncoder), val, len);
+  }
+
+  TD_CODER_MOVE_POS(pEncoder, len);
   return 0;
 }
 
-static FORCE_INLINE int tEncodeDouble(SEncoder* pEncoder, double val) {
+static FORCE_INLINE int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) {
-  // TODO
+  return tEncodeBinary(pEncoder, (void*)val, len + 1);
-  return 0;
 }
 
-static FORCE_INLINE int tEncodeCStr(SEncoder* pEncoder, const char* val) {
+static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) {
-  // TODO
+  return tEncodeCStrWithLen(pEncoder, val, (uint64_t)strlen(val));
-  return 0;
 }
 
 /* ------------------------ FOR DECODER ------------------------ */
-static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int32_t size) {
-  ASSERT(!TD_IS_NULL(data));
-  pDecoder->endian = endian;
-  pDecoder->data = data;
-  pDecoder->size = size;
-  pDecoder->pos = 0;
-}
-
 // 8
-static FORCE_INLINE int tDecodeU8(SDecoder* pDecoder, uint8_t* val) {
+static FORCE_INLINE int tDecodeU8(SCoder* pDecoder, uint8_t* val) {
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
   tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val);
   TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
   return 0;
 }
 
-static FORCE_INLINE int tDecodeI8(SDecoder* pDecoder, int8_t* val) {
+static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) {
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
   tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val);
   TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
   return 0;
 }
 
 // 16
-static FORCE_INLINE int tDecodeU16(SDecoder* pDecoder, uint16_t* val) {
+static FORCE_INLINE int tDecodeU16(SCoder* pDecoder, uint16_t* val) { TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); }
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+static FORCE_INLINE int tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); }
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(uint16_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet16(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
-
-static FORCE_INLINE int tDecodeI16(SDecoder* pDecoder, int16_t* val) {
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(int16_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet16(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
-
 // 32
-static FORCE_INLINE int tDecodeU32(SDecoder* pDecoder, uint32_t* val) {
+static FORCE_INLINE int tDecodeU32(SCoder* pDecoder, uint32_t* val) { TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); }
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+static FORCE_INLINE int tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); }
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(uint32_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet32(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
-
-static FORCE_INLINE int tDecodeI32(SDecoder* pDecoder, int32_t* val) {
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(int32_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet32(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
-
 // 64
-static FORCE_INLINE int tDecodeU64(SDecoder* pDecoder, uint64_t* val) {
+static FORCE_INLINE int tDecodeU64(SCoder* pDecoder, uint64_t* val) { TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); }
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
+static FORCE_INLINE int tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); }
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(uint64_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet64(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
-
-static FORCE_INLINE int tDecodeI64(SDecoder* pDecoder, int64_t* val) {
-  if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1;
-  if (TD_RT_ENDIAN() == pDecoder->endian) {
-    tGet(int64_t, TD_CODER_CURRENT(pDecoder), *val);
-  } else {
-    tRGet64(val, TD_CODER_CURRENT(pDecoder));
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, sizeof(*val));
-  return 0;
-}
 
 // 16v
-static FORCE_INLINE int tDecodeU16v(SDecoder* pDecoder, uint16_t* val) {
+static FORCE_INLINE int tDecodeU16v(SCoder* pDecoder, uint16_t* val) {
-  int64_t i = 0;
+  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint16_t);
-  *val = 0;
-  for (;;) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
-    uint16_t tval = TD_CODER_CURRENT(pDecoder)[i];
-    if (tval < ENCODE_LIMIT) {
-      (*val) |= (tval << (7 * i));
-      break;
-    } else {
-      (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
-      i++;
-    }
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, i);
-
-  return 0;
 }
 
-static FORCE_INLINE int tDecodeI16v(SDecoder* pDecoder, int16_t* val) {
+static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) {
   uint16_t tval;
   if (tDecodeU16v(pDecoder, &tval) < 0) {
     return -1;
@@ -395,27 +311,11 @@ static FORCE_INLINE int tDecodeI16v(SDecoder* pDecoder, int16_t* val) {
 }
 
 // 32v
-static FORCE_INLINE int tDecodeU32v(SDecoder* pDecoder, uint32_t* val) {
+static FORCE_INLINE int tDecodeU32v(SCoder* pDecoder, uint32_t* val) {
-  int64_t i = 0;
+  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint32_t);
-  *val = 0;
-  for (;;) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
-    uint32_t tval = TD_CODER_CURRENT(pDecoder)[i];
-    if (tval < ENCODE_LIMIT) {
-      (*val) |= (tval << (7 * i));
-      break;
-    } else {
-      (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
-      i++;
-    }
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, i);
-
-  return 0;
 }
 
-static FORCE_INLINE int tDecodeI32v(SDecoder* pDecoder, int32_t* val) {
+static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) {
   uint32_t tval;
   if (tDecodeU32v(pDecoder, &tval) < 0) {
     return -1;
@@ -425,27 +325,11 @@ static FORCE_INLINE int tDecodeI32v(SDecoder* pDecoder, int32_t* val) {
 }
 
 // 64v
-static FORCE_INLINE int tDecodeU64v(SDecoder* pDecoder, uint64_t* val) {
+static FORCE_INLINE int tDecodeU64v(SCoder* pDecoder, uint64_t* val) {
-  int64_t i = 0;
+  TD_DECODE_VARIANT_MACRO(pDecoder, val, uint64_t);
-  *val = 0;
-  for (;;) {
-    if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1;
-    uint64_t tval = TD_CODER_CURRENT(pDecoder)[i];
-    if (tval < ENCODE_LIMIT) {
-      (*val) |= (tval << (7 * i));
-      break;
-    } else {
-      (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i));
-      i++;
-    }
-  }
-
-  TD_CODER_MOVE_POS(pDecoder, i);
-
-  return 0;
 }
 
-static FORCE_INLINE int tDecodeI64v(SDecoder* pDecoder, int64_t* val) {
+static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) {
   uint64_t tval;
   if (tDecodeU64v(pDecoder, &tval) < 0) {
     return -1;
@@ -454,21 +338,66 @@ static FORCE_INLINE int tDecodeI64v(SDecoder* pDecoder, int64_t* val) {
   return 0;
 }
 
-static FORCE_INLINE int tDecodeFloat(SDecoder* pDecoder, float* val) {
+static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) {
-  // TODO
+  union {
+    uint32_t ui;
+    float    f;
+  } v;
+
+  if (tDecodeU32(pDecoder, &(v.ui)) < 0) {
+    return -1;
+  }
+
+  *val = v.f;
   return 0;
 }
 
-static FORCE_INLINE int tDecodeDouble(SDecoder* pDecoder, double* val) {
+static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) {
-  // TODO
+  union {
+    uint64_t ui;
+    double   d;
+  } v;
+
+  if (tDecodeU64(pDecoder, &(v.ui)) < 0) {
+    return -1;
+  }
+
+  *val = v.d;
   return 0;
 }
 
-static FORCE_INLINE int tDecodeCStr(SDecoder* pEncoder, const char** val) {
+static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) {
-  // TODO
+  if (tDecodeU64v(pDecoder, len) < 0) return -1;
+
+  if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
+  *val = (void*)TD_CODER_CURRENT(pDecoder);
+
+  TD_CODER_MOVE_POS(pDecoder, *len);
   return 0;
 }
 
+static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) {
+  if (tDecodeBinary(pDecoder, (const void**)val, len) < 0) return -1;
+  (*len) -= 1;
+  return 0;
+}
+
+static FORCE_INLINE int tDecodeCStr(SCoder* pDecoder, const char** val) {
+  uint64_t len;
+  return tDecodeCStrAndLen(pDecoder, val, &len);
+}
+
+static int tDecodeCStrTo(SCoder* pDecoder, char* val) {
+  const char* pStr;
+  uint64_t    len;
+  if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1;
+
+  memcpy(val, pStr, len + 1);
+  return 0;
+}
+
+static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); }
+
 #ifdef __cplusplus
 }
 #endif

include/util/freelist.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_FREELIST_H_
+#define _TD_UTIL_FREELIST_H_
+
+#include "os.h"
+#include "tlist.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct SFreeListNode {
+  TD_SLIST_NODE(SFreeListNode);
+  char payload[];
+};
+
+typedef TD_SLIST(SFreeListNode) SFreeList;
+
+#define TFL_MALLOC(SIZE, LIST)                                 \
+  ({                                                           \
+    void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \
+    if (ptr) {                                                 \
+      TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr);      \
+      ptr = ((struct SFreeListNode *)ptr)->payload;            \
+    }                                                          \
+    ptr;                                                       \
+  })
+
+#define tFreeListInit(pFL) TD_SLIST_INIT(pFL)
+
+static FORCE_INLINE void tFreeListClear(SFreeList *pFL) {
+  struct SFreeListNode *pNode;
+  for (;;) {
+    pNode = TD_SLIST_HEAD(pFL);
+    if (pNode == NULL) break;
+    TD_SLIST_POP(pFL);
+    free(pNode);
+  }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_FREELIST_H_*/

source/client/CMakeLists.txt

@@ -11,4 +11,6 @@ target_link_libraries(
     PRIVATE os util common transport parser planner catalog scheduler function qcom
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -146,6 +146,8 @@ int   taos_init();
 void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo);
 void  destroyTscObj(void*pObj);
 
+uint64_t generateRequestId();
+
 void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
 void  destroyRequest(SRequestObj* pRequest);
 

source/client/src/clientEnv.c

@@ -87,6 +87,7 @@ static void tscInitLogFile() {
   }
 }
 
+// todo close the transporter properly
 void closeTransporter(STscObj* pTscObj)  {
   if (pTscObj == NULL || pTscObj->pTransporter == NULL) {
     return;
@@ -166,8 +167,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
     return NULL;
   }
 
-  // TODO generated request uuid
+  pRequest->requestId  = generateRequestId();
-  pRequest->requestId  = 0;
   pRequest->metric.start = taosGetTimestampMs();
 
   pRequest->type       = type;
@@ -410,6 +410,36 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
   return 0;
 }
 
+/**
+ * The request id is an unsigned integer format of 64bit.
+ *+------------+-----+-----------+---------------+
+ *| uid|localIp| PId | timestamp | serial number |
+ *+------------+-----+-----------+---------------+
+ *| 16bit      |12bit|20bit      |16bit          |
+ *+------------+-----+-----------+---------------+
+ * @return
+ */
+static int32_t requestSerialId = 0;
+uint64_t generateRequestId() {
+  uint64_t hashId = 0;
+
+  char uid[64] = {0};
+  int32_t code = taosGetSystemUid(uid, tListLen(uid));
+  if (code != TSDB_CODE_SUCCESS) {
+    tscError("Failed to get the system uid to generated request id, reason:%s. use ip address instead", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+
+  } else {
+    hashId = MurmurHash3_32(uid, strlen(uid));
+  }
+
+  int64_t ts      = taosGetTimestampUs();
+  uint64_t pid    = taosGetPId();
+  int32_t val     = atomic_add_fetch_32(&requestSerialId, 1);
+
+  uint64_t id = ((hashId & 0xFFFF) << 48) | ((pid & 0x0FFF) << 36) | ((ts & 0xFFFFF) << 16) | (val & 0xFFFF);
+  return id;
+}
+
 #if 0
 #include "cJSON.h"
 static setConfRet taos_set_config_imp(const char *config){

source/client/src/clientImpl.c

@@ -198,13 +198,14 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
 
 int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) {
   pRequest->type = pQuery->type;
-  return qCreateQueryDag(pQuery, pDag);
+  return qCreateQueryDag(pQuery, pDag, pRequest->requestId);
 }
 
 int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) {
-  if (TSDB_SQL_INSERT == pRequest->type) {
+  if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
     return scheduleExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob, &pRequest->affectedRows);
   }
+
   return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob);
 }
 

source/client/src/clientMain.c

@@ -29,7 +29,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
 
 // this function may be called by user or system, or by both simultaneously.
 void taos_cleanup(void) {
-  tscDebug("start to cleanup client environment");
+  tscInfo("start to cleanup client environment");
 
   if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
     return;
@@ -47,6 +47,8 @@ void taos_cleanup(void) {
 
   rpcCleanup();
   taosCloseLog();
+
+  tscInfo("all local resources released");
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
@@ -140,7 +142,9 @@ TAOS_ROW taos_fetch_row(TAOS_RES *pRes) {
 
   SRequestObj *pRequest = (SRequestObj *) pRes;
   if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      pRequest->type == TSDB_SQL_INSERT || pRequest->code != TSDB_CODE_SUCCESS) {
+      pRequest->type == TSDB_SQL_INSERT ||
+      pRequest->code != TSDB_CODE_SUCCESS ||
+      taos_num_fields(pRes) == 0) {
     return NULL;
   }
 

source/client/test/clientTests.cpp

@@ -147,29 +147,29 @@ TEST(testCase, connect_Test) {
 //  taos_close(pConn);
 //}
 
-//TEST(testCase, create_db_Test) {
+TEST(testCase, create_db_Test) {
-//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  assert(pConn != NULL);
+  assert(pConn != NULL);
-//
+
-//  TAOS_RES* pRes = taos_query(pConn, "create database abc1");
+  TAOS_RES* pRes = taos_query(pConn, "create database abc1");
-//  if (taos_errno(pRes) != 0) {
+  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
+  }
-//
+
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  ASSERT_TRUE(pFields == NULL);
+  ASSERT_TRUE(pFields == NULL);
-//
+
-//  int32_t numOfFields = taos_num_fields(pRes);
+  int32_t numOfFields = taos_num_fields(pRes);
-//  ASSERT_EQ(numOfFields, 0);
+  ASSERT_EQ(numOfFields, 0);
-//
+
-//  taos_free_result(pRes);
+  taos_free_result(pRes);
-//
+
-//  pRes = taos_query(pConn, "create database abc1 vgroups 4");
+  pRes = taos_query(pConn, "create database abc1 vgroups 4");
-//  if (taos_errno(pRes) != 0) {
+  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
+  }
-//  taos_close(pConn);
+  taos_close(pConn);
-//}
+}
 //
 //TEST(testCase, create_dnode_Test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -249,37 +249,37 @@ TEST(testCase, connect_Test) {
 ////  taos_close(pConn);
 //}
 
-// TEST(testCase, create_stable_Test) {
+ TEST(testCase, create_stable_Test) {
-//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  assert(pConn != NULL);
+  assert(pConn != NULL);
-//
+
-//  TAOS_RES* pRes = taos_query(pConn, "create database abc1");
+  TAOS_RES* pRes = taos_query(pConn, "create database abc1");
-//  if (taos_errno(pRes) != 0) {
+  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
+  }
-//  taos_free_result(pRes);
+  taos_free_result(pRes);
-//
+
-//  pRes = taos_query(pConn, "use abc1");
+  pRes = taos_query(pConn, "use abc1");
-//  if (taos_errno(pRes) != 0) {
+  if (taos_errno(pRes) != 0) {
-//    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
-//  }
+  }
-//  taos_free_result(pRes);
+  taos_free_result(pRes);
-//
+
-//  pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
+  pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
-//  if (taos_errno(pRes) != 0) {
+  if (taos_errno(pRes) != 0) {
-//    printf("error in create stable, reason:%s\n", taos_errstr(pRes));
+    printf("error in create stable, reason:%s\n", taos_errstr(pRes));
-//  }
+  }
-//
+
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  ASSERT_TRUE(pFields == NULL);
+  ASSERT_TRUE(pFields == NULL);
-//
+
-//  int32_t numOfFields = taos_num_fields(pRes);
+  int32_t numOfFields = taos_num_fields(pRes);
-//  ASSERT_EQ(numOfFields, 0);
+  ASSERT_EQ(numOfFields, 0);
-//
+
-//  taos_free_result(pRes);
+  taos_free_result(pRes);
-//  taos_close(pConn);
+  taos_close(pConn);
-//}
+}
-//
+
 //TEST(testCase, create_table_Test) {
 //  //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  //  assert(pConn != NULL);
@@ -435,16 +435,52 @@ TEST(testCase, connect_Test) {
 //  taos_close(pConn);
 //}
 
-TEST(testCase, show_table_Test) {
+//TEST(testCase, show_table_Test) {
+//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+//  assert(pConn != NULL);
+//
+//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+//  taos_free_result(pRes);
+//
+//  pRes = taos_query(pConn, "show tables");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
+//    taos_free_result(pRes);
+//    ASSERT_TRUE(false);
+//  }
+//
+//  TAOS_ROW pRow = NULL;
+//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+//  int32_t numOfFields = taos_num_fields(pRes);
+//
+//  char str[512] = {0};
+//  while((pRow = taos_fetch_row(pRes)) != NULL) {
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    printf("%s\n", str);
+//  }
+//
+//  taos_free_result(pRes);
+//  taos_close(pConn);
+//}
+
+TEST(testCase, create_multiple_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-  assert(pConn != NULL);
+  ASSERT_NE(pConn, nullptr);
 
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "show tables");
+  pRes = taos_query(pConn, "create table t_2 using st1 tags(1)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    ASSERT_TRUE(false);
+  }
+
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "create table t_3 using st1 tags(2)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);
     ASSERT_TRUE(false);
   }
@@ -462,3 +498,51 @@ TEST(testCase, show_table_Test) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
+
+TEST(testCase, generated_request_id_test) {
+  uint64_t id0 = generateRequestId();
+
+  uint64_t id1 = generateRequestId();
+  uint64_t id2 = generateRequestId();
+  uint64_t id3 = generateRequestId();
+  uint64_t id4 = generateRequestId();
+
+  ASSERT_NE(id0, id1);
+  ASSERT_NE(id1, id2);
+  ASSERT_NE(id2, id3);
+  ASSERT_NE(id4, id3);
+  ASSERT_NE(id0, id2);
+  ASSERT_NE(id0, id4);
+  ASSERT_NE(id0, id3);
+
+//  SHashObj *phash = taosHashInit()
+}
+
+//TEST(testCase, projection_query_tables) {
+//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+//  ASSERT_EQ(pConn, nullptr);
+//
+//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+//  taos_free_result(pRes);
+//
+//  pRes = taos_query(pConn, "select * from t_2");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
+//    taos_free_result(pRes);
+//    ASSERT_TRUE(false);
+//  }
+//
+//  TAOS_ROW pRow = NULL;
+//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+//  int32_t numOfFields = taos_num_fields(pRes);
+//
+//  char str[512] = {0};
+//  while((pRow = taos_fetch_row(pRes)) != NULL) {
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    printf("%s\n", str);
+//  }
+//
+//  taos_free_result(pRes);
+//  taos_close(pConn);
+//}
+

source/common/CMakeLists.txt

@@ -12,4 +12,6 @@ target_link_libraries(
     INTERFACE api
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/common/src/tmsg.c

@@ -27,78 +27,7 @@
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
-static int  tmsgStartEncode(SMsgEncoder *pME);
+int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
-static void tmsgEndEncode(SMsgEncoder *pME);
-static int  tmsgStartDecode(SMsgDecoder *pMD);
-static void tmsgEndDecode(SMsgDecoder *pMD);
-
-/* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */
-void tmsgInitMsgEncoder(SMsgEncoder *pME, td_endian_t endian, uint8_t *data, int64_t size) {
-  tInitEncoder(&(pME->coder), endian, data, size);
-  TD_SLIST_INIT(&(pME->eStack));
-}
-
-void tmsgClearMsgEncoder(SMsgEncoder *pME) {
-  struct SMEListNode *pNode;
-  for (;;) {
-    pNode = TD_SLIST_HEAD(&(pME->eStack));
-    if (TD_IS_NULL(pNode)) break;
-    TD_SLIST_POP(&(pME->eStack));
-    free(pNode);
-  }
-}
-
-void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int64_t size) {
-  tInitDecoder(&pMD->coder, endian, data, size);
-  TD_SLIST_INIT(&(pMD->dStack));
-  TD_SLIST_INIT(&(pMD->freeList));
-}
-
-void tmsgClearMsgDecoder(SMsgDecoder *pMD) {
-  {
-    struct SMDFreeListNode *pNode;
-    for (;;) {
-      pNode = TD_SLIST_HEAD(&(pMD->freeList));
-      if (TD_IS_NULL(pNode)) break;
-      TD_SLIST_POP(&(pMD->freeList));
-      free(pNode);
-    }
-  }
-  {
-    struct SMDListNode *pNode;
-    for (;;) {
-      pNode = TD_SLIST_HEAD(&(pMD->dStack));
-      if (TD_IS_NULL(pNode)) break;
-      TD_SLIST_POP(&(pMD->dStack));
-      free(pNode);
-    }
-  }
-}
-
-/* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */
-int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) {
-  tmsgStartEncode(pCoder);
-  // TODO
-
-  tmsgEndEncode(pCoder);
-  return 0;
-}
-
-int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) {
-  tmsgStartDecode(pCoder);
-
-  // TODO: decode
-
-  // Decode is not end
-  if (pCoder->coder.pos != pCoder->coder.size) {
-    // Continue decode
-  }
-
-  tmsgEndDecode(pCoder);
-  return 0;
-}
-
-int tSerializeSVCreateTbReq(void **buf, const SVCreateTbReq *pReq) {
   int tlen = 0;
 
   tlen += taosEncodeFixedU64(buf, pReq->ver);
@@ -193,62 +122,30 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   return buf;
 }
 
-/* ------------------------ STATIC METHODS ------------------------ */
+int tSVCreateTbBatchReqSerialize(void **buf, SVCreateTbBatchReq *pReq) {
-static int tmsgStartEncode(SMsgEncoder *pME) {
+  int tlen = 0;
-  struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode));
-  if (TD_IS_NULL(pNode)) return -1;
 
-  pNode->coder = pME->coder;
+  tlen += taosEncodeFixedU64(buf, pReq->ver);
-  TD_SLIST_PUSH(&(pME->eStack), pNode);
+  tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
-  TD_CODER_MOVE_POS(&(pME->coder), sizeof(int32_t));
+  for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
+    SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
+    tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq);
+  }
 
-  return 0;
+  return tlen;
 }
 
-static void tmsgEndEncode(SMsgEncoder *pME) {
+void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) {
-  int32_t             size;
+  uint32_t nsize = 0;
-  struct SMEListNode *pNode;
 
-  pNode = TD_SLIST_HEAD(&(pME->eStack));
+  buf = taosDecodeFixedU64(buf, &pReq->ver);
-  ASSERT(pNode);
+  buf = taosDecodeFixedU32(buf, &nsize);
-  TD_SLIST_POP(&(pME->eStack));
+  pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
+  for (size_t i = 0; i < nsize; i++) {
+    SVCreateTbReq req;
+    buf = tDeserializeSVCreateTbReq(buf, &req);
+    taosArrayPush(pReq->pArray, &req);
+  }
 
-  size = pME->coder.pos - pNode->coder.pos;
+  return buf;
-  tEncodeI32(&(pNode->coder), size);
-
-  free(pNode);
-}
-
-static int tmsgStartDecode(SMsgDecoder *pMD) {
-  struct SMDListNode *pNode;
-  int32_t             size;
-
-  pNode = (struct SMDListNode *)malloc(sizeof(*pNode));
-  if (pNode == NULL) return -1;
-
-  tDecodeI32(&(pMD->coder), &size);
-
-  pNode->coder = pMD->coder;
-  TD_SLIST_PUSH(&(pMD->dStack), pNode);
-
-  pMD->coder.pos = 0;
-  pMD->coder.size = size - sizeof(int32_t);
-  pMD->coder.data = TD_CODER_CURRENT(&(pNode->coder));
-
-  return 0;
-}
-
-static void tmsgEndDecode(SMsgDecoder *pMD) {
-  ASSERT(pMD->coder.pos == pMD->coder.size);
-  struct SMDListNode *pNode;
-
-  pNode = TD_SLIST_HEAD(&(pMD->dStack));
-  ASSERT(pNode);
-  TD_SLIST_POP(&(pMD->dStack));
-
-  pNode->coder.pos += pMD->coder.size;
-
-  pMD->coder = pNode->coder;
-
-  free(pNode);
 }

source/dnode/mgmt/impl/src/dndTransport.c

@@ -121,8 +121,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg;
-  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeQueryMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg;
@@ -160,8 +160,8 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
 
   DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
   if (fp != NULL) {
+    dTrace("RPC %p, rsp:%s will be processed, code:0x%x", pMsg->handle, TMSG_INFO(msgType), pMsg->code & 0XFFFF);
     (*fp)(pDnode, pMsg, pEpSet);
-    dTrace("RPC %p, rsp:%s is processed, code:0x%x", pMsg->handle, TMSG_INFO(msgType), pMsg->code & 0XFFFF);
   } else {
     dError("RPC %p, rsp:%s not processed", pMsg->handle, TMSG_INFO(msgType));
     rpcFreeCont(pMsg->pCont);

source/dnode/mnode/impl/src/mndStb.c

@@ -232,7 +232,10 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb
   SMsgHead     *pMsgHead;
 
   req.ver = 0;
-  req.name = pStb->name;
+  SName name = {0};
+  tNameFromString(&name, pStb->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
+
+  req.name = (char*) tNameGetTableName(&name);
   req.ttl = 0;
   req.keep = 0;
   req.type = TD_SUPER_TABLE;

source/dnode/mnode/impl/test/user/user.cpp

@@ -43,6 +43,153 @@ TEST_F(DndTestUser, 01_ShowUser) {
   CheckBinary("root", TSDB_USER_LEN);
 }
 
+TEST_F(DndTestUser, 02_Create_User) {
+  {
+    int32_t contLen = sizeof(SCreateUserMsg);
+
+    SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "");
+    strcpy(pReq->pass, "p1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT);
+  }
+
+  {
+    int32_t contLen = sizeof(SCreateUserMsg);
+
+    SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u1");
+    strcpy(pReq->pass, "");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_PASS_FORMAT);
+  }
+
+  {
+    int32_t contLen = sizeof(SCreateUserMsg);
+
+    SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "root");
+    strcpy(pReq->pass, "1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_ALREADY_EXIST);
+  }
+
+  {
+    int32_t contLen = sizeof(SCreateUserMsg);
+
+    SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u1");
+    strcpy(pReq->pass, "p1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+
+  test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, "");
+  CHECK_META("show users", 4);
+
+  test.SendShowRetrieveMsg();
+  EXPECT_EQ(test.GetShowRows(), 2);
+}
+
+TEST_F(DndTestUser, 03_Alter_User) {
+  {
+    int32_t contLen = sizeof(SAlterUserMsg);
+
+    SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "");
+    strcpy(pReq->pass, "p1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT);
+  }
+
+  {
+    int32_t contLen = sizeof(SAlterUserMsg);
+
+    SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u1");
+    strcpy(pReq->pass, "");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_PASS_FORMAT);
+  }
+
+  {
+    int32_t contLen = sizeof(SAlterUserMsg);
+
+    SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u4");
+    strcpy(pReq->pass, "1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_NOT_EXIST);
+  }
+
+  {
+    int32_t contLen = sizeof(SAlterUserMsg);
+
+    SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u1");
+    strcpy(pReq->pass, "1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+}
+
+TEST_F(DndTestUser, 04_Drop_User) {
+  {
+    int32_t contLen = sizeof(SDropUserMsg);
+
+    SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT);
+  }
+
+  {
+    int32_t contLen = sizeof(SDropUserMsg);
+
+    SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u4");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_NOT_EXIST);
+  }
+
+  {
+    int32_t contLen = sizeof(SDropUserMsg);
+
+    SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->user, "u1");
+
+    SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen);
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+
+  test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, "");
+  CHECK_META("show users", 4);
+
+  test.SendShowRetrieveMsg();
+  EXPECT_EQ(test.GetShowRows(), 1);
+}
+
 TEST_F(DndTestUser, 02_Create_Drop_Alter_User) {
   {
     int32_t contLen = sizeof(SCreateUserMsg);
@@ -161,4 +308,4 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) {
   CheckTimestamp();
   CheckBinary("root", TSDB_USER_LEN);
   CheckBinary("root", TSDB_USER_LEN);
-}
+}

source/dnode/vnode/impl/src/vnodeQuery.c

@@ -17,6 +17,7 @@
 #include "vnodeDef.h"
 
 static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg);
+static int     vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
 
 int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); }
 
@@ -42,7 +43,9 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       return qWorkerProcessShowMsg(pVnode, pVnode->pQuery, pMsg);
     case TDMT_VND_SHOW_TABLES_FETCH:
       return vnodeGetTableList(pVnode, pMsg);
-//      return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg);
+      //      return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg);
+    case TDMT_VND_TABLE_META:
+      return vnodeGetTableMeta(pVnode, pMsg, pRsp);
     default:
       vError("unknown msg type:%d in fetch queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;
@@ -57,18 +60,21 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   int32_t         nCols;
   int32_t         nTagCols;
   SSchemaWrapper *pSW;
-  STableMetaMsg * pTbMetaMsg;
+  STableMetaMsg * pTbMetaMsg = NULL;
   SSchema *       pTagSchema;
+  SRpcMsg         rpcMsg;
+  int             msgLen = 0;
+  int32_t         code = TSDB_CODE_VND_APP_ERROR;
 
   pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid);
   if (pTbCfg == NULL) {
-    return -1;
+    goto _exit;
   }
 
   if (pTbCfg->type == META_CHILD_TABLE) {
     pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid);
     if (pStbCfg == NULL) {
-      return -1;
+      goto _exit;
     }
 
     pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true);
@@ -88,11 +94,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
     pTagSchema = NULL;
   }
 
-  pTbMetaMsg = (STableMetaMsg *)calloc(1, sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols));
+  msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols);
+  pTbMetaMsg = (STableMetaMsg *)rpcMallocCont(msgLen);
   if (pTbMetaMsg == NULL) {
-    return -1;
+    goto _exit;
   }
 
+  memcpy(pTbMetaMsg->dbFname, pReq->dbFname, sizeof(pTbMetaMsg->dbFname));
   strcpy(pTbMetaMsg->tbFname, pTbCfg->name);
   if (pTbCfg->type == META_CHILD_TABLE) {
     strcpy(pTbMetaMsg->stbFname, pStbCfg->name);
@@ -115,6 +123,18 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
     pSch->bytes = htonl(pSch->bytes);
   }
 
+  code = 0;
+
+_exit:
+
+  rpcMsg.handle = pMsg->handle;
+  rpcMsg.ahandle = pMsg->ahandle;
+  rpcMsg.pCont = pTbMetaMsg;
+  rpcMsg.contLen = msgLen;
+  rpcMsg.code = code;
+
+  rpcSendResponse(&rpcMsg);
+
   return 0;
 }
 
@@ -124,10 +144,10 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
  * @param pRsp
  */
 static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
-  SMTbCursor* pCur = metaOpenTbCursor(pVnode->pMeta);
+  SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta);
-  SArray* pArray = taosArrayInit(10, POINTER_BYTES);
+  SArray *    pArray = taosArrayInit(10, POINTER_BYTES);
 
-  char* name = NULL;
+  char *  name = NULL;
   int32_t totalLen = 0;
   while ((name = metaTbCursorNext(pCur)) != NULL) {
     taosArrayPush(pArray, &name);
@@ -136,34 +156,35 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
 
   metaCloseTbCursor(pCur);
 
-  int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4;
+  int32_t rowLen =
-  int32_t numOfTables = (int32_t) taosArrayGetSize(pArray);
+      (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4;
+  int32_t numOfTables = (int32_t)taosArrayGetSize(pArray);
 
   int32_t payloadLen = rowLen * numOfTables;
-//  SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
+  //  SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
 
   SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp) + payloadLen);
   memset(pFetchRsp, 0, sizeof(struct SVShowTablesFetchRsp) + payloadLen);
 
-  char* p = pFetchRsp->data;
+  char *p = pFetchRsp->data;
-  for(int32_t i = 0; i < numOfTables; ++i) {
+  for (int32_t i = 0; i < numOfTables; ++i) {
-    char* n = taosArrayGetP(pArray, i);
+    char *n = taosArrayGetP(pArray, i);
     STR_TO_VARSTR(p, n);
 
-    p += rowLen;
+    p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
   }
 
   pFetchRsp->numOfRows = htonl(numOfTables);
   pFetchRsp->precision = 0;
 
   SRpcMsg rpcMsg = {
-      .handle  = pMsg->handle,
+      .handle = pMsg->handle,
       .ahandle = pMsg->ahandle,
-      .pCont   = pFetchRsp,
+      .pCont = pFetchRsp,
       .contLen = sizeof(SVShowTablesFetchRsp) + payloadLen,
-      .code    = 0,
+      .code = 0,
   };
 
   rpcSendResponse(&rpcMsg);
   return 0;
-}
+}

source/dnode/vnode/impl/src/vnodeWrite.c

@@ -27,7 +27,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
 }
 
 int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
-  SRpcMsg *  pMsg;
+  SRpcMsg *pMsg;
 
   for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
     pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
@@ -50,8 +50,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
 }
 
 int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
-  SVCreateTbReq vCreateTbReq;
+  SVCreateTbReq      vCreateTbReq;
-  void *        ptr = vnodeMalloc(pVnode, pMsg->contLen);
+  SVCreateTbBatchReq vCreateTbBatchReq;
+  void *             ptr = vnodeMalloc(pVnode, pMsg->contLen);
   if (ptr == NULL) {
     // TODO: handle error
   }
@@ -68,7 +69,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
   switch (pMsg->msgType) {
     case TDMT_VND_CREATE_STB:
-    case TDMT_VND_CREATE_TABLE:
       tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq);
       if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) {
         // TODO: handle error
@@ -76,6 +76,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
       // TODO: maybe need to clear the requst struct
       break;
+    case TDMT_VND_CREATE_TABLE:
+      tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq);
+      for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) {
+        SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
+        if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
+          // TODO: handle error
+        }
+      }
+      
     case TDMT_VND_DROP_STB:
     case TDMT_VND_DROP_TABLE:
       // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) {

source/dnode/vnode/meta/inc/metaTbCfg.h

@@ -22,10 +22,6 @@
 extern "C" {
 #endif
 
-#define META_SUPER_TABLE TD_SUPER_TABLE
-#define META_CHILD_TABLE TD_CHILD_TABLE
-#define META_NORMAL_TABLE TD_NORMAL_TABLE
-
 int    metaValidateTbCfg(SMeta *pMeta, const STbCfg *);
 size_t metaEncodeTbObjFromTbOptions(const STbCfg *, void *pBuf, size_t bsize);
 

source/dnode/vnode/meta/src/metaBDBImpl.c

@@ -23,6 +23,7 @@
 typedef struct {
   tb_uid_t uid;
   int32_t  sver;
+  int32_t  padding;
 } SSchemaKey;
 
 struct SMetaDB {
@@ -55,6 +56,8 @@ static int      metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *
 static int      metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
 static void *   metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static void     metaClearTbCfg(STbCfg *pTbCfg);
+static int      metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
+static void *   metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
 
 #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code))
 
@@ -169,18 +172,13 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     pBuf = buf;
     memset(&key, 0, sizeof(key));
     memset(&value, 0, sizeof(key));
-    SSchemaKey schemaKey = {uid, 0 /*TODO*/};
+    SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0};
 
     key.data = &schemaKey;
     key.size = sizeof(schemaKey);
 
-    taosEncodeFixedU32(&pBuf, ncols);
+    SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema};
-    for (size_t i = 0; i < ncols; i++) {
+    metaEncodeSchema(&pBuf, &sw);
-      taosEncodeFixedI8(&pBuf, pSchema[i].type);
-      taosEncodeFixedI32(&pBuf, pSchema[i].colId);
-      taosEncodeFixedI32(&pBuf, pSchema[i].bytes);
-      taosEncodeString(&pBuf, pSchema[i].name);
-    }
 
     value.data = buf;
     value.size = POINTER_DISTANCE(pBuf, buf);
@@ -197,6 +195,38 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
 }
 
 /* ------------------------ STATIC METHODS ------------------------ */
+static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
+  int      tlen = 0;
+  SSchema *pSchema;
+
+  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  for (int i = 0; i < pSW->nCols; i++) {
+    pSchema = pSW->pSchema + i;
+    tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI32(buf, pSchema->colId);
+    tlen += taosEncodeFixedI32(buf, pSchema->bytes);
+    tlen += taosEncodeString(buf, pSchema->name);
+  }
+
+  return tlen;
+}
+
+static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
+  SSchema *pSchema;
+
+  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * pSW->nCols);
+  for (int i = 0; i < pSW->nCols; i++) {
+    pSchema = pSW->pSchema + i;
+    buf = taosDecodeFixedI8(buf, &pSchema->type);
+    buf = taosDecodeFixedI32(buf, &pSchema->colId);
+    buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+    buf = taosDecodeStringTo(buf, pSchema->name);
+  }
+
+  return buf;
+}
+
 static SMetaDB *metaNewDB() {
   SMetaDB *pDB = NULL;
   pDB = (SMetaDB *)calloc(1, sizeof(*pDB));
@@ -376,15 +406,8 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
   tsize += taosEncodeFixedU8(buf, pTbCfg->type);
 
   if (pTbCfg->type == META_SUPER_TABLE) {
-    tsize += taosEncodeVariantU32(buf, pTbCfg->stbCfg.nTagCols);
+    SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema};
-    for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) {
+    tsize += metaEncodeSchema(buf, &sw);
-      tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pTagSchema[i].type);
-      tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].colId);
-      tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].bytes);
-      tsize += taosEncodeString(buf, pTbCfg->stbCfg.pTagSchema[i].name);
-    }
-
-    // tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema);
   } else if (pTbCfg->type == META_CHILD_TABLE) {
     tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid);
     tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag);
@@ -403,14 +426,10 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
   buf = taosDecodeFixedU8(buf, &(pTbCfg->type));
 
   if (pTbCfg->type == META_SUPER_TABLE) {
-    buf = taosDecodeVariantU32(buf, &(pTbCfg->stbCfg.nTagCols));
+    SSchemaWrapper sw;
-    pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols);
+    buf = metaDecodeSchema(buf, &sw);
-    for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) {
+    pTbCfg->stbCfg.nTagCols = sw.nCols;
-      buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pTagSchema[i].type));
+    pTbCfg->stbCfg.pTagSchema = sw.pSchema;
-      buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].colId);
-      buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].bytes);
-      buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pTagSchema[i].name);
-    }
   } else if (pTbCfg->type == META_CHILD_TABLE) {
     buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid));
     buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag));
@@ -496,7 +515,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   int             ret;
   void *          pBuf;
   SSchema *       pSchema;
-  SSchemaKey      schemaKey = {uid, sver};
+  SSchemaKey      schemaKey = {uid, sver, 0};
   DBT             key = {0};
   DBT             value = {0};
 
@@ -507,38 +526,14 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   // Query
   ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0);
   if (ret != 0) {
+    printf("failed to query schema DB since %s================\n", db_strerror(ret));
     return NULL;
   }
 
   // Decode the schema
   pBuf = value.data;
-  taosDecodeFixedI32(&pBuf, &nCols);
+  pSW = malloc(sizeof(*pSW));
-  if (isinline) {
+  metaDecodeSchema(pBuf, pSW);
-    pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols);
-    if (pSW == NULL) {
-      return NULL;
-    }
-    pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW));
-  } else {
-    pSW = (SSchemaWrapper *)malloc(sizeof(*pSW));
-    if (pSW == NULL) {
-      return NULL;
-    }
-
-    pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * nCols);
-    if (pSW->pSchema == NULL) {
-      free(pSW);
-      return NULL;
-    }
-  }
-
-  for (int i = 0; i < nCols; i++) {
-    pSchema = pSW->pSchema + i;
-    taosDecodeFixedI8(&pBuf, &(pSchema->type));
-    taosDecodeFixedI32(&pBuf, &(pSchema->colId));
-    taosDecodeFixedI32(&pBuf, &(pSchema->bytes));
-    taosDecodeStringTo(&pBuf, pSchema->name);
-  }
 
   return pSW;
 }

source/libs/catalog/CMakeLists.txt

@@ -11,4 +11,6 @@ target_link_libraries(
    PRIVATE os util transport qcom
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/catalog/inc/catalogInt.h

@@ -77,27 +77,37 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 #define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
 #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
+#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
+
 #define CTG_LOCK(type, _lock) do {   \
   if (CTG_READ == (type)) {          \
-    if ((*(_lock)) < 0) assert(0);  \
+    assert(atomic_load_32((_lock)) >= 0);  \
+    ctgDebug("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosRLockLatch(_lock);           \
-    ctgDebug("CTG RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
+    ctgDebug("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_32((_lock)) > 0);  \
   } else {                                                \
-    if ((*(_lock)) < 0) assert(0);                          \
+    assert(atomic_load_32((_lock)) >= 0);  \
+    ctgDebug("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
     taosWLockLatch(_lock);                                \
-    ctgDebug("CTG WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__);  \
+    ctgDebug("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__);  \
+    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
   }                                                       \
 } while (0)
 
 #define CTG_UNLOCK(type, _lock) do {                       \
   if (CTG_READ == (type)) {                                \
-    if ((*(_lock)) <= 0) assert(0);                         \
+    assert(atomic_load_32((_lock)) > 0);  \
+    ctgDebug("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosRUnLockLatch(_lock);                              \
-    ctgDebug("CTG RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
+    ctgDebug("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_32((_lock)) >= 0);  \
   } else {                                                \
-    if ((*(_lock)) <= 0) assert(0);                         \
+    assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY);  \
+    ctgDebug("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
     taosWUnLockLatch(_lock);                              \
-    ctgDebug("CTG WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
+    ctgDebug("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \
+    assert(atomic_load_32((_lock)) >= 0);  \
   }                                                       \
 } while (0)
 

source/libs/catalog/src/catalog.c

@@ -23,21 +23,31 @@ SCatalogMgmt ctgMgmt = {0};
 int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) {
   if (NULL == pCatalog->dbCache.cache) {
     *inCache = false;
+    ctgWarn("no db cache");
     return TSDB_CODE_SUCCESS;
   }
 
-  SDBVgroupInfo *info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName));
+  SDBVgroupInfo *info = NULL;
 
-  if (NULL == info) {
+  while (true) {
-    *inCache = false;
+    info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName));
-    return TSDB_CODE_SUCCESS;
-  }
 
-  CTG_LOCK(CTG_READ, &info->lock);
+    if (NULL == info) {
-  if (NULL == info->vgInfo) {
+      *inCache = false;
-    CTG_UNLOCK(CTG_READ, &info->lock);
+      ctgWarn("no db cache, dbName:%s", dbName);
-    *inCache = false;
+      return TSDB_CODE_SUCCESS;
-    return TSDB_CODE_SUCCESS;
+    }
+
+    CTG_LOCK(CTG_READ, &info->lock);
+    if (NULL == info->vgInfo) {
+      CTG_UNLOCK(CTG_READ, &info->lock);
+      taosHashRelease(pCatalog->dbCache.cache, info);
+      ctgWarn("db cache vgInfo is NULL, dbName:%s", dbName);
+      
+      continue;
+    }
+
+    break;
   }
 
   *dbInfo = info;
@@ -151,7 +161,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
   char tbFullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pTableName, tbFullName);
 
-  SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName};
+  SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName};
   char *msg = NULL;
   SEpSet *pVnodeEpSet = NULL;
   int32_t msgLen = 0;
@@ -179,16 +189,15 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
 }
 
 
-int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
+int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
-  if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
+  if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
     CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
-  char tbFullName[TSDB_TABLE_FNAME_LEN];
+  char dbFullName[TSDB_DB_FNAME_LEN];
+  tNameGetFullDbName(pTableName, dbFullName);
 
-  snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
+  SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = pTableName->tname};
-
-  SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName};
   char *msg = NULL;
   SEpSet *pVnodeEpSet = NULL;
   int32_t msgLen = 0;
@@ -271,8 +280,6 @@ _return:
 int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) {
   int32_t code = 0;
   
-  CTG_LOCK(CTG_READ, &dbInfo->lock);
-  
   int32_t vgNum = taosHashGetSize(dbInfo->vgInfo);
   char db[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(pTableName, db);
@@ -311,8 +318,6 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const SName *pTableName
   *pVgroup = *vgInfo;
 
 _return:
-
-  CTG_UNLOCK(CTG_READ, &dbInfo->lock);
   
   CTG_RET(TSDB_CODE_SUCCESS);
 }
@@ -350,19 +355,19 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
   
   if (output->metaNum != 1 && output->metaNum != 2) {
     ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum);
-    CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
+    CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
   if (NULL == output->tbMeta) {
     ctgError("no valid table meta got from meta rsp");
-    CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
+    CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
   if (NULL == pCatalog->tableCache.cache) {
     pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
     if (NULL == pCatalog->tableCache.cache) {
       ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
   }
 
@@ -370,19 +375,19 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
     pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
     if (NULL == pCatalog->tableCache.stableCache) {
       ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
   }
 
   if (output->metaNum == 2) {
     if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) {
       ctgError("push ctable[%s] to table cache failed", output->ctbFname);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
 
     if (TSDB_SUPER_TABLE != output->tbMeta->tableType) {
       ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE);
-      CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
     }    
   }
 
@@ -393,26 +398,23 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
     if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) {
       CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock);
       ctgError("push table[%s] to table cache failed", output->tbFname);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
 
     STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname));
     if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES) != 0) {
       CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock);
       ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
     CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock);
   } else {
     if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) {
       ctgError("push table[%s] to table cache failed", output->tbFname);
-      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
     }
   }
 
-_return:
-  tfree(output->tbMeta);
-  
   CTG_RET(code);
 }
 
@@ -434,11 +436,47 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm
   input.db[sizeof(input.db) - 1] = 0;
   input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
 
-  CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
+  while (true) {
+    CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
 
-  CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup));
+    CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup));
 
-  CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache));
+    CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache));
+
+    if (!inCache) {
+      ctgWarn("get db vgroup from cache failed, db:%s", dbName);
+      continue;
+    }
+
+    break;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
+  SDBVgroupInfo *oldInfo = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName));
+  if (oldInfo) {
+    CTG_LOCK(CTG_WRITE, &oldInfo->lock);
+    if (dbInfo->vgVersion <= oldInfo->vgVersion) {
+      ctgInfo("dbName:%s vg will not update, vgVersion:%d , current:%d", dbName, dbInfo->vgVersion, oldInfo->vgVersion);
+      CTG_UNLOCK(CTG_WRITE, &oldInfo->lock);
+      taosHashRelease(pCatalog->dbCache.cache, oldInfo);
+      
+      return TSDB_CODE_SUCCESS;
+    }
+    
+    if (oldInfo->vgInfo) {
+      ctgInfo("dbName:%s vg will be cleanup", dbName);
+      taosHashCleanup(oldInfo->vgInfo);
+      oldInfo->vgInfo = NULL;
+    }
+    
+    CTG_UNLOCK(CTG_WRITE, &oldInfo->lock);
+  
+    taosHashRelease(pCatalog->dbCache.cache, oldInfo);
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -580,55 +618,57 @@ _return:
 
 
 int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
+  int32_t code = 0;
+  
   if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) {
-    CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
+    CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT);
+  }
+
+  if (NULL == dbInfo->vgInfo || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgInfo) <= 0) {
+    ctgError("invalid db vg, dbName:%s", dbName);
+    CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
   if (dbInfo->vgVersion < 0) {
-    if (pCatalog->dbCache.cache) {
+    ctgWarn("invalid db vgVersion:%d, dbName:%s", dbInfo->vgVersion, dbName);
-      SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName));
-      if (oldInfo) {
-        CTG_LOCK(CTG_WRITE, &oldInfo->lock);
-        if (oldInfo->vgInfo) {
-          taosHashCleanup(oldInfo->vgInfo);
-          oldInfo->vgInfo = NULL;
-        }
-        CTG_UNLOCK(CTG_WRITE, &oldInfo->lock);
 
-        taosHashRelease(pCatalog->dbCache.cache, oldInfo);
+    if (pCatalog->dbCache.cache) {
-      }
+      CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo));
+      
+      CTG_ERR_JRET(taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)));
     }
     
     ctgWarn("remove db [%s] from cache", dbName);
-    return TSDB_CODE_SUCCESS;
+    goto _return;
   }
 
   if (NULL == pCatalog->dbCache.cache) {
     pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
     if (NULL == pCatalog->dbCache.cache) {
       ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
-      CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
+      CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
     }
   } else {
-    SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName));
+    CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo));
-    if (oldInfo) {
-      CTG_LOCK(CTG_WRITE, &oldInfo->lock);
-      if (oldInfo->vgInfo) {
-        taosHashCleanup(oldInfo->vgInfo);
-        oldInfo->vgInfo = NULL;
-      }
-      CTG_UNLOCK(CTG_WRITE, &oldInfo->lock);
-    
-      taosHashRelease(pCatalog->dbCache.cache, oldInfo);
-    }
   }
 
   if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) {
     ctgError("push to vgroup hash cache failed");
-    CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
+    CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
-  return TSDB_CODE_SUCCESS;
+  ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, dbInfo->vgVersion);
+
+  dbInfo->vgInfo = NULL;
+
+_return:
+
+  if (dbInfo && dbInfo->vgInfo) {
+    taosHashCleanup(dbInfo->vgInfo);
+    dbInfo->vgInfo = NULL;
+  }
+  
+  CTG_RET(code);
 }
 
 int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
@@ -647,9 +687,9 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSe
 
   STableMetaOutput output = {0};
   
-  //CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
+  CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pTableName, &vgroupInfo, &output));
 
-  CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output));
+  //CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output));
 
   CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output));
 

source/libs/catalog/test/catalogTests.cpp

@@ -36,11 +36,19 @@
 
 namespace {
 
+extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist);
+extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output);
+
 void ctgTestSetPrepareTableMeta();
 void ctgTestSetPrepareCTableMeta();
 void ctgTestSetPrepareSTableMeta();
 
+bool ctgTestStop = false;
+bool ctgTestEnableSleep = false;
+bool ctgTestDeadLoop = true;
 
+int32_t ctgTestCurrentVgVersion = 0;
+int32_t ctgTestVgVersion = 1;
 int32_t ctgTestVgNum = 10;
 int32_t ctgTestColNum = 2;
 int32_t ctgTestTagNum = 1;
@@ -89,6 +97,113 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
   ASSERT_EQ(rpcRsp.code, 0);
 }
 
+void ctgTestInitLogFile() {
+  const char    *defaultLogFileNamePrefix = "taoslog";
+  const int32_t  maxLogFileNum = 10;
+
+  ctgDebugFlag = 159;
+  tsAsyncLog = 0;
+
+  char temp[128] = {0};
+  sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
+  if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
+    printf("failed to open log file in directory:%s\n", tsLogDir);
+  }
+
+}
+
+int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) {
+  return ((vgVersion % 2) == 0) ? ctgTestVgNum - 2 : ctgTestVgNum;
+}
+
+void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
+  SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(cn.dbname, "db1");
+  strcpy(cn.tname, ctgTestCTablename);
+
+  SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(sn.dbname, "db1");
+  strcpy(sn.tname, ctgTestSTablename);
+
+  char tbFullName[TSDB_TABLE_FNAME_LEN];
+  tNameExtractFullName(&cn, tbFullName);
+
+  output->metaNum = 2;
+
+  strcpy(output->ctbFname, tbFullName);
+
+  tNameExtractFullName(&cn, tbFullName);
+  strcpy(output->tbFname, tbFullName);
+
+  output->ctbMeta.vgId = 9;
+  output->ctbMeta.tableType = TSDB_CHILD_TABLE;
+  output->ctbMeta.uid = 3;
+  output->ctbMeta.suid = 2;
+
+  output->tbMeta = (STableMeta *)calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum));
+  output->tbMeta->vgId = 9;
+  output->tbMeta->tableType = TSDB_SUPER_TABLE;
+  output->tbMeta->uid = 2;
+  output->tbMeta->suid = 2;
+
+  output->tbMeta->tableInfo.numOfColumns = ctgTestColNum;
+  output->tbMeta->tableInfo.numOfTags = ctgTestTagNum;
+  
+  output->tbMeta->sversion = ctgTestSVersion;
+  output->tbMeta->tversion = ctgTestTVersion;
+  
+  SSchema *s = NULL;
+  s = &output->tbMeta->schema[0];
+  s->type = TSDB_DATA_TYPE_TIMESTAMP;
+  s->colId = 1;
+  s->bytes = 8;
+  strcpy(s->name, "ts");
+
+  s = &output->tbMeta->schema[1];
+  s->type = TSDB_DATA_TYPE_INT;
+  s->colId = 2;
+  s->bytes = 4;
+  strcpy(s->name, "col1s");
+
+  s = &output->tbMeta->schema[2];
+  s->type = TSDB_DATA_TYPE_BINARY;
+  s->colId = 3;
+  s->bytes = 12;
+  strcpy(s->name, "tag1s");
+  
+}
+
+void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) {
+  static int32_t vgVersion = ctgTestVgVersion + 1;
+  int32_t vgNum = 0;
+  SVgroupInfo vgInfo = {0};
+  
+  dbVgroup->vgVersion = vgVersion++;
+  
+  ctgTestCurrentVgVersion = dbVgroup->vgVersion;
+  
+  dbVgroup->hashMethod = 0;
+  dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+
+  vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion);
+  uint32_t hashUnit = UINT32_MAX / vgNum;
+
+  for (int32_t i = 0; i < vgNum; ++i) {
+    vgInfo.vgId = i + 1;
+    vgInfo.hashBegin = i * hashUnit;
+    vgInfo.hashEnd = hashUnit * (i + 1) - 1;
+    vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1;
+    vgInfo.inUse = i % vgInfo.numOfEps;
+    for (int32_t n = 0; n < vgInfo.numOfEps; ++n) {
+      SEpAddrMsg *addr = &vgInfo.epAddr[n];
+      strcpy(addr->fqdn, "a0");
+      addr->port = htons(n + 22);
+    }
+
+    taosHashPut(dbVgroup->vgInfo, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo));
+  }
+}
+
 void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   SUseDbRsp *rspMsg = NULL; //todo
 
@@ -97,7 +212,8 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
   pRsp->pCont = calloc(1, pRsp->contLen);
   rspMsg = (SUseDbRsp *)pRsp->pCont;
   strcpy(rspMsg->db, ctgTestDbname);
-  rspMsg->vgVersion = htonl(1);
+  rspMsg->vgVersion = htonl(ctgTestVgVersion);
+  ctgTestCurrentVgVersion = ctgTestVgVersion;
   rspMsg->vgNum = htonl(ctgTestVgNum);
   rspMsg->hashMethod = 0;
 
@@ -148,13 +264,13 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM
   SSchema *s = NULL;
   s = &rspMsg->pSchema[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(0);
+  s->colId = htonl(1);
   s->bytes = htonl(8);
   strcpy(s->name, "ts");
 
   s = &rspMsg->pSchema[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(1);
+  s->colId = htonl(2);
   s->bytes = htonl(4);
   strcpy(s->name, "col1");
   
@@ -185,19 +301,19 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
   SSchema *s = NULL;
   s = &rspMsg->pSchema[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(0);
+  s->colId = htonl(1);
   s->bytes = htonl(8);
   strcpy(s->name, "ts");
 
   s = &rspMsg->pSchema[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(1);
+  s->colId = htonl(2);
   s->bytes = htonl(4);
   strcpy(s->name, "col1s");
 
   s = &rspMsg->pSchema[2];
   s->type = TSDB_DATA_TYPE_BINARY;
-  s->colId = htonl(2);
+  s->colId = htonl(3);
   s->bytes = htonl(12);
   strcpy(s->name, "tag1s");
 
@@ -229,19 +345,19 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc
   SSchema *s = NULL;
   s = &rspMsg->pSchema[0];
   s->type = TSDB_DATA_TYPE_TIMESTAMP;
-  s->colId = htonl(0);
+  s->colId = htonl(1);
   s->bytes = htonl(8);
   strcpy(s->name, "ts");
 
   s = &rspMsg->pSchema[1];
   s->type = TSDB_DATA_TYPE_INT;
-  s->colId = htonl(1);
+  s->colId = htonl(2);
   s->bytes = htonl(4);
   strcpy(s->name, "col1s");
 
   s = &rspMsg->pSchema[2];
   s->type = TSDB_DATA_TYPE_BINARY;
-  s->colId = htonl(2);
+  s->colId = htonl(3);
   s->bytes = htonl(12);
   strcpy(s->name, "tag1s");
 
@@ -371,6 +487,121 @@ void ctgTestSetPrepareDbVgroupsAndSuperMeta() {
 
 }
 
+void *ctgTestGetDbVgroupThread(void *param) {
+  struct SCatalog* pCtg = (struct SCatalog*)param;
+  int32_t code = 0;
+  void *mockPointer = (void *)0x1;
+  SArray *vgList = NULL;
+  int32_t n = 0;
+  
+  while (!ctgTestStop) {
+    code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
+    if (code) {
+      assert(0);
+    }
+
+    if (vgList) {
+      taosArrayDestroy(vgList);
+    }
+
+    if (ctgTestEnableSleep) {
+      usleep(rand()%5);
+    }
+    if (++n % 50000 == 0) {
+      printf("Get:%d\n", n);
+    }
+  }
+
+  return NULL;
+}
+
+void *ctgTestSetDbVgroupThread(void *param) {
+  struct SCatalog* pCtg = (struct SCatalog*)param;
+  int32_t code = 0;
+  SDBVgroupInfo dbVgroup = {0};
+  int32_t n = 0;
+  
+  while (!ctgTestStop) {
+    ctgTestBuildDBVgroup(&dbVgroup);
+    code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup);
+    if (code) {
+      assert(0);
+    }
+
+    if (ctgTestEnableSleep) {    
+      usleep(rand()%5);
+    }
+    if (++n % 50000 == 0) {
+      printf("Set:%d\n", n);
+    }
+  }
+
+  return NULL;
+
+}
+
+void *ctgTestGetCtableMetaThread(void *param) {
+  struct SCatalog* pCtg = (struct SCatalog*)param;
+  int32_t code = 0;
+  int32_t n = 0;
+  STableMeta* tbMeta = NULL;
+  int32_t exist = 0;
+
+  SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(cn.dbname, "db1");
+  strcpy(cn.tname, ctgTestCTablename);
+  
+  while (!ctgTestStop) {
+    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist);
+    if (code || 0 == exist) {
+      assert(0);
+    }
+
+    tfree(tbMeta);
+
+    if (ctgTestEnableSleep) {
+      usleep(rand()%5);
+    }
+    
+    if (++n % 50000 == 0) {
+      printf("Get:%d\n", n);
+    }
+  }
+
+  return NULL;
+}
+
+void *ctgTestSetCtableMetaThread(void *param) {
+  struct SCatalog* pCtg = (struct SCatalog*)param;
+  int32_t code = 0;
+  SDBVgroupInfo dbVgroup = {0};
+  int32_t n = 0;
+  STableMetaOutput output = {0};
+
+  ctgTestBuildCTableMetaOutput(&output);
+  
+  while (!ctgTestStop) {
+    code = ctgUpdateTableMetaCache(pCtg, &output);
+    if (code) {
+      assert(0);
+    }
+
+    if (ctgTestEnableSleep) {    
+      usleep(rand()%5);
+    }
+    if (++n % 50000 == 0) {
+      printf("Set:%d\n", n);
+    }
+  }
+
+  tfree(output.tbMeta);
+
+  return NULL;
+
+}
+
+#if 0
+
 TEST(tableMeta, normalTable) {
   struct SCatalog* pCtg = NULL;
   void *mockPointer = (void *)0x1;
@@ -388,7 +619,7 @@ TEST(tableMeta, normalTable) {
   code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestTablename);
 
@@ -436,11 +667,13 @@ TEST(tableMeta, childTableCase) {
   initQueryModuleMsgHandle();
 
   //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
   
-  int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestCTablename);
 
@@ -494,11 +727,14 @@ TEST(tableMeta, superTableCase) {
 
   initQueryModuleMsgHandle();
 
-  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  int32_t code = catalogInit(NULL);
-  int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestSTablename);
 
@@ -558,12 +794,15 @@ TEST(tableDistVgroup, normalTable) {
 
   initQueryModuleMsgHandle();
 
-  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  int32_t code = catalogInit(NULL);
-  
-  int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestTablename);
 
@@ -595,7 +834,7 @@ TEST(tableDistVgroup, childTableCase) {
   code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestCTablename);
 
@@ -620,11 +859,14 @@ TEST(tableDistVgroup, superTableCase) {
 
   initQueryModuleMsgHandle();
 
-  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  int32_t code = catalogInit(NULL);
-  int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg);
   ASSERT_EQ(code, 0);
 
-  SName n = {.type = T_NAME_TABLE, .acctId = 1};
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(n.dbname, "db1");
   strcpy(n.tname, ctgTestSTablename);
 
@@ -645,6 +887,167 @@ TEST(tableDistVgroup, superTableCase) {
   catalogDestroy();
 }
 
+TEST(dbVgroup, getSetDbVgroupCase) {
+  struct SCatalog* pCtg = NULL;
+  void *mockPointer = (void *)0x1;
+  SVgroupInfo vgInfo = {0};
+  SVgroupInfo *pvgInfo = NULL;
+  SDBVgroupInfo dbVgroup = {0};
+  SArray *vgList = NULL;
+
+  ctgTestSetPrepareDbVgroupsAndNormalMeta();
+
+  initQueryModuleMsgHandle();
+
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestTablename);
+
+  code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum);
+  
+  code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(vgInfo.vgId, 8);
+  ASSERT_EQ(vgInfo.numOfEps, 3);
+
+  code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
+  pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
+  ASSERT_EQ(pvgInfo->vgId, 8);
+  ASSERT_EQ(pvgInfo->numOfEps, 3);
+  taosArrayDestroy(vgList);
+
+  ctgTestBuildDBVgroup(&dbVgroup);
+  code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(vgInfo.vgId, 7);
+  ASSERT_EQ(vgInfo.numOfEps, 2);
+
+  code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
+  pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
+  ASSERT_EQ(pvgInfo->vgId, 8);
+  ASSERT_EQ(pvgInfo->numOfEps, 3);
+  taosArrayDestroy(vgList);
+  
+  catalogDestroy();
+}
+
+
+TEST(multiThread, getSetDbVgroupCase) {
+  struct SCatalog* pCtg = NULL;
+  void *mockPointer = (void *)0x1;
+  SVgroupInfo vgInfo = {0};
+  SVgroupInfo *pvgInfo = NULL;
+  SDBVgroupInfo dbVgroup = {0};
+  SArray *vgList = NULL;
+
+  ctgTestInitLogFile();
+  
+  ctgTestSetPrepareDbVgroups();
+
+  initQueryModuleMsgHandle();
+
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestTablename);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+
+  pthread_t thread1, thread2;
+  pthread_create(&(thread1), &thattr, ctgTestSetDbVgroupThread, pCtg);
+
+  sleep(1);
+  pthread_create(&(thread1), &thattr, ctgTestGetDbVgroupThread, pCtg);
+
+  while (true) {
+    if (ctgTestDeadLoop) {
+      sleep(1);
+    } else {
+      sleep(600);
+      break;
+    }
+  }
+  
+  ctgTestStop = true;
+  sleep(1);
+  
+  catalogDestroy();
+}
+
+#endif
+
+
+TEST(multiThread, ctableMeta) {
+  struct SCatalog* pCtg = NULL;
+  void *mockPointer = (void *)0x1;
+  SVgroupInfo vgInfo = {0};
+  SVgroupInfo *pvgInfo = NULL;
+  SDBVgroupInfo dbVgroup = {0};
+  SArray *vgList = NULL;
+
+  ctgTestSetPrepareDbVgroupsAndChildMeta();
+
+  initQueryModuleMsgHandle();
+
+  //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestTablename);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+
+  pthread_t thread1, thread2;
+  pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg);
+  sleep(1);
+  pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg);
+
+  while (true) {
+    if (ctgTestDeadLoop) {
+      sleep(1);
+    } else {
+      sleep(600);
+      break;
+    }
+  }
+  
+  ctgTestStop = true;
+  sleep(1);
+  
+  catalogDestroy();
+}
 
 
 int main(int argc, char** argv) {

source/libs/index/inc/indexInt.h

@@ -108,8 +108,17 @@ void iterateValueDestroy(IterateValue* iv, bool destroy);
 
 extern void* indexQhandle;
 
+typedef struct TFileCacheKey {
+  uint64_t suid;
+  uint8_t  colType;
+  char*    colName;
+  int32_t  nColName;
+} ICacheKey;
+
 int indexFlushCacheTFile(SIndex* sIdx, void*);
 
+int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
+
 #define indexFatal(...)                                                               \
   do {                                                                                \
     if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \

source/libs/index/inc/index_cache.h

@@ -42,6 +42,7 @@ typedef struct IndexCache {
   int32_t   version;
   int32_t   nTerm;
   int8_t    type;
+  uint64_t  suid;
 
   pthread_mutex_t mtx;
 } IndexCache;
@@ -58,7 +59,7 @@ typedef struct CacheTerm {
 } CacheTerm;
 //
 
-IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type);
+IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type);
 
 void indexCacheDestroy(void* cache);
 

source/libs/index/inc/index_fst_counting_writer.h

@@ -38,6 +38,7 @@ typedef struct WriterCtx {
       int  fd;
       bool readOnly;
       char buf[256];
+      int  size;
     } file;
     struct {
       int32_t capa;

source/libs/index/inc/index_tfile.h

@@ -49,13 +49,6 @@ typedef struct TFileValue {
   int32_t offset;
 } TFileValue;
 
-typedef struct TFileCacheKey {
-  uint64_t suid;
-  uint8_t  colType;
-  char*    colName;
-  int32_t  nColName;
-} TFileCacheKey;
-
 // table cache
 // refactor to LRU cache later
 typedef struct TFileCache {
@@ -103,10 +96,10 @@ typedef struct TFileReaderOpt {
 // tfile cache, manage tindex reader
 TFileCache*  tfileCacheCreate(const char* path);
 void         tfileCacheDestroy(TFileCache* tcache);
-TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key);
+TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key);
-void         tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader);
+void         tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader);
 
-TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName);
+TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName);
 
 TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName);
 TFileReader* tfileReaderCreate(WriterCtx* ctx);
@@ -124,6 +117,7 @@ int          tfileWriterFinish(TFileWriter* tw);
 
 //
 IndexTFile* indexTFileCreate(const char* path);
+void        indexTFileDestroy(IndexTFile* tfile);
 int         indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid);
 int         indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result);
 

source/libs/index/inc/index_util.h

@@ -34,7 +34,7 @@ extern "C" {
 #define SERIALIZE_VAR_TO_BUF(buf, var, type)  \
   do {                                        \
     type c = var;                             \
-    assert(sizeof(var) == sizeof(type));      \
+    assert(sizeof(type) == sizeof(c));        \
     memcpy((void*)buf, (void*)&c, sizeof(c)); \
     buf += sizeof(c);                         \
   } while (0)

source/libs/index/src/index.c

@@ -17,6 +17,7 @@
 #include "indexInt.h"
 #include "index_cache.h"
 #include "index_tfile.h"
+#include "index_util.h"
 #include "tdef.h"
 #include "tsched.h"
 
@@ -72,6 +73,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
 #ifdef USE_INVERTED_INDEX
   // sIdx->cache = (void*)indexCacheCreate(sIdx);
   sIdx->tindex = indexTFileCreate(path);
+  if (sIdx->tindex == NULL) { goto END; }
   sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   sIdx->cVersion = 1;
   sIdx->path = calloc(1, strlen(path) + 1);
@@ -82,6 +84,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
 
   return 0;
 #endif
+END:
+  if (sIdx != NULL) { indexClose(sIdx); }
 
   *index = NULL;
   return -1;
@@ -102,6 +106,7 @@ void indexClose(SIndex* sIdx) {
   }
   taosHashCleanup(sIdx->colObj);
   pthread_mutex_destroy(&sIdx->mtx);
+  indexTFileDestroy(sIdx->tindex);
 #endif
   free(sIdx->path);
   free(sIdx);
@@ -130,18 +135,28 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
   // TODO(yihao): reduce the lock range
   pthread_mutex_lock(&index->mtx);
   for (int i = 0; i < taosArrayGetSize(fVals); i++) {
-    SIndexTerm*  p = taosArrayGetP(fVals, i);
+    SIndexTerm* p = taosArrayGetP(fVals, i);
-    IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName);
+
+    char      buf[128] = {0};
+    ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)};
+    int32_t   sz = indexSerialCacheKey(&key, buf);
+
+    IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     if (cache == NULL) {
-      IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType);
+      IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType);
-      taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*));
+      taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*));
     }
   }
   pthread_mutex_unlock(&index->mtx);
 
   for (int i = 0; i < taosArrayGetSize(fVals); i++) {
-    SIndexTerm*  p = taosArrayGetP(fVals, i);
+    SIndexTerm* p = taosArrayGetP(fVals, i);
-    IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName);
+
+    char      buf[128] = {0};
+    ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)};
+    int32_t   sz = indexSerialCacheKey(&key, buf);
+
+    IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
     int ret = indexCachePut(*cache, p, uid);
     if (ret != 0) { return ret; }
@@ -200,7 +215,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
   indexInterResultsDestroy(interResults);
 
 #endif
-  return 1;
+  return 0;
 }
 
 int indexDelete(SIndex* index, SIndexMultiTermQuery* query) {
@@ -296,7 +311,12 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   // Get col info
   IndexCache* cache = NULL;
   pthread_mutex_lock(&sIdx->mtx);
-  IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName);
+
+  char      buf[128] = {0};
+  ICacheKey key = {.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName)};
+  int32_t   sz = indexSerialCacheKey(&key, buf);
+
+  IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz);
   if (pCache == NULL) {
     pthread_mutex_unlock(&sIdx->mtx);
     return -1;
@@ -360,6 +380,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
   if (sz > 0) {
     // TODO(yihao): remove duplicate tableid
     TFileValue* lv = taosArrayGetP(result, sz - 1);
+    // indexError("merge colVal: %s", lv->colVal);
     if (strcmp(lv->colVal, tv->colVal) == 0) {
       taosArrayAddAll(lv->tableId, tv->tableId);
       tfileValueDestroy(tv);
@@ -368,6 +389,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) {
     }
   } else {
     taosArrayPush(result, &tv);
+    // indexError("merge colVal: %s", tv->colVal);
   }
 }
 static void indexDestroyTempResult(SArray* result) {
@@ -383,10 +405,12 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
   indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
 
   IndexCache*  pCache = (IndexCache*)cache;
-  TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName);
+  TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
+  if (pReader == NULL) { indexWarn("empty pReader found"); }
   // handle flush
   Iterate* cacheIter = indexCacheIteratorCreate(pCache);
   Iterate* tfileIter = tfileIteratorCreate(pReader);
+  if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); }
 
   SArray* result = taosArrayInit(1024, sizeof(void*));
 
@@ -459,14 +483,14 @@ void iterateValueDestroy(IterateValue* value, bool destroy) {
   } else {
     if (value->val != NULL) { taosArrayClear(value->val); }
   }
-  // free(value->colVal);
+  free(value->colVal);
   value->colVal = NULL;
 }
 static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   int32_t version = CACHE_VERSION(cache);
   uint8_t colType = cache->type;
 
-  TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType);
+  TFileWriter* tw = tfileWriterOpen(sIdx->path, cache->suid, version, cache->colName, colType);
   if (tw == NULL) {
     indexError("failed to open file to write");
     return -1;
@@ -479,14 +503,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   }
   tfileWriterClose(tw);
 
-  TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName);
+  TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName);
+
+  char         buf[128] = {0};
+  TFileHeader* header = &reader->header;
+  ICacheKey    key = {
+      .suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType};
 
-  char          buf[128] = {0};
-  TFileHeader*  header = &reader->header;
-  TFileCacheKey key = {.suid = header->suid,
-                       .colName = header->colName,
-                       .nColName = strlen(header->colName),
-                       .colType = header->colType};
   pthread_mutex_lock(&sIdx->mtx);
 
   IndexTFile* ifile = (IndexTFile*)sIdx->tindex;
@@ -497,3 +520,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
 END:
   tfileWriterClose(tw);
 }
+
+int32_t indexSerialCacheKey(ICacheKey* key, char* buf) {
+  char* p = buf;
+  SERIALIZE_MEM_TO_BUF(buf, key, suid);
+  SERIALIZE_VAR_TO_BUF(buf, '_', char);
+  // SERIALIZE_MEM_TO_BUF(buf, key, colType);
+  // SERIALIZE_VAR_TO_BUF(buf, '_', char);
+  SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
+  return buf - p;
+}

source/libs/index/src/index_cache.c

@@ -20,7 +20,7 @@
 
 #define MAX_INDEX_KEY_LEN 256  // test only, change later
 
-#define MEM_TERM_LIMIT 10000 * 10
+#define MEM_TERM_LIMIT 5 * 10000
 // ref index_cache.h:22
 //#define CACHE_KEY_LEN(p) \
 //  (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) +
@@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera);
 
 static IterateValue* indexCacheIteratorGetValue(Iterate* iter);
 
-IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) {
+IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) {
   IndexCache* cache = calloc(1, sizeof(IndexCache));
   if (cache == NULL) {
     indexError("failed to create index cache");
@@ -53,7 +53,7 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) {
   cache->type = type;
   cache->index = idx;
   cache->version = 0;
-
+  cache->suid = suid;
   pthread_mutex_init(&cache->mtx, NULL);
   indexCacheRef(cache);
   return cache;
@@ -150,6 +150,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
 
   MemTable* tbl = cache->imm;
   iiter->val.val = taosArrayInit(1, sizeof(uint64_t));
+  iiter->val.colVal = NULL;
   iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL;
   iiter->next = indexCacheIteratorNext;
   iiter->getValue = indexCacheIteratorGetValue;
@@ -353,6 +354,9 @@ static bool indexCacheIteratorNext(Iterate* itera) {
   SSkipListIterator* iter = itera->iter;
   if (iter == NULL) { return false; }
   IterateValue* iv = &itera->val;
+  if (iv->colVal != NULL && iv->val != NULL) {
+    // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
+  }
   iterateValueDestroy(iv, false);
 
   bool next = tSkipListIterNext(iter);

source/libs/index/src/index_fst.c

@@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) {
   assert(s->state == OneTransNext || s->state == OneTrans);
 
   uint8_t val;
-  COMMON_INDEX(inp, 0x111111, val);
+  COMMON_INDEX(inp, 0b111111, val);
   s->val = (s->val & fstStateDict[s->state].val) | val;
 }
 
@@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) {
   bool      null = false;
   uint8_t   inp = fstStateCommInput(s, &null);
   uint8_t*  data = fstSliceData(slice, NULL);
-  return null == false ? inp : data[-1];
+  return null == false ? inp : data[node->start - 1];
 }
 uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
   assert(s->state == AnyTrans);
@@ -1062,6 +1062,7 @@ Output fstEmptyFinalOutput(Fst* fst, bool* null) {
   } else {
     *null = true;
   }
+  fstNodeDestroy(node);
   return res;
 }
 
@@ -1286,6 +1287,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
       free(buf);
       fstSliceDestroy(&slice);
+      taosArrayDestroy(nodes);
       return result;
     }
     free(buf);

source/libs/index/src/index_fst_counting_writer.c

@@ -72,9 +72,17 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
     if (readOnly == false) {
       // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
       ctx->file.fd = tfOpenCreateWriteAppend(path);
+
+      struct stat fstat;
+      stat(path, &fstat);
+      ctx->file.size = fstat.st_size;
     } else {
       // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
       ctx->file.fd = tfOpenRead(path);
+
+      struct stat fstat;
+      stat(path, &fstat);
+      ctx->file.size = fstat.st_size;
     }
     memcpy(ctx->file.buf, path, strlen(path));
     if (ctx->file.fd < 0) {
@@ -104,6 +112,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
     free(ctx->mem.buf);
   } else {
     tfClose(ctx->file.fd);
+    ctx->flush(ctx);
     if (remove) { unlink(ctx->file.buf); }
   }
   free(ctx);

source/libs/index/src/index_tfile.c

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
+p *
  * This program is free software: you can use, redistribute, and/or modify
  * it under the terms of the GNU Affero General Public License, version 3
  * or later ("AGPL"), as published by the Free Software Foundation.
@@ -45,13 +45,13 @@ static int tfileReaderLoadHeader(TFileReader* reader);
 static int tfileReaderLoadFst(TFileReader* reader);
 static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result);
 
-static int  tfileGetFileList(const char* path, SArray* result);
+static SArray* tfileGetFileList(const char* path);
-static int  tfileRmExpireFile(SArray* result);
+static int     tfileRmExpireFile(SArray* result);
-static void tfileDestroyFileName(void* elem);
+static void    tfileDestroyFileName(void* elem);
-static int  tfileCompare(const void* a, const void* b);
+static int     tfileCompare(const void* a, const void* b);
-static int  tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version);
+static int     tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version);
-static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version);
+static void    tfileGenFileName(char* filename, uint64_t suid, const char* col, int version);
-static void tfileSerialCacheKey(TFileCacheKey* key, char* buf);
+static void    tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version);
 
 TFileCache* tfileCacheCreate(const char* path) {
   TFileCache* tcache = calloc(1, sizeof(TFileCache));
@@ -60,38 +60,41 @@ TFileCache* tfileCacheCreate(const char* path) {
   tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   tcache->capacity = 64;
 
-  SArray* files = taosArrayInit(4, sizeof(void*));
+  SArray* files = tfileGetFileList(path);
-  tfileGetFileList(path, files);
-  taosArraySort(files, tfileCompare);
-  tfileRmExpireFile(files);
 
   uint64_t suid;
   int32_t  colId, version;
   for (size_t i = 0; i < taosArrayGetSize(files); i++) {
     char* file = taosArrayGetP(files, i);
-    if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) {
+
+    // refactor later, use colname and version info
+    char colName[256] = {0};
+    if (0 != tfileParseFileName(file, &suid, colName, (int*)&version)) {
       indexInfo("try parse invalid file:  %s, skip it", file);
       continue;
     }
 
-    WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64);
+    char fullName[256] = {0};
+    sprintf(fullName, "%s/%s", path, file);
+
+    WriterCtx* wc = writerCtxCreate(TFile, fullName, true, 1024 * 1024 * 64);
     if (wc == NULL) {
       indexError("failed to open index:%s", file);
       goto End;
     }
 
-    char          buf[128] = {0};
+    char         buf[128] = {0};
-    TFileReader*  reader = tfileReaderCreate(wc);
+    TFileReader* reader = tfileReaderCreate(wc);
-    TFileHeader*  header = &reader->header;
+    TFileHeader* header = &reader->header;
-    TFileCacheKey key = {.suid = header->suid,
+    ICacheKey    key = {.suid = header->suid,
-                         .colName = header->colName,
+                     .colName = header->colName,
-                         .nColName = strlen(header->colName),
+                     .nColName = strlen(header->colName),
-                         .colType = header->colType};
+                     .colType = header->colType};
-    tfileSerialCacheKey(&key, buf);
 
+    int32_t sz = indexSerialCacheKey(&key, buf);
+    assert(sz < sizeof(buf));
+    taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
     tfileReaderRef(reader);
-    // indexTable
-    taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
   }
   taosArrayDestroyEx(files, tfileDestroyFileName);
   return tcache;
@@ -117,30 +120,30 @@ void tfileCacheDestroy(TFileCache* tcache) {
   free(tcache);
 }
 
-TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) {
+TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
-  char buf[128] = {0};
+  char    buf[128] = {0};
-  tfileSerialCacheKey(key, buf);
+  int32_t sz = indexSerialCacheKey(key, buf);
-
+  assert(sz < sizeof(buf));
-  TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
+  TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
   if (reader == NULL) { return NULL; }
   tfileReaderRef(*reader);
 
   return *reader;
 }
-void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) {
+void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
-  char buf[128] = {0};
+  char    buf[128] = {0};
-  tfileSerialCacheKey(key, buf);
+  int32_t sz = indexSerialCacheKey(key, buf);
   // remove last version index reader
-  TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf));
+  TFileReader** p = taosHashGet(tcache->tableCache, buf, sz);
   if (p != NULL) {
     TFileReader* oldReader = *p;
-    taosHashRemove(tcache->tableCache, buf, strlen(buf));
+    taosHashRemove(tcache->tableCache, buf, sz);
     oldReader->remove = true;
     tfileReaderUnRef(oldReader);
   }
 
+  taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
   tfileReaderRef(reader);
-  taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
   return;
 }
 TFileReader* tfileReaderCreate(WriterCtx* ctx) {
@@ -201,12 +204,9 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
 }
 
 TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) {
-  char    filename[128] = {0};
-  int32_t coldId = 1;
-  tfileGenFileName(filename, suid, coldId, version);
-
   char fullname[256] = {0};
-  snprintf(fullname, sizeof(fullname), "%s/%s", path, filename);
+  tfileGenFileFullName(fullname, path, suid, colName, version);
+  // indexInfo("open write file name %s", fullname);
   WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
   if (wcx == NULL) { return NULL; }
 
@@ -219,19 +219,15 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c
   return tfileWriterCreate(wcx, &tfh);
 }
 TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) {
-  char    filename[128] = {0};
-  int32_t coldId = 1;
-  tfileGenFileName(filename, suid, coldId, version);
-
   char fullname[256] = {0};
-  snprintf(fullname, sizeof(fullname), "%s/%s", path, filename);
+  tfileGenFileFullName(fullname, path, suid, colName, version);
+
   WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
+  // indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
   if (wc == NULL) { return NULL; }
 
   TFileReader* reader = tfileReaderCreate(wc);
   return reader;
-
-  // tfileSerialCacheKey(&key, buf);
 }
 TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
   // char pathBuf[128] = {0};
@@ -325,17 +321,19 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
     tfileWriterClose(tw);
     return -1;
   }
-  // write fst
+
-  indexError("--------Begin----------------");
+  // write data
   for (size_t i = 0; i < sz; i++) {
     // TODO, fst batch write later
     TFileValue* v = taosArrayGetP((SArray*)data, i);
-    if (tfileWriteData(tw, v) == 0) {
+    if (tfileWriteData(tw, v) != 0) {
-      //
+      indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
+                 (int)taosArrayGetSize(v->tableId));
+    } else {
+      // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
+      //          (int)taosArrayGetSize(v->tableId));
     }
-    indexError("data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId));
   }
-  indexError("--------End----------------");
   fstBuilderFinish(tw->fb);
   fstBuilderDestroy(tw->fb);
   tw->fb = NULL;
@@ -359,7 +357,8 @@ IndexTFile* indexTFileCreate(const char* path) {
   tfile->cache = tfileCacheCreate(path);
   return tfile;
 }
-void IndexTFileDestroy(IndexTFile* tfile) {
+void indexTFileDestroy(IndexTFile* tfile) {
+  if (tfile == NULL) { return; }
   tfileCacheDestroy(tfile->cache);
   free(tfile);
 }
@@ -369,9 +368,8 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
   if (tfile == NULL) { return ret; }
   IndexTFile* pTfile = (IndexTFile*)tfile;
 
-  SIndexTerm*   term = query->term;
+  SIndexTerm* term = query->term;
-  TFileCacheKey key = {
+  ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
-      .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
   TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
   if (reader == NULL) { return 0; }
 
@@ -385,8 +383,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
 }
 static bool tfileIteratorNext(Iterate* iiter) {
   IterateValue* iv = &iiter->val;
+  if (iv->colVal != NULL && iv->val != NULL) {
+    // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
+  }
   iterateValueDestroy(iv, false);
-  // SArray* tblIds = iv->val;
 
   char*    colVal = NULL;
   uint64_t offset = 0;
@@ -406,14 +406,14 @@ static bool tfileIteratorNext(Iterate* iiter) {
   if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
 
   iv->colVal = colVal;
-
+  return true;
   // std::string key(ch, sz);
 }
 
 static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
 
 static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
-  TFileFstIter* tIter = calloc(1, sizeof(Iterate));
+  TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter));
   if (tIter == NULL) { return NULL; }
 
   tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
@@ -435,6 +435,7 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
   iter->next = tfileIteratorNext;
   iter->getValue = tifileIterateGetValue;
   iter->val.val = taosArrayInit(1, sizeof(uint64_t));
+  iter->val.colVal = NULL;
   return iter;
 }
 void tfileIteratorDestroy(Iterate* iter) {
@@ -447,13 +448,14 @@ void tfileIteratorDestroy(Iterate* iter) {
   streamWithStateDestroy(tIter->st);
   fstStreamBuilderDestroy(tIter->fb);
   automCtxDestroy(tIter->ctx);
+  free(tIter);
 
   free(iter);
 }
 
-TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) {
+TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
   if (tf == NULL) { return NULL; }
-  TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
+  ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
   return tfileCacheGet(tf->cache, &key);
 }
 
@@ -480,7 +482,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
 TFileValue* tfileValueCreate(char* val) {
   TFileValue* tf = calloc(1, sizeof(TFileValue));
   if (tf == NULL) { return NULL; }
-  tf->colVal = val;
+  tf->colVal = tstrdup(val);
   tf->tableId = taosArrayInit(32, sizeof(uint64_t));
   return tf;
 }
@@ -491,6 +493,7 @@ int tfileValuePush(TFileValue* tf, uint64_t val) {
 }
 void tfileValueDestroy(TFileValue* tf) {
   taosArrayDestroy(tf->tableId);
+  free(tf->colVal);
   free(tf);
 }
 static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) {
@@ -545,6 +548,9 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
     //
     indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
                errno, reader->ctx->file.fd, reader->ctx->file.buf);
+  } else {
+    indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
+               errno, reader->ctx->file.fd, reader->ctx->file.buf);
   }
   // assert(nread == sizeof(buf));
   memcpy(&reader->header, buf, sizeof(buf));
@@ -553,13 +559,14 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
 }
 static int tfileReaderLoadFst(TFileReader* reader) {
   // current load fst into memory, refactor it later
-  static int FST_MAX_SIZE = 64 * 1024;
+  static int FST_MAX_SIZE = 64 * 1024 * 1024;
 
   char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
   if (buf == NULL) { return -1; }
 
   WriterCtx* ctx = reader->ctx;
   int32_t    nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset);
+  indexError("nread = %d, and fst offset=%d, filename: %s ", nread, reader->header.fstOffset, ctx->file.buf);
   // we assuse fst size less than FST_MAX_SIZE
   assert(nread > 0 && nread < FST_MAX_SIZE);
 
@@ -603,19 +610,26 @@ void tfileReaderUnRef(TFileReader* reader) {
   }
 }
 
-static int tfileGetFileList(const char* path, SArray* result) {
+static SArray* tfileGetFileList(const char* path) {
+  SArray* files = taosArrayInit(4, sizeof(void*));
+
   DIR* dir = opendir(path);
-  if (NULL == dir) { return -1; }
+  if (NULL == dir) { return NULL; }
 
   struct dirent* entry;
   while ((entry = readdir(dir)) != NULL) {
+    if (entry->d_type && DT_DIR) { continue; }
     size_t len = strlen(entry->d_name);
     char*  buf = calloc(1, len + 1);
     memcpy(buf, entry->d_name, len);
-    taosArrayPush(result, &buf);
+    taosArrayPush(files, &buf);
   }
   closedir(dir);
-  return 0;
+
+  taosArraySort(files, tfileCompare);
+  tfileRmExpireFile(files);
+
+  return files;
 }
 static int tfileRmExpireFile(SArray* result) {
   // TODO(yihao): remove expire tindex after restart
@@ -636,22 +650,21 @@ static int tfileCompare(const void* a, const void* b) {
   if (ret == 0) { return ret; }
   return ret < 0 ? -1 : 1;
 }
-// tfile name suid-colId-version.tindex
+
-static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) {
+static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) {
-  sprintf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version);
+  if (3 == sscanf(filename, "%" PRIu64 "-%[^-]-%d.tindex", suid, col, version)) {
-  return;
-}
-static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) {
-  if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) {
     // read suid & colid & version  success
     return 0;
   }
   return -1;
 }
-static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) {
+// tfile name suid-colId-version.tindex
-  // SERIALIZE_MEM_TO_BUF(buf, key, suid);
+static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version) {
-  // SERIALIZE_VAR_TO_BUF(buf, '_', char);
+  sprintf(filename, "%" PRIu64 "-%s-%d.tindex", suid, col, version);
-  // SERIALIZE_MEM_TO_BUF(buf, key, colType);
+  return;
-  // SERIALIZE_VAR_TO_BUF(buf, '_', char);
+}
-  SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName);
+static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version) {
+  char filename[128] = {0};
+  tfileGenFileName(filename, suid, col, version);
+  sprintf(fullname, "%s/%s", path, filename);
 }

source/libs/index/test/fstTest.cc

@@ -24,8 +24,13 @@ class FstWriter {
     _b = fstBuilderCreate(_wc, 0);
   }
   bool Put(const std::string& key, uint64_t val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
     FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
     bool     ok = fstBuilderInsert(_b, skey, val);
+
     fstSliceDestroy(&skey);
     return ok;
   }
@@ -61,6 +66,11 @@ class FstReadMemory {
     return _fst != NULL;
   }
   bool Get(const std::string& key, uint64_t* val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
+
     FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
     bool     ok = fstGet(_fst, &skey, val);
     fstSliceDestroy(&skey);
@@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) {
   }
   return L * M * N;
 }
+void Performance_fstReadRecords(FstReadMemory* m) {
+  std::string str("aa");
+  for (int i = 0; i < M; i++) {
+    str[0] = 'a' + i;
+    str.resize(2);
+    for (int j = 0; j < N; j++) {
+      str[1] = 'a' + j;
+      str.resize(2);
+      for (int k = 0; k < L; k++) {
+        str.push_back('a');
+        uint64_t val, cost;
+        if (m->GetWithTimeCostUs(str, &val, &cost)) {
+          printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
+        } else {
+          printf("failed to get key: %s\n", str.c_str());
+        }
+      }
+    }
+  }
+}
+
+void checkMillonWriteAndReadOfFst() {
+  tfInit();
+  FstWriter* fw = new FstWriter;
+  Performance_fstWriteRecords(fw);
+  delete fw;
+  FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
+
+  if (fr->init()) { printf("success to init fst read"); }
+
+  Performance_fstReadRecords(fr);
+  tfCleanup();
+  delete fr;
+}
+void checkFstLongTerm() {
+  tfInit();
+  FstWriter* fw = new FstWriter;
+  // Performance_fstWriteRecords(fw);
+
+  fw->Put("A B", 1);
+  fw->Put("C", 2);
+  fw->Put("a", 3);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("A B", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("C", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+  {
+    uint64_t val = 0;
+    if (m->Get("a", &val)) {
+      std::cout << "success to Get: " << val << std::endl;
+    } else {
+      std::cout << "failed to Get:" << val << std::endl;
+    }
+  }
+
+  // prefix search
+  // std::vector<uint64_t> result;
+
+  // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
+  // m->Search(ctx, result);
+  // std::cout << "size: " << result.size() << std::endl;
+  // assert(result.size() == count);
+  // for (int i = 0; i < result.size(); i++) {
+  // assert(result[i] == i);  // check result
+  //}
+  tfCleanup();
+  // free(ctx);
+  // delete m;
+}
 void checkFstCheckIterator() {
   tfInit();
   FstWriter* fw = new FstWriter;
   int64_t    s = taosGetTimestampUs();
   int        count = 2;
-  Performance_fstWriteRecords(fw);
+  // Performance_fstWriteRecords(fw);
   int64_t e = taosGetTimestampUs();
 
   std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("Hello world", 1);
+  fw->Put("hello world", 2);
+  fw->Put("hello worle", 3);
+  fw->Put("hello worlf", 4);
   delete fw;
 
   FstReadMemory* m = new FstReadMemory(1024 * 64);
@@ -171,7 +275,7 @@ void checkFstCheckIterator() {
 
 void fst_get(Fst* fst) {
   for (int i = 0; i < 10000; i++) {
-    std::string term = "Hello";
+    std::string term = "Hello World";
     FstSlice    key = fstSliceCreate((uint8_t*)term.c_str(), term.size());
     uint64_t    offset = 0;
     bool        ret = fstGet(fst, &key, &offset);
@@ -189,7 +293,7 @@ void validateTFile(char* arg) {
 
   std::thread threads[NUM_OF_THREAD];
   // std::vector<std::thread> threads;
-  TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1");
+  TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1");
 
   for (int i = 0; i < NUM_OF_THREAD; i++) {
     threads[i] = std::thread(fst_get, reader->fst);
@@ -203,9 +307,12 @@ void validateTFile(char* arg) {
   tfCleanup();
 }
 int main(int argc, char* argv[]) {
-  if (argc > 1) { validateTFile(argv[1]); }
+  // tool to check all kind of fst test 
+  // if (argc > 1) { validateTFile(argv[1]); }
   // checkFstCheckIterator();
+  // checkFstLongTerm();
   // checkFstPrefixSearch();
 
+  checkMillonWriteAndReadOfFst();
   return 1;
 }

source/libs/index/test/indexTests.cc

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include <gtest/gtest.h>
+#include <algorithm>
 #include <iostream>
 #include <string>
 #include <thread>
@@ -457,7 +458,10 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
   // taosArrayPush(data, &v4);
 
   fObj->Put(data);
-  for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); }
+  for (size_t i = 0; i < taosArrayGetSize(data); i++) {
+    // data
+    destroyTFileValue(taosArrayGetP(data, i));
+  }
   taosArrayDestroy(data);
 
   std::string     colName("voltage");
@@ -470,6 +474,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
   fObj->Get(&query, result);
   assert(taosArrayGetSize(result) == 200);
   indexTermDestroy(term);
+  taosArrayDestroy(result);
 
   // tfileWriterDestroy(twrite);
 }
@@ -477,7 +482,7 @@ class CacheObj {
  public:
   CacheObj() {
     // TODO
-    cache = indexCacheCreate(NULL, "voltage", TSDB_DATA_TYPE_BINARY);
+    cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY);
   }
   int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) {
     int ret = indexCachePut(cache, term, uid);
@@ -534,6 +539,7 @@ TEST_F(IndexCacheEnv, cache_test) {
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
+    indexTermDestroy(term);
     // indexTermDestry(term);
   }
   {
@@ -541,24 +547,28 @@ TEST_F(IndexCacheEnv, cache_test) {
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
+    indexTermDestroy(term);
   }
   {
     std::string colVal("v2");
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
+    indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
+    indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
+    indexTermDestroy(term);
   }
   coj->Debug();
   std::cout << "--------first----------" << std::endl;
@@ -567,12 +577,14 @@ TEST_F(IndexCacheEnv, cache_test) {
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
+    indexTermDestroy(term);
   }
   {
     std::string colVal("v4");
     SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
+    indexTermDestroy(term);
   }
   coj->Debug();
   std::cout << "--------second----------" << std::endl;
@@ -583,6 +595,7 @@ TEST_F(IndexCacheEnv, cache_test) {
       SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
       coj->Put(term, colId, version++, suid++);
+      indexTermDestroy(term);
     }
   }
   coj->Debug();
@@ -598,6 +611,9 @@ TEST_F(IndexCacheEnv, cache_test) {
     coj->Get(&query, colId, 10000, ret, &valType);
     std::cout << "size : " << taosArrayGetSize(ret) << std::endl;
     assert(taosArrayGetSize(ret) == 4);
+    taosArrayDestroy(ret);
+
+    indexTermDestroy(term);
   }
   {
     std::string     colVal("v2");
@@ -609,6 +625,9 @@ TEST_F(IndexCacheEnv, cache_test) {
 
     coj->Get(&query, colId, 10000, ret, &valType);
     assert(taosArrayGetSize(ret) == 1);
+    taosArrayDestroy(ret);
+
+    indexTermDestroy(term);
   }
 }
 class IndexObj {
@@ -620,7 +639,7 @@ class IndexObj {
     indexInit();
   }
   int Init(const std::string& dir) {
-    taosRemoveDir(dir.c_str());
+    // taosRemoveDir(dir.c_str());
     taosMkDir(dir.c_str());
     int ret = indexOpen(&opts, dir.c_str(), &idx);
     if (ret != 0) {
@@ -645,10 +664,11 @@ class IndexObj {
   int WriteMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                            size_t numOfTable = 100 * 10000) {
     std::string tColVal = colVal;
+    size_t      colValSize = tColVal.size();
     for (int i = 0; i < numOfTable; i++) {
-      tColVal[tColVal.size() - 1] = 'a' + i % 26;
+      tColVal[i % colValSize] = 'a' + i % 26;
       SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
-                                         colVal.c_str(), colVal.size());
+                                         tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
       for (size_t i = 0; i < 10; i++) {
@@ -677,14 +697,23 @@ class IndexObj {
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
     SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
-    if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; }
+
-    return taosArrayGetSize(result);
+    int64_t s = taosGetTimestampUs();
+    if (Search(mq, result) == 0) {
+      int64_t e = taosGetTimestampUs();
+      std::cout << "search one successfully and time cost:" << e - s << std::endl;
+    } else {
+    }
+    int sz = taosArrayGetSize(result);
+    indexMultiTermQueryDestroy(mq);
+    taosArrayDestroy(result);
+    return sz;
     // assert(taosArrayGetSize(result) == targetSize);
   }
   void PutOne(const std::string& colName, const std::string& colVal) {
+    SIndexMultiTerm* terms = indexMultiTermCreate();
     SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
-    SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
     Put(terms, 10);
     indexMultiTermDestroy(terms);
@@ -783,18 +812,21 @@ TEST_F(IndexEnv2, testIndexOpen) {
     index->Search(mq, result);
     std::cout << "target size: " << taosArrayGetSize(result) << std::endl;
     assert(taosArrayGetSize(result) == 400);
+    taosArrayDestroy(result);
+    indexMultiTermQueryDestroy(mq);
   }
 }
 
 TEST_F(IndexEnv2, testIndex_TrigeFlush) {
-  std::string path = "/tmp/test";
+  std::string path = "/tmp/testxxx";
   if (index->Init(path) != 0) {
     // r
     std::cout << "failed to init" << std::endl;
   }
   int numOfTable = 100 * 10000;
-  index->WriteMillonData("tag1", "Hello", numOfTable);
+  index->WriteMillonData("tag1", "Hello Wolrd", numOfTable);
-  int target = index->SearchOne("tag1", "Hello");
+  int target = index->SearchOne("tag1", "Hello Wolrd");
+  std::cout << "Get Index: " << target << std::endl;
   assert(numOfTable == target);
 }
 
@@ -802,6 +834,10 @@ static void write_and_search(IndexObj* idx) {
   std::string colName("tag1"), colVal("Hello");
 
   int target = idx->SearchOne("tag1", "Hello");
+  std::cout << "search: " << target << std::endl;
+  target = idx->SearchOne("tag2", "Test");
+  std::cout << "search: " << target << std::endl;
+
   idx->PutOne(colName, colVal);
 }
 TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
@@ -809,7 +845,10 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
   if (index->Init(path) != 0) {
     // opt
   }
-  index->WriteMultiMillonData("tag1", "Hello", 200000);
+  index->PutOne("tag1", "Hello");
+  index->PutOne("tag2", "Test");
+  index->WriteMultiMillonData("tag1", "Hello", 50 * 10000);
+  index->WriteMultiMillonData("tag2", "Test", 50 * 10000);
   std::thread threads[NUM_OF_THREAD];
 
   for (int i = 0; i < NUM_OF_THREAD; i++) {
@@ -821,25 +860,17 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
     threads[i].join();
   }
 }
-TEST_F(IndexEnv2, testIndex_multi_thread_write) {
-  std::string path = "/tmp";
-  if (index->Init(path) != 0) {}
-}
-TEST_F(IndexEnv2, testIndex_multi_thread_read) {
-  std::string path = "/tmp";
-  if (index->Init(path) != 0) {}
-}
 
 TEST_F(IndexEnv2, testIndex_restart) {
-  std::string path = "/tmp";
+  std::string path = "/tmp/test1";
   if (index->Init(path) != 0) {}
 }
 
 TEST_F(IndexEnv2, testIndex_performance) {
-  std::string path = "/tmp";
+  std::string path = "/tmp/test2";
   if (index->Init(path) != 0) {}
 }
 TEST_F(IndexEnv2, testIndexMultiTag) {
-  std::string path = "/tmp";
+  std::string path = "/tmp/test3";
   if (index->Init(path) != 0) {}
 }

source/libs/parser/CMakeLists.txt

@@ -11,4 +11,6 @@ target_link_libraries(
         PRIVATE os util catalog function transport qcom
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/parser/inc/astGenerator.h

@@ -125,7 +125,6 @@ typedef struct SCreatedTableInfo {
   SArray            *pTagNames;   // create by using super table, tag name
   SArray            *pTagVals;    // create by using super table, tag value
   char              *fullname;    // table full name
-  STagData           tagdata;     // true tag data, super table full name is in STagData
   int8_t             igExist;     // ignore if exists
 } SCreatedTableInfo;
 

source/libs/parser/inc/parserInt.h

@@ -68,7 +68,9 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ
  * @param type
  * @return
  */
-int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen);
+SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
+
+SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen);
 
 /**
  * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.

source/libs/parser/inc/sql.y

@@ -188,7 +188,7 @@ cmd ::= COMPACT VNODES IN LP exprlist(Y) RP.    { setCompactVnodeSql(pInfo, TSDB
 // And "ids" is an identifer-or-string.
 %type ids {SToken}
 ids(A) ::= ID(X).        {A = X; }
-ids(A) ::= STRING(X).    {A = X; }
+//ids(A) ::= STRING(X).    {A = X; }
 
 %type ifexists {SToken}
 ifexists(X) ::= IF EXISTS.          { X.n = 1;}
@@ -392,7 +392,7 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. {
 %type create_stable_args{SCreateTableSql*}
 create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. {
   A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE);
-  setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
+  setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_STABLE);
 
   V.n += Z.n;
   setCreatedTableName(pInfo, &V, &U);

source/libs/parser/src/astGenerator.c

@@ -692,7 +692,6 @@ void freeCreateTableInfo(void* p) {
   taosArrayDestroy(pInfo->pTagNames);
   taosArrayDestroyEx(pInfo->pTagVals, freeItem);
   tfree(pInfo->fullname);
-  tfree(pInfo->tagdata.data);
 }
 
 SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type) {
@@ -786,7 +785,7 @@ void destroySqlInfo(SSqlInfo *pInfo) {
   taosArrayDestroy(pInfo->funcs);
   if (pInfo->type == TSDB_SQL_SELECT) {
     destroyAllSqlNode(&pInfo->sub);
-  } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) {
+  } else if (pInfo->type == TSDB_SQL_CREATE_STABLE) {
     pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo);
   } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
     taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeItem);

source/libs/parser/src/dCDAstProcess.c

@@ -35,7 +35,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou
     SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq));
 
     SArray* array = NULL;
-    SName name = {0};
+    SName   name = {0};
     tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db));
 
     char dbFname[TSDB_DB_FNAME_LEN] = {0};
@@ -48,7 +48,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou
     pEpSet->numOfEps = info->numOfEps;
     pEpSet->inUse = info->inUse;
 
-    for(int32_t i = 0; i < pEpSet->numOfEps; ++i) {
+    for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
       strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
       pEpSet->port[i] = info->epAddr[i].port;
     }
@@ -190,7 +190,7 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
   val = htonl(pCreate->numOfVgroups);
   if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) {
     snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val,
-        TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
+             TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -321,8 +321,12 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len,
+typedef struct SVgroupTablesBatch {
-                               SEpSet* pEpSet) {
+  SVCreateTbBatchReq req;
+  SVgroupInfo        info;
+} SVgroupTablesBatch;
+
+int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) {
   const char* msg1 = "invalid table name";
   const char* msg2 = "tags number not matched";
   const char* msg3 = "tag value too long";
@@ -330,17 +334,14 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
 
   SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
 
+  SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+
   // super table name, create table by using dst
-  int32_t numOfTables = (int32_t)taosArrayGetSize(pCreateTable->childTableInfo);
+  size_t numOfTables = taosArrayGetSize(pCreateTable->childTableInfo);
   for (int32_t j = 0; j < numOfTables; ++j) {
     SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j);
 
     SToken* pSTableNameToken = &pCreateTableInfo->stbName;
-
-    char   buf[TSDB_TABLE_FNAME_LEN];
-    SToken sTblToken;
-    sTblToken.z = buf;
-
     int32_t code = parserValidateNameToken(pSTableNameToken);
     if (code != TSDB_CODE_SUCCESS) {
       return buildInvalidOperationMsg(pMsgBuf, msg1);
@@ -357,13 +358,16 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
     size_t      numOfInputTag = taosArrayGetSize(pValList);
     STableMeta* pSuperTableMeta = NULL;
 
-    catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta);
+    code = catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
     assert(pSuperTableMeta != NULL);
 
     // too long tag values will return invalid sql, not be truncated automatically
     SSchema*      pTagSchema = getTableTagSchema(pSuperTableMeta);
     STableComInfo tinfo = getTableInfo(pSuperTableMeta);
-    STagData*     pTag = &pCreateTableInfo->tagdata;
 
     SKVRowBuilder kvRowBuilder = {0};
     if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
@@ -461,28 +465,13 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
 
       for (int32_t i = 0; i < numOfInputTag; ++i) {
         SSchema* pSchema = &pTagSchema[i];
-        SToken*  pItem = taosArrayGet(pValList, i);
-
-        if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
-          if (pItem->n > pSchema->bytes) {
-            tdDestroyKVRowBuilder(&kvRowBuilder);
-            return buildInvalidOperationMsg(pMsgBuf, msg3);
-          }
-        } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
-          //          if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
-          ////            code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision);
-          //            if (code != TSDB_CODE_SUCCESS) {
-          //              return buildInvalidOperationMsg(pMsgBuf, msg4);
-          //            }
-          //          } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
-          //            pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision);
-          //          }
-        }
-
-        char     tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0};
-        SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema};
 
         char* endPtr = NULL;
+        char  tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0};
+
+        SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema};
+
+        SToken* pItem = taosArrayGet(pValList, i);
         code = parseValueToken(&endPtr, pItem, pSchema, tinfo.precision, tmpTokenBuf, KvRowAppend, &param, pMsgBuf);
 
         if (code != TSDB_CODE_SUCCESS) {
@@ -495,7 +484,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
     SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
     tdDestroyKVRowBuilder(&kvRowBuilder);
     if (row == NULL) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+      return TSDB_CODE_QRY_OUT_OF_MEMORY;
     }
 
     tdSortKVRowByColIdx(row);
@@ -506,40 +495,73 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
       return code;
     }
 
+    SVgroupInfo info = {0};
+    catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info);
+
     struct SVCreateTbReq req = {0};
     req.type = TD_CHILD_TABLE;
     req.name = strdup(tNameGetTableName(&tableName));
     req.ctbCfg.suid = pSuperTableMeta->suid;
     req.ctbCfg.pTag = row;
 
-    int32_t serLen = sizeof(SMsgHead) + tSerializeSVCreateTbReq(NULL, &req);
+    SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId));
-    char*   buf1 = calloc(1, serLen);
+    if (pTableBatch == NULL) {
-    *pOutput = buf1;
+      SVgroupTablesBatch tBatch = {0};
-    buf1 += sizeof(SMsgHead);
+      tBatch.info = info;
-    tSerializeSVCreateTbReq((void*)&buf1, &req);
-    *len = serLen;
 
-    SVgroupInfo info = {0};
+      tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
-    catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info);
+      taosArrayPush(tBatch.req.pArray, &req);
 
-    pEpSet->inUse = info.inUse;
+      taosHashPut(pVgroupHashmap, &info.vgId, sizeof(info.vgId), &tBatch, sizeof(tBatch));
-    pEpSet->numOfEps = info.numOfEps;
+    } else {  // add to the correct vgroup
-    for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
+      assert(info.vgId == pTableBatch->info.vgId);
-      pEpSet->port[i] = info.epAddr[i].port;
+      taosArrayPush(pTableBatch->req.pArray, &req);
-      tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
+    }
+  }
+
+  // TODO: serialize and
+  SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
+
+  SVgroupTablesBatch* pTbBatch = NULL;
+  do {
+    pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch);
+    if (pTbBatch == NULL) break;
+
+    int   tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req));
+    void* buf = malloc(tlen);
+    if (buf == NULL) {
+      // TODO: handle error
     }
 
-    ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId);
+    ((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId);
-    ((SMsgHead*)(*pOutput))->contLen = htonl(serLen);
+    ((SMsgHead*)buf)->contLen = htonl(tlen);
-  }
+
+    void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+    tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req));
+
+    SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks));
+    pVgData->vg    = pTbBatch->info;
+    pVgData->pData = buf;
+    pVgData->size  = tlen;
+    pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray);
+
+    taosArrayPush(pBufArray, &pVgData);
+  } while (true);
+
+  SInsertStmtInfo* pStmtInfo = calloc(1, sizeof(SInsertStmtInfo));
+  pStmtInfo->nodeType = TSDB_SQL_CREATE_TABLE;
+  pStmtInfo->pDataBlocks = pBufArray;
+  *pOutput = pStmtInfo;
+  *len = sizeof(SInsertStmtInfo);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf,
+SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
-                                  int32_t msgBufLen) {
   int32_t code = 0;
 
+  SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo));
+
   SMsgBuf  m = {.buf = msgBuf, .len = msgBufLen};
   SMsgBuf* pMsgBuf = &m;
 
@@ -556,21 +578,25 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
       SToken*    pPwd = &pUser->passwd;
 
       if (pName->n >= TSDB_USER_LEN) {
-        return buildInvalidOperationMsg(pMsgBuf, msg3);
+        code = buildInvalidOperationMsg(pMsgBuf, msg3);
+        goto _error;
       }
 
       if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
+        code = buildInvalidOperationMsg(pMsgBuf, msg2);
+        goto _error;
       }
 
       if (pInfo->type == TSDB_SQL_CREATE_USER) {
         if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
-          return TSDB_CODE_TSC_INVALID_OPERATION;
+          code = TSDB_CODE_TSC_INVALID_OPERATION;
+          goto _error;
         }
       } else {
         if (pUser->type == TSDB_ALTER_USER_PASSWD) {
           if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
+            code = TSDB_CODE_TSC_INVALID_OPERATION;
+            goto _error;
           }
         } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
           assert(pPwd->type == TSDB_DATA_TYPE_NULL);
@@ -581,10 +607,12 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
           } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) {
             //            pCmd->count = 2;
           } else {
-            return buildInvalidOperationMsg(pMsgBuf, msg4);
+            code = buildInvalidOperationMsg(pMsgBuf, msg4);
+            goto _error;
           }
         } else {
-          return buildInvalidOperationMsg(pMsgBuf, msg1);
+          code = buildInvalidOperationMsg(pMsgBuf, msg1);
+          goto _error;
         }
       }
 
@@ -603,15 +631,18 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
       SToken* pPwd = &pInfo->pMiscInfo->user.passwd;
 
       if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
+        code = TSDB_CODE_TSC_INVALID_OPERATION;
+        goto _error;
       }
 
       if (pName->n >= TSDB_USER_LEN) {
-        return buildInvalidOperationMsg(pMsgBuf, msg3);
+        code = buildInvalidOperationMsg(pMsgBuf, msg3);
+        goto _error;
       }
 
       if (parserValidateNameToken(pName) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
+        code = buildInvalidOperationMsg(pMsgBuf, msg2);
+        goto _error;
       }
 
       SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt;
@@ -621,7 +652,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
         } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) {
         } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) {
         } else {
-          return buildInvalidOperationMsg(pMsgBuf, msg1);
+          code = buildInvalidOperationMsg(pMsgBuf, msg1);
+          goto _error;
         }
       }
 
@@ -640,7 +672,11 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
     case TSDB_SQL_SHOW: {
       SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt;
       code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf);
-      pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE)? TDMT_VND_SHOW_TABLES:TDMT_MND_SHOW;
+      if (code != TSDB_CODE_SUCCESS) {
+        goto _error;
+      }
+      
+      pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE) ? TDMT_VND_SHOW_TABLES : TDMT_MND_SHOW;
       break;
     }
 
@@ -649,13 +685,15 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
 
       SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (parserValidateNameToken(pToken) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg);
+        code = buildInvalidOperationMsg(pMsgBuf, msg);
+        goto _error;
       }
 
       SName   n = {0};
       int32_t ret = tNameSetDbName(&n, pCtx->acctId, pToken->z, pToken->n);
       if (ret != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg);
+        code = buildInvalidOperationMsg(pMsgBuf, msg);
+        goto _error;
       }
 
       SUseDbMsg* pUseDbMsg = (SUseDbMsg*)calloc(1, sizeof(SUseDbMsg));
@@ -674,19 +712,22 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
 
       SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
       if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
+        code = buildInvalidOperationMsg(pMsgBuf, msg2);
+        goto _error;
       }
 
       char   buf[TSDB_DB_NAME_LEN] = {0};
       SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
 
       if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg1);
+        code = buildInvalidOperationMsg(pMsgBuf, msg1);
+        goto _error;
       }
 
       SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf);
       if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
+        code = TSDB_CODE_TSC_INVALID_OPERATION;
+        goto _error;
       }
 
       pDcl->pMsg = (char*)pCreateMsg;
@@ -704,7 +745,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
       SName name = {0};
       code = tNameSetDbName(&name, pCtx->acctId, dbName->z, dbName->n);
       if (code != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg1);
+        code = buildInvalidOperationMsg(pMsgBuf, msg1);
+        goto _error;
       }
 
       SDropDbMsg* pDropDbMsg = (SDropDbMsg*)calloc(1, sizeof(SDropDbMsg));
@@ -716,25 +758,21 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
       pDcl->msgType = TDMT_MND_DROP_DB;
       pDcl->msgLen = sizeof(SDropDbMsg);
       pDcl->pMsg = (char*)pDropDbMsg;
-      return TSDB_CODE_SUCCESS;
+      break;
     }
 
-    case TSDB_SQL_CREATE_TABLE: {
+    case TSDB_SQL_CREATE_STABLE: {
       SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
+      assert(pCreateTable->type != TSQL_CREATE_CTABLE);
 
       if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) {
         if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) {
-          return code;
+          terrno = code;
-        }
+          goto _error;
-        pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
-        pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB;
-      } else if (pCreateTable->type == TSQL_CREATE_CTABLE) {
-        if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet)) !=
-            TSDB_CODE_SUCCESS) {
-          return code;
         }
 
-        pDcl->msgType = TDMT_VND_CREATE_TABLE;
+        pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
+        pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB;
       } else if (pCreateTable->type == TSQL_CREATE_STREAM) {
         //        if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
         //          return code;
@@ -746,7 +784,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
     case TSDB_SQL_DROP_TABLE: {
       pDcl->pMsg = (char*)buildDropStableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf);
       if (pDcl->pMsg == NULL) {
-        code = terrno;
+        goto _error;
       }
 
       pDcl->msgType = TDMT_MND_DROP_STB;
@@ -756,7 +794,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
     case TSDB_SQL_CREATE_DNODE: {
       pDcl->pMsg = (char*)buildCreateDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf);
       if (pDcl->pMsg == NULL) {
-        code = terrno;
+        goto _error;
       }
 
       pDcl->msgType = TDMT_MND_CREATE_DNODE;
@@ -766,7 +804,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
     case TSDB_SQL_DROP_DNODE: {
       pDcl->pMsg = (char*)buildDropDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf);
       if (pDcl->pMsg == NULL) {
-        code = terrno;
+        goto _error;
       }
 
       pDcl->msgType = TDMT_MND_DROP_DNODE;
@@ -777,5 +815,29 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
       break;
   }
 
-  return code;
+  return pDcl;
+
+  _error:
+    terrno = code;
+    tfree(pDcl);
+    return NULL;
 }
+
+SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
+  SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
+  assert(pCreateTable->type == TSQL_CREATE_CTABLE);
+
+  SMsgBuf  m = {.buf = msgBuf, .len = msgBufLen};
+  SMsgBuf* pMsgBuf = &m;
+
+  SInsertStmtInfo* pInsertStmt = NULL;
+
+  int32_t msgLen = 0;
+  int32_t code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, (char**) &pInsertStmt, &msgLen);
+  if (code != TSDB_CODE_SUCCESS) {
+    tfree(pInsertStmt);
+    return NULL;
+  }
+
+  return pInsertStmt;
+}

source/libs/parser/src/parser.c

@@ -32,7 +32,7 @@ bool isInsertSql(const char* pStr, size_t length) {
 }
 
 bool qIsDdlQuery(const SQueryNode* pQuery) {
-  return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type;
+  return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type && TSDB_SQL_CREATE_TABLE != pQuery->type;
 }
 
 int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
@@ -44,16 +44,29 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
   }
 
   if (!isDqlSqlStatement(&info)) {
-    SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo));
+    bool toVnode = false;
-    if (NULL == pDcl) {
+    if (info.type == TSDB_SQL_CREATE_TABLE) {
-      terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
+      SCreateTableSql* pCreateSql = info.pCreateTableInfo;
-      return terrno;
+      if (pCreateSql->type == TSQL_CREATE_CTABLE || pCreateSql->type == TSQL_CREATE_TABLE) {
+        toVnode = true;
+      }
     }
 
-    pDcl->nodeType = info.type;
+    if (toVnode) {
-    int32_t code = qParserValidateDclSqlNode(&info, &pCxt->ctx, pDcl, pCxt->pMsg, pCxt->msgLen);
+      SInsertStmtInfo *pInsertInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
-    if (code == TSDB_CODE_SUCCESS) {
+      if (pInsertInfo == NULL) {
+        return terrno;
+      }
+
+      *pQuery = (SQueryNode*) pInsertInfo;
+    } else {
+      SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
+      if (pDcl == NULL) {
+        return terrno;
+      }
+
       *pQuery = (SQueryNode*)pDcl;
+      pDcl->nodeType = info.type;
     }
   } else {
     SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));

source/libs/parser/src/parserUtil.c

@@ -571,7 +571,9 @@ void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t fla
   pColumn->info.type  = pSchema->type;
 
   if (tableName != NULL) {
-    snprintf(pColumn->name, tListLen(pColumn->name), "%s.%s", tableName, pSchema->name);
+    char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0};
+    snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name);
+    tstrncpy(pColumn->name, n, tListLen(pColumn->name));
   } else {
     tstrncpy(pColumn->name, pSchema->name, tListLen(pColumn->name));
   }
@@ -586,7 +588,10 @@ SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSc
   c.info.type = pSchema->type;
 
   if (tableName != NULL) {
-    snprintf(c.name, tListLen(c.name), "%s.%s", tableName, pSchema->name);
+    char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0};
+    snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name);
+
+    tstrncpy(c.name, n, tListLen(c.name));
   } else {
     tstrncpy(c.name, pSchema->name, tListLen(c.name));
   }

source/libs/parser/test/parserTests.cpp

@@ -714,10 +714,9 @@ TEST(testCase, show_user_Test) {
   SSqlInfo info1 = doGenerateAST(sql1);
   ASSERT_EQ(info1.valid, true);
 
-  SDclStmtInfo output;
   SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL};
-  int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
+  SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len);
-  ASSERT_EQ(code, 0);
+  ASSERT_NE(output, nullptr);
 
   // convert the show command to be the select query
   // select name, privilege, create_time, account from information_schema.users;
@@ -735,10 +734,9 @@ TEST(testCase, create_user_Test) {
   ASSERT_EQ(info1.valid, true);
   ASSERT_EQ(isDclSqlStatement(&info1), true);
 
-  SDclStmtInfo output;
   SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"};
-  int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
+  SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len);
-  ASSERT_EQ(code, 0);
+  ASSERT_NE(output, nullptr);
 
   destroySqlInfo(&info1);
 }

source/libs/planner/CMakeLists.txt

@@ -11,4 +11,6 @@ target_link_libraries(
     PRIVATE os util catalog cjson parser transport function qcom
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/planner/inc/plannerInt.h

@@ -40,7 +40,7 @@ extern "C" {
 #define QNODE_SESSIONWINDOW 12
 #define QNODE_STATEWINDOW   13
 #define QNODE_FILL          14
-#define QNODE_INSERT        15
+#define QNODE_MODIFY        15
 
 typedef struct SQueryDistPlanNodeInfo {
   bool      stableQuery;   // super table query or not
@@ -70,6 +70,11 @@ typedef struct SQueryPlanNode {
   struct SQueryPlanNode  *pParent;
 } SQueryPlanNode;
 
+typedef struct SDataPayloadInfo {
+  int32_t msgType;
+  SArray *payload;
+} SDataPayloadInfo;
+
 /**
  * Optimize the query execution plan, currently not implement yet.
  * @param pQueryNode
@@ -100,7 +105,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
  */
 int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
 
-int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag);
+int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId);
 int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep);
 int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len);
 int32_t stringToSubplan(const char* str, SSubplan** subplan);

source/libs/planner/src/logicPlan.c

@@ -37,15 +37,29 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
   return 0;
 }
 
-int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) {
+static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) {
+  SInsertStmtInfo* pInsert = (SInsertStmtInfo*)pNode;
+
   *pQueryPlan = calloc(1, sizeof(SQueryPlanNode));
   SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES);
-  if (NULL == *pQueryPlan || NULL == blocks) {
+
+  SDataPayloadInfo* pPayload = calloc(1, sizeof(SDataPayloadInfo));
+  if (NULL == *pQueryPlan || NULL == blocks || NULL == pPayload) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
-  (*pQueryPlan)->info.type = QNODE_INSERT;
+
+  (*pQueryPlan)->info.type = QNODE_MODIFY;
   taosArrayAddAll(blocks, pInsert->pDataBlocks);
-  (*pQueryPlan)->pExtInfo = blocks;
+
+  if (pNode->type == TSDB_SQL_INSERT) {
+    pPayload->msgType = TDMT_VND_SUBMIT;
+  } else if (pNode->type == TSDB_SQL_CREATE_TABLE) {
+    pPayload->msgType = TDMT_VND_CREATE_TABLE;
+  }
+
+  pPayload->payload = blocks;
+  (*pQueryPlan)->pExtInfo = pPayload;
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -62,13 +76,14 @@ int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) {
     case TSDB_SQL_SELECT: {
       return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan);
     }
+
     case TSDB_SQL_INSERT:
-      return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan);
+    case TSDB_SQL_CREATE_TABLE:
+      return createModificationOpPlan(pNode, pQueryPlan);
+
     default:
       return TSDB_CODE_FAILED;
   }
-
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {

source/libs/planner/src/physicalPlan.c

@@ -34,7 +34,7 @@ static const char* gOpName[] = {
 #undef INCLUDE_AS_NAME
 };
 
-static void* vailidPointer(void* p) {
+static void* validPointer(void* p) {
   if (NULL == p) {
     THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
   }
@@ -76,7 +76,7 @@ int32_t dsinkNameToDsinkType(const char* name) {
 }
 
 static SDataSink* initDataSink(int32_t type, int32_t size) {
-  SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size));
+  SDataSink* sink = (SDataSink*)validPointer(calloc(1, size));
   sink->info.type = type;
   sink->info.name = dsinkTypeToDsinkName(type);
   return sink;
@@ -121,7 +121,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) {
 }
 
 static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) {
-  SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size));
+  SPhyNode* node = (SPhyNode*)validPointer(calloc(1, size));
   node->info.type = type;
   node->info.name = opTypeToOpName(type);
   if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) {
@@ -184,27 +184,30 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable
 }
 
 static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
-  SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan)));
+  SSubplan* subplan = validPointer(calloc(1, sizeof(SSubplan)));
   subplan->id = pCxt->nextId;
   ++(pCxt->nextId.subplanId);
   subplan->type = type;
   subplan->level = 0;
   if (NULL != pCxt->pCurrentSubplan) {
     subplan->level = pCxt->pCurrentSubplan->level + 1;
-    if (NULL == pCxt->pCurrentSubplan->pChildern) {
+    if (NULL == pCxt->pCurrentSubplan->pChildren) {
-      pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
+      pCxt->pCurrentSubplan->pChildren = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
     }
-    taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan);
+
-    subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
+    taosArrayPush(pCxt->pCurrentSubplan->pChildren, &subplan);
+    subplan->pParents = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
     taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan);
   }
+
   SArray* currentLevel;
   if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) {
-    currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
+    currentLevel = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
     taosArrayPush(pCxt->pDag->pSubplans, &currentLevel);
   } else {
     currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level);
   }
+
   taosArrayPush(currentLevel, &subplan);
   pCxt->pCurrentSubplan = subplan;
   ++(pCxt->pDag->numOfSubplans);
@@ -272,12 +275,13 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
     case QNODE_TABLESCAN:
       node = createTableScanNode(pCxt, pPlanNode);
       break;
-    case QNODE_INSERT:
+    case QNODE_MODIFY:
       // Insert is not an operator in a physical plan.
       break;
     default:
       assert(false);
   }
+
   if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) {
     node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
     size_t size = taosArrayGetSize(pPlanNode->pChildren);
@@ -287,46 +291,57 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
       taosArrayPush(node->pChildren, &child);
     }
   }
+
   return node;
 }
 
-static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
+static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
-  SArray* vgs = (SArray*)pPlanNode->pExtInfo;
+  SDataPayloadInfo* pPayload = (SDataPayloadInfo*) pPlanNode->pExtInfo;
-  size_t numOfVg = taosArrayGetSize(vgs);
+
-  for (int32_t i = 0; i < numOfVg; ++i) {
+  size_t numOfVgroups = taosArrayGetSize(pPayload->payload);
+  for (int32_t i = 0; i < numOfVgroups; ++i) {
     STORE_CURRENT_SUBPLAN(pCxt);
     SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY);
-    SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i);
+    SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i);
+
     vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet);
-    subplan->pNode = NULL;
     subplan->pDataSink = createDataInserter(pCxt, blocks);
-    subplan->type = QUERY_TYPE_MODIFY;
+    subplan->pNode   = NULL;
+    subplan->type    = QUERY_TYPE_MODIFY;
+    subplan->msgType = pPayload->msgType;
+    subplan->id.queryId = pCxt->pDag->queryId;
+
     RECOVERY_CURRENT_SUBPLAN(pCxt);
   }
 }
 
 static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) {
-  if (QNODE_INSERT == pRoot->info.type) {
+  if (QNODE_MODIFY == pRoot->info.type) {
-    splitInsertSubplan(pCxt, pRoot);
+    splitModificationOpSubPlan(pCxt, pRoot);
   } else {
-    SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE);
+    SSubplan* subplan  = initSubplan(pCxt, QUERY_TYPE_MERGE);
     ++(pCxt->nextId.templateId);
-    subplan->pNode = createPhyNode(pCxt, pRoot);
+
+    subplan->msgType   = TDMT_VND_QUERY;
+    subplan->pNode     = createPhyNode(pCxt, pRoot);
     subplan->pDataSink = createDataDispatcher(pCxt, pRoot);
   }
   // todo deal subquery
 }
 
-int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) {
+int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId) {
   TRY(TSDB_MAX_TAG_CONDITIONS) {
     SPlanContext context = {
       .pCatalog = pCatalog,
-      .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))),
+      .pDag = validPointer(calloc(1, sizeof(SQueryDag))),
       .pCurrentSubplan = NULL,
       .nextId = {0} // todo queryid
     };
+
     *pDag = context.pDag;
-    context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
+    context.pDag->queryId = requestId;
+
+    context.pDag->pSubplans = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES));
     createSubplanByLevel(&context, pQueryNode);
   } CATCH(code) {
     CLEANUP_EXECUTE();

source/libs/planner/src/physicalPlanJson.c

@@ -793,7 +793,7 @@ static cJSON* subplanToJson(const SSubplan* subplan) {
     return NULL;
   }
 
-  // The 'type', 'level', 'execEpSet', 'pChildern' and 'pParents' fields do not need to be serialized.
+  // The 'type', 'level', 'execEpSet', 'pChildren' and 'pParents' fields do not need to be serialized.
 
   bool res = addObject(jSubplan, jkSubplanId, subplanIdToJson, &subplan->id);
   if (res) {
@@ -835,7 +835,7 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
     SDataInserter* insert = (SDataInserter*)(subplan->pDataSink);
     *len = insert->size;
     *str = insert->pData;
-    insert->pData == NULL;
+    insert->pData = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
@@ -844,6 +844,7 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return TSDB_CODE_FAILED;
   }
+
   *str = cJSON_Print(json);
   *len = strlen(*str) + 1;
   return TSDB_CODE_SUCCESS;

source/libs/planner/src/planner.c

@@ -24,24 +24,27 @@ void qDestroyQueryDag(struct SQueryDag* pDag) {
   // todo
 }
 
-int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag) {
+int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) {
   SQueryPlanNode* logicPlan;
   int32_t code = createQueryPlan(pNode, &logicPlan);
   if (TSDB_CODE_SUCCESS != code) {
     destroyQueryPlan(logicPlan);
     return code;
   }
+
   code = optimizeQueryPlan(logicPlan);
   if (TSDB_CODE_SUCCESS != code) {
     destroyQueryPlan(logicPlan);
     return code;
   }
-  code = createDag(logicPlan, NULL, pDag);
+
+  code = createDag(logicPlan, NULL, pDag, requestId);
   if (TSDB_CODE_SUCCESS != code) {
     destroyQueryPlan(logicPlan);
     qDestroyQueryDag(*pDag);
     return code;
   }
+
   destroyQueryPlan(logicPlan);
   return TSDB_CODE_SUCCESS;
 }

source/libs/planner/test/phyPlanTests.cpp

@@ -45,7 +45,8 @@ protected:
 
   int32_t run() {
     SQueryDag* dag = nullptr;
-    int32_t code = createDag(logicPlan_.get(), nullptr, &dag);
+    uint64_t requestId = 20;
+    int32_t code = createDag(logicPlan_.get(), nullptr, &dag, requestId);
     dag_.reset(dag);
     return code;
   }
@@ -60,7 +61,8 @@ protected:
       return code;
     }
     SQueryDag* dag = nullptr;
-    code = qCreateQueryDag(query, &dag);
+    uint64_t requestId = 20;
+    code = qCreateQueryDag(query, &dag, requestId);
     dag_.reset(dag);
     return code;
   }

source/libs/qcom/CMakeLists.txt

@@ -7,8 +7,10 @@ target_include_directories(
 )
 
 target_link_libraries(
-   qcom
+    qcom
-   PRIVATE os util transport
+    PRIVATE os util transport
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/qcom/src/querymsg.c

@@ -40,7 +40,12 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3
 
   STableInfoMsg *bMsg = (STableInfoMsg *)*msg;
 
-  bMsg->vgId = bInput->vgId;
+  bMsg->header.vgId = htonl(bInput->vgId);
+
+  if (bInput->dbName) {
+    strncpy(bMsg->dbFname, bInput->dbName, sizeof(bMsg->dbFname));
+    bMsg->dbFname[sizeof(bMsg->dbFname) - 1] = 0;
+  }
 
   strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname));
   bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0;
@@ -243,9 +248,14 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
 
   if (pMetaMsg->tableType == TSDB_CHILD_TABLE) {
     pOut->metaNum = 2;
-    
+
-    memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname));
+    if (pMetaMsg->dbFname[0]) {
-    memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname));
+      snprintf(pOut->ctbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname);
+      snprintf(pOut->tbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->stbFname);
+    } else {
+      memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname));
+      memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname));
+    }
     
     pOut->ctbMeta.vgId = pMetaMsg->vgId;
     pOut->ctbMeta.tableType = pMetaMsg->tableType;
@@ -256,7 +266,11 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
   } else {
     pOut->metaNum = 1;
     
-    memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname));
+    if (pMetaMsg->dbFname[0]) {
+      snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname);
+    } else {
+      memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname));
+    }
     
     code = queryCreateTableMetaFromMsg(pMetaMsg, false, &pOut->tbMeta);
   }

source/libs/qworker/CMakeLists.txt

@@ -11,4 +11,6 @@ target_link_libraries(
    PRIVATE os util transport planner qcom
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/scheduler/CMakeLists.txt

@@ -12,4 +12,6 @@ target_link_libraries(
         PRIVATE os util planner qcom common catalog transport
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})

source/libs/scheduler/inc/schedulerInt.h

@@ -89,12 +89,12 @@ typedef struct SSchJob {
   SEpSet           dataSrcEps;
   SEpAddr          resEp;
   void            *transport;
-  SArray          *qnodeList;
+  SArray          *nodeList;   // qnode/vnode list, element is SQueryNodeAddr
   tsem_t           rspSem;
   int32_t          userFetch;
   int32_t          remoteFetch;
 
-  SSchTask      *fetchTask;
+  SSchTask        *fetchTask;
   int32_t          errCode;
   void            *res;
   int32_t          resNumOfRows;

source/libs/scheduler/src/scheduler.c

@@ -28,7 +28,7 @@ int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) {
     for (int32_t m = 0; m < level->taskNum; ++m) {
       SSchTask *task = taosArrayGet(level->subTasks, m);
       SSubplan *plan = task->plan;
-      int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0;
+      int32_t childNum = plan->pChildren ? (int32_t)taosArrayGetSize(plan->pChildren) : 0;
       int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0;
 
       if (childNum > 0) {
@@ -40,7 +40,7 @@ int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) {
       }
 
       for (int32_t n = 0; n < childNum; ++n) {
-        SSubplan **child = taosArrayGet(plan->pChildern, n);
+        SSubplan **child = taosArrayGet(plan->pChildren, n);
         SSchTask **childTask = taosHashGet(planToTask, child, POINTER_BYTES);
         if (NULL == childTask || NULL == *childTask) {
           qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@@ -122,6 +122,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) {
     SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  //??
   job->attr.needFetch = true;
   
   job->levelNum = levelNum;
@@ -220,10 +221,10 @@ int32_t schSetTaskExecEpSet(SSchJob *job, SEpSet *epSet) {
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t qnodeNum = taosArrayGetSize(job->qnodeList);
+  int32_t nodeNum = taosArrayGetSize(job->nodeList);
   
-  for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) {
+  for (int32_t i = 0; i < nodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) {
-    SEpAddr *addr = taosArrayGet(job->qnodeList, i);
+    SEpAddr *addr = taosArrayGet(job->nodeList, i);
     
     strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn));
     epSet->port[epSet->numOfEps] = addr->port;
@@ -455,15 +456,27 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) {
 
 int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) {
   int32_t code = 0;
-  
+
+
   switch (msgType) {
+    case TDMT_VND_CREATE_TABLE_RSP: {
+      if (rspCode != TSDB_CODE_SUCCESS) {
+        SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode));
+      } else {
+//        job->resNumOfRows += rsp->affectedRows;
+        code = schProcessOnTaskSuccess(job, task);
+        if (code) {
+          goto _task_error;
+        }
+      }
+    }
     case TDMT_VND_SUBMIT_RSP: {
-        SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg;
+        if (rspCode != TSDB_CODE_SUCCESS) {
-        if (rsp->code != TSDB_CODE_SUCCESS) {
+          SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode));
-          SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code));
         } else {
+          SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg;
           job->resNumOfRows += rsp->affectedRows;
-          
+
           code = schProcessOnTaskSuccess(job, task);
           if (code) {
             goto _task_error;
@@ -547,22 +560,29 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in
 
 _return:  
   tfree(param);
-
   SCH_RET(code);
 }
 
 int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code);
 }
+
+int32_t schHandleCreateTableCallback(void* param, const SDataBuf* pMsg, int32_t code) {
+  return schHandleCallback(param, pMsg, TDMT_VND_CREATE_TABLE_RSP, code);
+}
+
 int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code);
 }
+
 int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code);
 }
+
 int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code);
 }
+
 int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) {  
   SSchCallbackParam *pParam = (SSchCallbackParam *)param;
   qDebug("drop task rsp received, queryId:%"PRIx64 ",taksId:%"PRIx64 ",code:%d", pParam->queryId, pParam->taskId, code);
@@ -570,6 +590,9 @@ int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) {
 
 int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
   switch (msgType) {
+    case TDMT_VND_CREATE_TABLE:
+      *fp = schHandleCreateTableCallback;
+      break;
     case TDMT_VND_SUBMIT: 
       *fp = schHandleSubmitCallback;
       break;
@@ -640,6 +663,7 @@ int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) {
   int32_t code = 0;
   
   switch (msgType) {
+    case TDMT_VND_CREATE_TABLE:
     case TDMT_VND_SUBMIT: {
       if (NULL == task->msg || task->msgLen <= 0) {
         qError("submit msg is NULL");
@@ -664,7 +688,7 @@ int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) {
       }
 
       SSubQueryMsg *pMsg = msg;
-      
+
       pMsg->sId = htobe64(schMgmt.sId);
       pMsg->queryId = htobe64(job->queryId);
       pMsg->taskId = htobe64(task->taskId);
@@ -746,19 +770,15 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) {
     SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
   }
 
-  int32_t msgType = (plan->type == QUERY_TYPE_MODIFY) ? TDMT_VND_SUBMIT : TDMT_VND_QUERY;
+//  int32_t msgType = (plan->type == QUERY_TYPE_MODIFY)? TDMT_VND_SUBMIT : TDMT_VND_QUERY;
   
-  SCH_ERR_RET(schBuildAndSendMsg(job, task, msgType));
+  SCH_ERR_RET(schBuildAndSendMsg(job, task, plan->msgType));
-
   SCH_ERR_RET(schPushTaskToExecList(job, task));
 
   task->status = JOB_TASK_STATUS_EXECUTING;
-
   return TSDB_CODE_SUCCESS;
 }
 
-
-
 int32_t schLaunchJob(SSchJob *job) {
   SSchLevel *level = taosArrayGet(job->levels, job->levelIdx);
   for (int32_t i = 0; i < level->taskNum; ++i) {
@@ -829,8 +849,8 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
 }
 
 
-int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) {
+int32_t scheduleExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) {
-  if (qnodeList && taosArrayGetSize(qnodeList) <= 0) {
+  if (nodeList && taosArrayGetSize(nodeList) <= 0) {
     qInfo("qnodeList is empty");
   }
 
@@ -842,7 +862,7 @@ int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag,
 
   job->attr.syncSchedule = syncSchedule;
   job->transport = transport;
-  job->qnodeList = qnodeList;
+  job->nodeList = nodeList;
 
   SCH_ERR_JRET(schValidateAndBuildJob(pDag, job));
 
@@ -897,28 +917,27 @@ _return:
   SCH_RET(code);
 }
 
-int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows) {
+int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) {
-  if (NULL == transport || /* NULL == qnodeList || */ NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == numOfRows) {
+  if (NULL == transport || /* NULL == nodeList || */ NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) {
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
-  *numOfRows = 0;
+  SCH_ERR_RET(scheduleExecJobImpl(transport, nodeList, pDag, pJob, true));
-  
-  SCH_ERR_RET(scheduleExecJobImpl(transport, qnodeList, pDag, pJob, true));
 
   SSchJob *job = *(SSchJob **)pJob;
   
-  *numOfRows = job->resNumOfRows;
+  pRes->code = job->errCode;
-
+  pRes->numOfRows = job->resNumOfRows;
+  
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) {
+int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) {
-  if (NULL == transport || NULL == qnodeList ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
+  if (NULL == transport || NULL == nodeList ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
-  return scheduleExecJobImpl(transport, qnodeList, pDag, pJob, false);
+  return scheduleExecJobImpl(transport, nodeList, pDag, pJob, false);
 }
 
 

source/libs/scheduler/test/schedulerTests.cpp

@@ -58,7 +58,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
   scanPlan.execEpSet.numOfEps = 1;
   scanPlan.execEpSet.port[0] = 6030;
   strcpy(scanPlan.execEpSet.fqdn[0], "ep0");
-  scanPlan.pChildern = NULL;
+  scanPlan.pChildren = NULL;
   scanPlan.pParents = taosArrayInit(1, POINTER_BYTES);
   scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
 
@@ -68,14 +68,14 @@ void schtBuildQueryDag(SQueryDag *dag) {
   mergePlan.type = QUERY_TYPE_MERGE;
   mergePlan.level = 0;
   mergePlan.execEpSet.numOfEps = 0;
-  mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES);
+  mergePlan.pChildren = taosArrayInit(1, POINTER_BYTES);
   mergePlan.pParents = NULL;
   mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
 
   SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan);
   SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan);
 
-  taosArrayPush(mergePointer->pChildern, &scanPointer);
+  taosArrayPush(mergePointer->pChildren, &scanPointer);
   taosArrayPush(scanPointer->pParents, &mergePointer);
 
   taosArrayPush(dag->pSubplans, &merge);  
@@ -100,7 +100,7 @@ void schtBuildInsertDag(SQueryDag *dag) {
   insertPlan[0].execEpSet.numOfEps = 1;
   insertPlan[0].execEpSet.port[0] = 6030;
   strcpy(insertPlan[0].execEpSet.fqdn[0], "ep0");
-  insertPlan[0].pChildern = NULL;
+  insertPlan[0].pChildren = NULL;
   insertPlan[0].pParents = NULL;
   insertPlan[0].pNode = NULL;
   insertPlan[0].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink));
@@ -113,7 +113,7 @@ void schtBuildInsertDag(SQueryDag *dag) {
   insertPlan[1].execEpSet.numOfEps = 1;
   insertPlan[1].execEpSet.port[0] = 6030;
   strcpy(insertPlan[1].execEpSet.fqdn[0], "ep1");
-  insertPlan[1].pChildern = NULL;
+  insertPlan[1].pChildren = NULL;
   insertPlan[1].pParents = NULL;
   insertPlan[1].pNode = NULL;
   insertPlan[1].pDataSink = (SDataSink*)calloc(1, sizeof(SDataSink));
@@ -321,10 +321,11 @@ TEST(insertTest, normalCase) {
 
   pthread_t thread1;
   pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob);
-  
+
-  code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &numOfRows);
+  SQueryResult res = {0};
+  code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res);
   ASSERT_EQ(code, 0);
-  ASSERT_EQ(numOfRows, 20);
+  ASSERT_EQ(res.numOfRows, 20);
 
   scheduleFreeJob(pInsertJob);
 }

source/libs/transport/src/rpcMain.c

@@ -700,7 +700,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->sid = sid;
     pConn->tranId = (uint16_t)(taosRand() & 0xFFFF);
     pConn->ownId = htonl(pConn->sid);
-    pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId);
+    pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPId() + (int64_t)pConn->tranId);
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN);

source/os/src/osSysinfo.c

@@ -1130,8 +1130,4 @@ SysNameInfo taosGetSysNameInfo() {
   return info;
 }
 
-int64_t taosGetPid() {
-  getpid();
-}
-
 #endif

source/util/CMakeLists.txt

@@ -14,4 +14,7 @@ target_link_libraries(
     PUBLIC api
 )
 
-ADD_SUBDIRECTORY(test)
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})
+

source/util/src/encode.c

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "encode.h"
+
+#if __STDC_VERSION__ >= 201112L
+static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
+static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
+#endif
+
+void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type) {
+  if (type == TD_ENCODER) {
+    if (data == NULL) size = 0;
+  } else {
+    ASSERT(data && size > 0);
+  }
+
+  pCoder->type = type;
+  pCoder->endian = endian;
+  pCoder->data = data;
+  pCoder->size = size;
+  pCoder->pos = 0;
+  tFreeListInit(&(pCoder->fl));
+  TD_SLIST_INIT(&(pCoder->stack));
+}
+
+void tCoderClear(SCoder* pCoder) {
+  tFreeListClear(&(pCoder->fl));
+  struct SCoderNode* pNode;
+  for (;;) {
+    pNode = TD_SLIST_HEAD(&(pCoder->stack));
+    if (pNode == NULL) break;
+    TD_SLIST_POP(&(pCoder->stack));
+    free(pNode);
+  }
+}
+
+int tStartEncode(SCoder* pCoder) {
+  struct SCoderNode* pNode;
+
+  ASSERT(pCoder->type == TD_ENCODER);
+  if (pCoder->data) {
+    if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1;
+
+    pNode = malloc(sizeof(*pNode));
+    if (pNode == NULL) return -1;
+
+    pNode->data = pCoder->data;
+    pNode->pos = pCoder->pos;
+    pNode->size = pCoder->size;
+
+    pCoder->data = pNode->data + pNode->pos + sizeof(int32_t);
+    pCoder->pos = 0;
+    pCoder->size = pNode->size - pNode->pos - sizeof(int32_t);
+
+    TD_SLIST_PUSH(&(pCoder->stack), pNode);
+  } else {
+    pCoder->pos += sizeof(int32_t);
+  }
+  return 0;
+}
+
+void tEndEncode(SCoder* pCoder) {
+  struct SCoderNode* pNode;
+  int32_t            len;
+
+  ASSERT(pCoder->type == TD_ENCODER);
+  if (pCoder->data) {
+    pNode = TD_SLIST_HEAD(&(pCoder->stack));
+    ASSERT(pNode);
+    TD_SLIST_POP(&(pCoder->stack));
+
+    len = pCoder->pos;
+
+    pCoder->data = pNode->data;
+    pCoder->size = pNode->size;
+    pCoder->pos = pNode->pos;
+
+    if (TD_RT_ENDIAN() == pCoder->endian) {
+      tPut(int32_t, pCoder->data + pCoder->pos, len);
+    } else {
+      tRPut32(pCoder->data + pCoder->pos, len);
+    }
+
+    TD_CODER_MOVE_POS(pCoder, len + sizeof(int32_t));
+
+    free(pNode);
+  }
+}
+
+int tStartDecode(SCoder* pCoder) {
+  int32_t            len;
+  struct SCoderNode* pNode;
+
+  ASSERT(pCoder->type == TD_DECODER);
+  if (tDecodeI32(pCoder, &len) < 0) return -1;
+
+  pNode = malloc(sizeof(*pNode));
+  if (pNode == NULL) return -1;
+
+  pNode->data = pCoder->data;
+  pNode->pos = pCoder->pos;
+  pNode->size = pCoder->size;
+
+  pCoder->data = pNode->data + pNode->pos;
+  pCoder->size = len;
+  pCoder->pos = 0;
+
+  TD_SLIST_PUSH(&(pCoder->stack), pNode);
+
+  return 0;
+}
+
+void tEndDecode(SCoder* pCoder) {
+  struct SCoderNode* pNode;
+
+  ASSERT(pCoder->type == TD_DECODER);
+  ASSERT(tDecodeIsEnd(pCoder));
+
+  pNode = TD_SLIST_HEAD(&(pCoder->stack));
+  ASSERT(pNode);
+  TD_SLIST_POP(&(pCoder->stack));
+
+  pCoder->data = pNode->data;
+  pCoder->size = pNode->size;
+  pCoder->pos = pCoder->pos + pNode->pos;
+
+  free(pNode);
+}

source/util/src/thash.c

@@ -132,7 +132,7 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry*
   } else {
     pNewNode->next = pNode;    
     pe->num++;
-    atomic_add_fetch_64(&pHashObj->size, 1);
+    atomic_add_fetch_32(&pHashObj->size, 1);
   }
   
   return pNewNode;
@@ -209,7 +209,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
   if (!pHashObj) {
     return 0;
   }
-  return (int32_t)atomic_load_64(&pHashObj->size);
+  return (int32_t)atomic_load_32(&pHashObj->size);
 }
 
 static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) {
@@ -273,7 +273,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 
     // enable resize
     __rd_unlock(&pHashObj->lock, pHashObj->type);
-    atomic_add_fetch_64(&pHashObj->size, 1);
+    atomic_add_fetch_32(&pHashObj->size, 1);
 
     return 0;
   } else {
@@ -405,7 +405,7 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v
     }
 
     if (acquire) {
-      pNode->count++;
+      atomic_add_fetch_16(&pNode->count, 1);
     }
 
     data = GET_HASH_NODE_DATA(pNode);
@@ -482,7 +482,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi
 //      if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
 
       pe->num--;
-      atomic_sub_fetch_64(&pHashObj->size, 1);
+      atomic_sub_fetch_32(&pHashObj->size, 1);
       FREE_HASH_NODE(pHashObj, pNode);
     }
   }
@@ -520,7 +520,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
     while((pNode = pEntry->next) != NULL) {
       if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) {
         pEntry->num -= 1;
-        atomic_sub_fetch_64(&pHashObj->size, 1);
+        atomic_sub_fetch_32(&pHashObj->size, 1);
 
         pEntry->next = pNode->next;
 
@@ -546,7 +546,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
         if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) {
           pNode->next = pNext->next;
           pEntry->num -= 1;
-          atomic_sub_fetch_64(&pHashObj->size, 1);
+          atomic_sub_fetch_32(&pHashObj->size, 1);
 
           if (pEntry->num == 0) {
             assert(pEntry->next == NULL);
@@ -600,7 +600,7 @@ void taosHashClear(SHashObj *pHashObj) {
     pEntry->next = NULL;
   }
 
-  pHashObj->size = 0;
+  atomic_store_32(&pHashObj->size, 0);
   __wr_unlock(&pHashObj->lock, pHashObj->type);
 }
 
@@ -847,7 +847,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
       }
    
       pe->num--;
-      atomic_sub_fetch_64(&pHashObj->size, 1);
+      atomic_sub_fetch_32(&pHashObj->size, 1);
       FREE_HASH_NODE(pHashObj, pOld);
     } 
   } else {

source/util/test/CMakeLists.txt

@@ -33,4 +33,16 @@ ENDIF()
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 
+# freelistTest
+add_executable(freelistTest "")
+target_sources(freelistTest
+    PRIVATE
+    "freelistTest.cpp"
+)
+target_link_libraries(freelistTest os util gtest gtest_main)
+
+# encodeTest
+add_executable(encodeTest "encodeTest.cpp")
+target_link_libraries(encodeTest os util gtest gtest_main)
+
 

source/util/test/encodeTest.cpp

@@ -0,0 +1,421 @@
+#include <iostream>
+
+#include "gtest/gtest.h"
+
+#include "encode.h"
+
+#define BUF_SIZE 64
+td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN};
+
+static int encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); }
+static int encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); }
+static int encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); }
+static int encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); }
+static int encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); }
+static int encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); }
+static int encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); }
+static int encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); }
+
+static int decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); }
+static int decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); }
+static int decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); }
+static int decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); }
+static int decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); }
+static int decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); }
+static int decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); }
+static int decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); }
+
+static int encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); }
+static int encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); }
+static int encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); }
+static int encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); }
+static int encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); }
+static int encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); }
+static int encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); }
+static int encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); }
+
+static int decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); }
+static int decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); }
+static int decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); }
+static int decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); }
+static int decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); }
+static int decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); }
+static int decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); }
+static int decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); }
+
+template <typename T>
+static void simple_encode_decode_func(bool var_len) {
+  uint8_t buf[BUF_SIZE];
+  SCoder  coder;
+  T       min_val, max_val;
+  T       step = 1;
+
+  if (typeid(T) == typeid(int8_t)) {
+    min_val = INT8_MIN;
+    max_val = INT8_MAX;
+    step = 1;
+  } else if (typeid(T) == typeid(uint8_t)) {
+    min_val = 0;
+    max_val = UINT8_MAX;
+    step = 1;
+  } else if (typeid(T) == typeid(int16_t)) {
+    min_val = INT16_MIN;
+    max_val = INT16_MAX;
+    step = 1;
+  } else if (typeid(T) == typeid(uint16_t)) {
+    min_val = 0;
+    max_val = UINT16_MAX;
+    step = 1;
+  } else if (typeid(T) == typeid(int32_t)) {
+    min_val = INT32_MIN;
+    max_val = INT32_MAX;
+    step = ((T)1) << 16;
+  } else if (typeid(T) == typeid(uint32_t)) {
+    min_val = 0;
+    max_val = UINT32_MAX;
+    step = ((T)1) << 16;
+  } else if (typeid(T) == typeid(int64_t)) {
+    min_val = INT64_MIN;
+    max_val = INT64_MAX;
+    step = ((T)1) << 48;
+  } else if (typeid(T) == typeid(uint64_t)) {
+    min_val = 0;
+    max_val = UINT64_MAX;
+    step = ((T)1) << 48;
+  }
+
+  T i = min_val;
+  for (;; /*T i = min_val; i <= max_val; i += step*/) {
+    T dval;
+
+    // Encode NULL
+    for (td_endian_t endian : endian_arr) {
+      tCoderInit(&coder, endian, NULL, 0, TD_ENCODER);
+
+      if (var_len) {
+        GTEST_ASSERT_EQ(encodev(&coder, i), 0);
+      } else {
+        GTEST_ASSERT_EQ(encode(&coder, i), 0);
+        GTEST_ASSERT_EQ(coder.pos, sizeof(T));
+      }
+
+      tCoderClear(&coder);
+    }
+
+    // Encode and decode
+    for (td_endian_t e_endian : endian_arr) {
+      for (td_endian_t d_endian : endian_arr) {
+        // Encode
+        tCoderInit(&coder, e_endian, buf, BUF_SIZE, TD_ENCODER);
+
+        if (var_len) {
+          GTEST_ASSERT_EQ(encodev(&coder, i), 0);
+        } else {
+          GTEST_ASSERT_EQ(encode(&coder, i), 0);
+          GTEST_ASSERT_EQ(coder.pos, sizeof(T));
+        }
+
+        int32_t epos = coder.pos;
+
+        tCoderClear(&coder);
+        // Decode
+        tCoderInit(&coder, d_endian, buf, BUF_SIZE, TD_DECODER);
+
+        if (var_len) {
+          GTEST_ASSERT_EQ(decodev(&coder, &dval), 0);
+        } else {
+          GTEST_ASSERT_EQ(decode(&coder, &dval), 0);
+          GTEST_ASSERT_EQ(coder.pos, sizeof(T));
+        }
+
+        GTEST_ASSERT_EQ(coder.pos, epos);
+
+        if (typeid(T) == typeid(int8_t) || typeid(T) == typeid(uint8_t) || e_endian == d_endian) {
+          GTEST_ASSERT_EQ(i, dval);
+        }
+
+        tCoderClear(&coder);
+      }
+    }
+
+    if (i == max_val) break;
+
+    if (max_val - i < step) {
+      i = max_val;
+    } else {
+      i = i + step;
+    }
+  }
+}
+
+TEST(td_encode_test, encode_decode_fixed_len_integer) {
+  simple_encode_decode_func<int8_t>(false);
+  simple_encode_decode_func<uint8_t>(false);
+  simple_encode_decode_func<int16_t>(false);
+  simple_encode_decode_func<uint16_t>(false);
+  simple_encode_decode_func<int32_t>(false);
+  simple_encode_decode_func<uint32_t>(false);
+  simple_encode_decode_func<int64_t>(false);
+  simple_encode_decode_func<uint64_t>(false);
+}
+
+TEST(td_encode_test, encode_decode_variant_len_integer) {
+  simple_encode_decode_func<int16_t>(true);
+  simple_encode_decode_func<uint16_t>(true);
+  simple_encode_decode_func<int32_t>(true);
+  simple_encode_decode_func<uint32_t>(true);
+  simple_encode_decode_func<int64_t>(true);
+  simple_encode_decode_func<uint64_t>(true);
+}
+
+TEST(td_encode_test, encode_decode_cstr) {
+  uint8_t *   buf = new uint8_t[1024 * 1024];
+  char *      cstr = new char[1024 * 1024];
+  const char *dcstr;
+  SCoder      encoder;
+  SCoder      decoder;
+
+  for (size_t i = 0; i < 1024 * 2 - 1; i++) {
+    memset(cstr, 'a', i);
+    cstr[i] = '\0';
+    for (td_endian_t endian : endian_arr) {
+      // Encode
+      tCoderInit(&encoder, endian, buf, 1024 * 1024, TD_ENCODER);
+
+      GTEST_ASSERT_EQ(tEncodeCStr(&encoder, cstr), 0);
+
+      tCoderClear(&encoder);
+
+      // Decode
+      tCoderInit(&decoder, endian, buf, 1024 * 1024, TD_DECODER);
+
+      GTEST_ASSERT_EQ(tDecodeCStr(&decoder, &dcstr), 0);
+      GTEST_ASSERT_EQ(memcmp(dcstr, cstr, i + 1), 0);
+
+      tCoderClear(&decoder);
+    }
+  }
+
+  delete buf;
+  delete cstr;
+}
+
+typedef struct {
+  int32_t A_a;
+  int64_t A_b;
+  char *  A_c;
+} SStructA_v1;
+
+static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI32(pCoder, pSAV1->A_a) < 0) return -1;
+  if (tEncodeI64(pCoder, pSAV1->A_b) < 0) return -1;
+  if (tEncodeCStr(pCoder, pSAV1->A_c) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+static int tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI32(pCoder, &pSAV1->A_a) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSAV1->A_b) < 0) return -1;
+  const char *tstr;
+  uint64_t    len;
+  if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1;
+  pSAV1->A_c = (char *)TCODER_MALLOC(len + 1, pCoder);
+  memcpy(pSAV1->A_c, tstr, len + 1);
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+typedef struct {
+  int32_t A_a;
+  int64_t A_b;
+  char *  A_c;
+  // -------------------BELOW FEILDS ARE ADDED IN A NEW VERSION--------------
+  int16_t A_d;
+  int16_t A_e;
+} SStructA_v2;
+
+static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tEncodeI32(pCoder, pSAV2->A_a) < 0) return -1;
+  if (tEncodeI64(pCoder, pSAV2->A_b) < 0) return -1;
+  if (tEncodeCStr(pCoder, pSAV2->A_c) < 0) return -1;
+
+  // ------------------------NEW FIELDS ENCODE-------------------------------
+  if (tEncodeI16(pCoder, pSAV2->A_d) < 0) return -1;
+  if (tEncodeI16(pCoder, pSAV2->A_e) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+static int tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  if (tDecodeI32(pCoder, &pSAV2->A_a) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSAV2->A_b) < 0) return -1;
+  const char *tstr;
+  uint64_t    len;
+  if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1;
+  pSAV2->A_c = (char *)TCODER_MALLOC(len + 1, pCoder);
+  memcpy(pSAV2->A_c, tstr, len + 1);
+
+  // ------------------------NEW FIELDS DECODE-------------------------------
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI16(pCoder, &pSAV2->A_d) < 0) return -1;
+    if (tDecodeI16(pCoder, &pSAV2->A_e) < 0) return -1;
+  } else {
+    pSAV2->A_d = 0;
+    pSAV2->A_e = 0;
+  }
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+typedef struct {
+  SStructA_v1 *pA;
+  int32_t      v_a;
+  int8_t       v_b;
+} SFinalReq_v1;
+
+static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tSStructA_v1_encode(pCoder, ps1->pA) < 0) return -1;
+  if (tEncodeI32(pCoder, ps1->v_a) < 0) return -1;
+  if (tEncodeI8(pCoder, ps1->v_b) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+static int tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder);
+  if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1;
+  if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1;
+  if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1;
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+typedef struct {
+  SStructA_v2 *pA;
+  int32_t      v_a;
+  int8_t       v_b;
+  // ----------------------- Feilds added -----------------------
+  int16_t v_c;
+} SFinalReq_v2;
+
+static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) {
+  if (tStartEncode(pCoder) < 0) return -1;
+
+  if (tSStructA_v2_encode(pCoder, ps2->pA) < 0) return -1;
+  if (tEncodeI32(pCoder, ps2->v_a) < 0) return -1;
+  if (tEncodeI8(pCoder, ps2->v_b) < 0) return -1;
+
+  // ----------------------- Feilds added encode -----------------------
+  if (tEncodeI16(pCoder, ps2->v_c) < 0) return -1;
+
+  tEndEncode(pCoder);
+  return 0;
+}
+
+static int tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) {
+  if (tStartDecode(pCoder) < 0) return -1;
+
+  ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder);
+  if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1;
+  if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1;
+  if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1;
+
+  // ----------------------- Feilds added decode -----------------------
+  if (tDecodeIsEnd(pCoder)) {
+    ps2->v_c = 0;
+  } else {
+    if (tDecodeI16(pCoder, &ps2->v_c) < 0) return -1;
+  }
+
+  tEndDecode(pCoder);
+  return 0;
+}
+
+TEST(td_encode_test, compound_struct_encode_test) {
+  SCoder       encoder, decoder;
+  uint8_t *    buf1;
+  int32_t      buf1size;
+  uint8_t *    buf2;
+  int32_t      buf2size;
+  SStructA_v1  sa1 = {.A_a = 10, .A_b = 65478, .A_c = "Hello"};
+  SStructA_v2  sa2 = {.A_a = 10, .A_b = 65478, .A_c = "Hello", .A_d = 67, .A_e = 13};
+  SFinalReq_v1 req1 = {.pA = &sa1, .v_a = 15, .v_b = 35};
+  SFinalReq_v2 req2 = {.pA = &sa2, .v_a = 15, .v_b = 32, .v_c = 37};
+  SFinalReq_v1 dreq1;
+  SFinalReq_v2 dreq21, dreq22;
+
+  // Get size
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, nullptr, 0, TD_ENCODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v1_encode(&encoder, &req1), 0);
+  buf1size = encoder.pos;
+  buf1 = new uint8_t[encoder.pos];
+  tCoderClear(&encoder);
+
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, nullptr, 0, TD_ENCODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v2_encode(&encoder, &req2), 0);
+  buf2size = encoder.pos;
+  buf2 = new uint8_t[encoder.pos];
+  tCoderClear(&encoder);
+
+  // Encode
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_ENCODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v1_encode(&encoder, &req1), 0);
+  tCoderClear(&encoder);
+
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf2, buf2size, TD_ENCODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v2_encode(&encoder, &req2), 0);
+  tCoderClear(&encoder);
+
+  // Decode
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v1_decode(&decoder, &dreq1), 0);
+  GTEST_ASSERT_EQ(dreq1.pA->A_a, req1.pA->A_a);
+  GTEST_ASSERT_EQ(dreq1.pA->A_b, req1.pA->A_b);
+  GTEST_ASSERT_EQ(strcmp(dreq1.pA->A_c, req1.pA->A_c), 0);
+  GTEST_ASSERT_EQ(dreq1.v_a, req1.v_a);
+  GTEST_ASSERT_EQ(dreq1.v_b, req1.v_b);
+  tCoderClear(&decoder);
+
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq21), 0);
+  GTEST_ASSERT_EQ(dreq21.pA->A_a, req1.pA->A_a);
+  GTEST_ASSERT_EQ(dreq21.pA->A_b, req1.pA->A_b);
+  GTEST_ASSERT_EQ(strcmp(dreq21.pA->A_c, req1.pA->A_c), 0);
+  GTEST_ASSERT_EQ(dreq21.pA->A_d, 0);
+  GTEST_ASSERT_EQ(dreq21.pA->A_e, 0);
+  GTEST_ASSERT_EQ(dreq21.v_a, req1.v_a);
+  GTEST_ASSERT_EQ(dreq21.v_b, req1.v_b);
+  GTEST_ASSERT_EQ(dreq21.v_c, 0);
+  tCoderClear(&decoder);
+
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf2, buf2size, TD_DECODER);
+  GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq22), 0);
+  GTEST_ASSERT_EQ(dreq22.pA->A_a, req2.pA->A_a);
+  GTEST_ASSERT_EQ(dreq22.pA->A_b, req2.pA->A_b);
+  GTEST_ASSERT_EQ(strcmp(dreq22.pA->A_c, req2.pA->A_c), 0);
+  GTEST_ASSERT_EQ(dreq22.pA->A_d, req2.pA->A_d);
+  GTEST_ASSERT_EQ(dreq22.pA->A_e, req2.pA->A_e);
+  GTEST_ASSERT_EQ(dreq22.v_a, req2.v_a);
+  GTEST_ASSERT_EQ(dreq22.v_b, req2.v_b);
+  GTEST_ASSERT_EQ(dreq22.v_c, req2.v_c);
+  tCoderClear(&decoder);
+}

source/util/test/freelistTest.cpp

@@ -0,0 +1,16 @@
+#include "gtest/gtest.h"
+
+#include "freelist.h"
+
+TEST(TD_UTIL_FREELIST_TEST, simple_test) {
+  SFreeList fl;
+
+  tFreeListInit(&fl);
+
+  for (size_t i = 0; i < 1000; i++) {
+    void *ptr = TFL_MALLOC(1024, &fl);
+    GTEST_ASSERT_NE(ptr, nullptr);
+  }
+
+  tFreeListClear(&fl);
+}