[td-10564] connect to server success.

.gitignore

@@ -12,6 +12,7 @@ debug/
 release/
 target/
 debs/
+deps/
 rpms/
 mac/
 *.pyc

include/client/taos.h

@@ -75,7 +75,6 @@ typedef struct taosField {
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-DLL_EXPORT int   taos_init();
 DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);

include/common/taosmsg.h

@@ -74,10 +74,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" )	
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STB_VGROUP, "stb-vgroup" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" )	
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" )	
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" )
@@ -94,9 +94,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
 // message from vnode to dnode
 
 // message from mnode to vnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE_IN, "create-stable" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE_IN, "alter-stable" )	
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE_IN, "drop-stable" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" )
 // message from mnode to mnode
 // message from mnode to qnode
 // message from mnode to dnode
@@ -159,7 +159,7 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_DNODE,
   TSDB_MGMT_TABLE_MNODE,
   TSDB_MGMT_TABLE_VGROUP,
-  TSDB_MGMT_TABLE_STABLE,
+  TSDB_MGMT_TABLE_STB,
   TSDB_MGMT_TABLE_MODULE,
   TSDB_MGMT_TABLE_QUERIES,
   TSDB_MGMT_TABLE_STREAMS,
@@ -294,7 +294,7 @@ typedef struct {
   uint64_t superTableUid;
   uint64_t createdTime;
   char     tableFname[TSDB_TABLE_FNAME_LEN];
-  char     stableFname[TSDB_TABLE_FNAME_LEN];
+  char     stbFname[TSDB_TABLE_FNAME_LEN];
   char     data[];
 } SMDCreateTableMsg;
 
@@ -311,16 +311,23 @@ typedef struct {
 } SCreateTableMsg;
 
 typedef struct {
-  int32_t numOfTables;
+  char    name[TSDB_TABLE_FNAME_LEN];
-  int32_t contLen;
+  int8_t  igExists;
-} SCMCreateTableMsg;
+  int32_t numOfTags;
+  int32_t numOfColumns;
+  SSchema pSchema[];
+} SCreateStbMsg;
 
 typedef struct {
   char   name[TSDB_TABLE_FNAME_LEN];
-  // if user specify DROP STABLE, this flag will be set. And an error will be returned if it is not a super table
-  int8_t supertable;
   int8_t igNotExists;
-} SCMDropTableMsg;
+} SDropStbMsg;
+
+typedef struct {
+  char    name[TSDB_TABLE_FNAME_LEN];
+  int8_t  alterType;
+  SSchema schema;
+} SAlterStbMsg;
 
 typedef struct {
   char    tableFname[TSDB_TABLE_FNAME_LEN];
@@ -663,7 +670,6 @@ typedef struct {
 
 typedef struct {
   int32_t statusInterval;
-  int32_t mnodeEqualVnodeNum;
   int64_t checkTime;                    // 1970-01-01 00:00:00.000
   char    timezone[TSDB_TIMEZONE_LEN];  // tsTimezone
   char    locale[TSDB_LOCALE_LEN];      // tsLocale
@@ -689,7 +695,7 @@ typedef struct SStatusMsg {
   int32_t     sver;
   int32_t     dnodeId;
   int32_t     clusterId;
-  uint32_t    rebootTime;  // time stamp for last reboot
+  int64_t     rebootTime;  // time stamp for last reboot
   int16_t     numOfCores;
   int16_t     numOfSupportMnodes;
   int16_t     numOfSupportVnodes;
@@ -765,7 +771,7 @@ typedef struct {
 
 typedef struct {
   char name[TSDB_TABLE_FNAME_LEN];
-} SStableInfoMsg;
+} SStbInfoMsg;
 
 typedef struct {
   char   tableFname[TSDB_TABLE_FNAME_LEN];
@@ -797,8 +803,8 @@ typedef struct {
 } SVgroupsMsg, SVgroupsInfo;
 
 typedef struct {
-  char       tableFname[TSDB_TABLE_FNAME_LEN];  // table id
+  char       tbFname[TSDB_TABLE_FNAME_LEN];  // table id
-  char       stableFname[TSDB_TABLE_FNAME_LEN];
+  char       stbFname[TSDB_TABLE_FNAME_LEN];
   int32_t    numOfTags;
   int32_t    numOfColumns;
   int8_t     precision;
@@ -862,7 +868,7 @@ typedef struct {
 
 typedef struct {
   int32_t dnodeId;
-  char    config[128];
+  char    config[TSDB_DNODE_CONFIG_LEN];
 } SCfgDnodeMsg;
 
 typedef struct {

include/common/tdataformat.h

@@ -544,7 +544,7 @@ void   tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
+static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);

include/common/tep.h

@@ -1,6 +1,17 @@
 #ifndef TDENGINE_TEP_H
 #define TDENGINE_TEP_H
 
+#include "os.h"
+#include "taosmsg.h"
+
+typedef struct SCorEpSet {
+  int32_t version;
+  SEpSet  epSet;
+} SCorEpSet;
+
 int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
+bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
+
+void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
 
 #endif  // TDENGINE_TEP_H

include/common/tglobal.h

@@ -81,8 +81,6 @@ extern int64_t tsMaxRetentWindow;
 // db parameters in client
 extern int32_t tsCacheBlockSize;
 extern int32_t tsBlocksPerVnode;
-extern int32_t tsMinTablePerVnode;
-extern int32_t tsMaxTablePerVnode;
 extern int32_t tsTableIncStepPerVnode;
 extern int32_t tsMaxVgroupsPerDb;
 extern int16_t tsDaysPerFile;
@@ -108,7 +106,6 @@ extern int8_t  tsEnableBalance;
 extern int8_t  tsAlternativeRole;
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
-extern int32_t tsMnodeEqualVnodeNum;
 extern int8_t  tsEnableFlowCtrl;
 extern int8_t  tsEnableSlaveQuery;
 extern int8_t  tsEnableAdjustMaster;

include/common/ttime.h

@@ -44,10 +44,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
 int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
 
-int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
+int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
 int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
+int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);

include/common/tvariant.h

@@ -40,7 +40,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
 
 bool taosVariantIsValid(SVariant *pVar);
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type);
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type);
 
 void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
 

include/dnode/mgmt/dnode.h

@@ -33,7 +33,6 @@ typedef struct {
   int16_t  numOfSupportQnodes;
   int8_t   enableTelem;
   int32_t  statusInterval;
-  int32_t  mnodeEqualVnodeNum;
   float    numOfThreadsPerCore;
   float    ratioOfQueryCores;
   int32_t  maxShellConns;

include/dnode/mnode/mnode.h

@@ -47,7 +47,6 @@ typedef struct SMnodeCfg {
   int32_t sver;
   int8_t  enableTelem;
   int32_t statusInterval;
-  int32_t mnodeEqualVnodeNum;
   int32_t shellActivityTimer;
   char   *timezone;
   char   *locale;

include/dnode/mnode/sdb/sdb.h

@@ -158,8 +158,8 @@ typedef enum {
   SDB_USER = 5,
   SDB_AUTH = 6,
   SDB_ACCT = 7,
-  SDB_VGROUP = 9,
+  SDB_VGROUP = 8,
-  SDB_STABLE = 9,
+  SDB_STB = 9,
   SDB_DB = 10,
   SDB_FUNC = 11,
   SDB_MAX = 12

include/libs/parser/parser.h

@@ -131,6 +131,18 @@ struct SInsertStmtInfo;
  */
 bool qIsInsertSql(const char* pStr, size_t length);
 
+typedef struct SParseContext {
+  const char* pSql;    // sql string
+  size_t sqlLen;       // length of the sql string
+  int64_t id;          // operator id, generated by uuid generator
+  const char* pDbname;
+  const SEpSet* pEpSet; 
+  int8_t schemaAttached; // denote if submit block is built with table schema or not
+
+  char* pMsg;           // extended error message if exists to help avoid the problem in sql statement.
+  int32_t msgLen;      // max length of the msg
+} SParseContext;
+
 /**
  * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST.
  * @param pSql     sql string
@@ -141,16 +153,35 @@ bool qIsInsertSql(const char* pStr, size_t length);
  */
 int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen);
 
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+typedef struct SVgDataBlocks {
+  int64_t     vgId;         // virtual group id
+  int32_t     numOfTables;  // number of tables in current submit block
+  uint32_t    size;
+  char       *pData;
+} SVgDataBlocks;
+
+typedef struct SInsertStmtInfo {
+  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  int8_t      schemaAttache;        // denote if submit block is built with table schema or not
+  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  const char* sql;                 // current sql statement position
+} SInsertStmtInfo;
+
 /**
  * Parse the insert sql statement.
  * @param pStr            sql string
  * @param length          length of the sql string
- * @param pInsertParam    data in binary format to submit to vnode directly.
  * @param id              operator id, generated by uuid generator.
  * @param msg             extended error message if exists to help avoid the problem in sql statement.
- * @return
+ * @return                data in binary format to submit to vnode directly.
  */
-int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen);
+ int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo);
 
 /**
  * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that

include/libs/transport/trpc.h

@@ -29,11 +29,6 @@ extern "C" {
 
 extern int tsRpcHeadSize;
 
-typedef struct SRpcCorEpSet {
-  int32_t version; 
-  SEpSet epSet; 
-} SRpcCorEpSet;
-
 typedef struct SRpcConnInfo {
   uint32_t  clientIp;
   uint16_t  clientPort;

include/libs/wal/wal.h

@@ -38,6 +38,25 @@ typedef enum {
   TAOS_WAL_FSYNC = 2
 } EWalType;
 
+typedef struct SWalReadHead {
+  int8_t   sver;
+  uint8_t  msgType;
+  int8_t   reserved[2];
+  int32_t  len;
+  int64_t  version;
+  char     cont[];
+} SWalReadHead;
+
+typedef struct {
+  int32_t  vgId;
+  int32_t  fsyncPeriod;      // millisecond
+  int32_t  retentionPeriod;  // secs
+  int32_t  rollPeriod;       // secs
+  int32_t  retentionSize;       // secs
+  int64_t  segSize;
+  EWalType walLevel;         // wal level
+} SWalCfg;
+
 typedef struct {
   //union {
     //uint32_t info;
@@ -47,25 +66,11 @@ typedef struct {
       //uint32_t reserved : 24;
     //};
   //};
-  int8_t   sver;
-  uint8_t  msgType;
-  int8_t   reserved[2];
-  int32_t  len;
-  int64_t  version;
-  uint32_t signature;
   uint32_t cksumHead;
   uint32_t cksumBody;
-  char     cont[];
+  SWalReadHead head;
 } SWalHead;
 
-typedef struct {
-  int32_t  vgId;
-  int32_t  fsyncPeriod;  // millisecond
-  int32_t  rollPeriod;
-  int64_t  segSize;
-  EWalType walLevel;     // wal level
-} SWalCfg;
-
 #define WAL_PREFIX       "wal"
 #define WAL_PREFIX_LEN   3
 #define WAL_NOSUFFIX_LEN 20
@@ -103,21 +108,17 @@ typedef struct SWal {
   //write tfd
   int64_t writeLogTfd;
   int64_t writeIdxTfd;
-  //read tfd
-  int64_t readLogTfd;
-  int64_t readIdxTfd;
-  //current version
-  int64_t curVersion;
   //wal lifecycle
   int64_t firstVersion;
   int64_t snapshotVersion;
   int64_t commitVersion;
   int64_t lastVersion;
+  //snapshotting version
+  int64_t snapshottingVer;
   //roll status
   int64_t lastRollSeq;
   //file set
   int32_t writeCur;
-  int32_t readCur;
   SArray* fileInfoSet;
   //ctl
   int32_t curStatus;
@@ -148,7 +149,8 @@ int32_t walCommit(SWal *, int64_t ver);
 // truncate after
 int32_t walRollback(SWal *, int64_t ver);
 // notify that previous logs can be pruned safely
-int32_t walTakeSnapshot(SWal *, int64_t ver);
+int32_t walBeginTakeSnapshot(SWal *, int64_t ver);
+int32_t walEndTakeSnapshot(SWal *);
 //int32_t  walDataCorrupted(SWal*);
 
 // read

include/os/osString.h

@@ -38,11 +38,11 @@ extern "C" {
     (dst)[(size)-1] = 0;            \
   } while (0)
 
-int64_t taosStr2int64(char *str);
+int64_t taosStr2int64(const char *str);
 
 // USE_LIBICONV
 int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
+bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
 char *  taosCharsetReplace(char *charsetstr);

include/util/taoserror.h

@@ -152,22 +152,23 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SDB_INVALID_DATA_LEN          TAOS_DEF_ERROR_CODE(0, 0x032A)
 #define TSDB_CODE_SDB_INVALID_DATA_CONTENT      TAOS_DEF_ERROR_CODE(0, 0x032B)
 
-#define TSDB_CODE_MND_DNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0330)  //"DNode already exists")
+// mnode-dnode
-#define TSDB_CODE_MND_DNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0331)  //"DNode does not exist")
+#define TSDB_CODE_MND_DNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0330)
+#define TSDB_CODE_MND_DNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0331)
+#define TSDB_CODE_MND_NO_ENOUGH_DNODES          TAOS_DEF_ERROR_CODE(0, 0x0332)
+#define TSDB_CODE_MND_INVALID_CLUSTER_CFG       TAOS_DEF_ERROR_CODE(0, 0x0333)
+#define TSDB_CODE_MND_INVALID_CLUSTER_ID        TAOS_DEF_ERROR_CODE(0, 0x0334)
+#define TSDB_CODE_MND_INVALID_DNODE_CFG         TAOS_DEF_ERROR_CODE(0, 0x0335)
+#define TSDB_CODE_MND_INVALID_DNODE_EP          TAOS_DEF_ERROR_CODE(0, 0x0336)
+#define TSDB_CODE_MND_INVALID_DNODE_ID          TAOS_DEF_ERROR_CODE(0, 0x0337)
+
+// mnode-vgroup
 #define TSDB_CODE_MND_VGROUP_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x0332)  //"VGroup does not exist")
-#define TSDB_CODE_MND_NO_REMOVE_MASTER          TAOS_DEF_ERROR_CODE(0, 0x0333)  //"Master DNode cannot be removed")
-#define TSDB_CODE_MND_NO_ENOUGH_DNODES          TAOS_DEF_ERROR_CODE(0, 0x0334)  //"Out of DNodes")
-#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT  TAOS_DEF_ERROR_CODE(0, 0x0335)  //"Cluster cfg inconsistent")
-#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION  TAOS_DEF_ERROR_CODE(0, 0x0336)  //"Invalid dnode cfg option")
-#define TSDB_CODE_MND_BALANCE_ENABLED           TAOS_DEF_ERROR_CODE(0, 0x0337)  //"Balance already enabled")
 #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE       TAOS_DEF_ERROR_CODE(0, 0x0338)  //"Vgroup not in dnode")
 #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE   TAOS_DEF_ERROR_CODE(0, 0x0339)  //"Vgroup already in dnode")
-#define TSDB_CODE_MND_DNODE_NOT_FREE            TAOS_DEF_ERROR_CODE(0, 0x033A)  //"Dnode not avaliable")
-#define TSDB_CODE_MND_INVALID_CLUSTER_ID        TAOS_DEF_ERROR_CODE(0, 0x033B)  //"Cluster id not match")
 #define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x033C)  //"Cluster not ready")
-#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED   TAOS_DEF_ERROR_CODE(0, 0x033D)  //"Dnode Id not configured")
-#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED   TAOS_DEF_ERROR_CODE(0, 0x033E)  //"Dnode Ep not configured")
 
+// mnode-acct
 #define TSDB_CODE_MND_ACCT_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x0340)  //"Account already exists")
 #define TSDB_CODE_MND_ACCT_NOT_EXIST            TAOS_DEF_ERROR_CODE(0, 0x0341)  //"Invalid account")
 #define TSDB_CODE_MND_INVALID_ACCT_OPTION       TAOS_DEF_ERROR_CODE(0, 0x0342)  //"Invalid account options")
@@ -183,8 +184,18 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_MNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0348)  //"Mnode already exists")
 #define TSDB_CODE_MND_MNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0349)  //"Mnode not there")
 
-// mnode-table
+// mnode-stable
-#define TSDB_CODE_MND_TABLE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0360)  //"Table already exists")
+#define TSDB_CODE_MND_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_IGEXIST       TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COLS_NUM      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_TYPE      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_ID        TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_BYTES     TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_NAME      TAOS_DEF_ERROR_CODE(0, 0x0360)
+
+
 #define TSDB_CODE_MND_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0361)  //"Table name too long")
 #define TSDB_CODE_MND_INVALID_TABLE_NAME        TAOS_DEF_ERROR_CODE(0, 0x0362)  //"Table does not exist")
 #define TSDB_CODE_MND_INVALID_TABLE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x0363)  //"Invalid table type in tsdb")

include/util/tarray.h

@@ -43,6 +43,13 @@ typedef struct SArray {
  */
 void* taosArrayInit(size_t size, size_t elemSize);
 
+/**
+ *
+ * @param tsize
+ * @return
+ */
+int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize);
+
 /**
  *
  * @param pArray
@@ -153,6 +160,13 @@ void taosArraySet(SArray* pArray, size_t index, void* pData);
  */
 void taosArrayPopFrontBatch(SArray* pArray, size_t cnt);
 
+/**
+ * remove some data entry from front
+ * @param pArray
+ * @param cnt
+ */
+void taosArrayPopTailBatch(SArray* pArray, size_t cnt);
+
 /**
  * remove data entry of the given index
  * @param pArray
@@ -213,6 +227,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
  */
 void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
 
+/**
+ * search the array, return index of the element
+ * @param pArray
+ * @param compar
+ * @param key
+ */
+int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
+
 /**
  * search the array
  * @param pArray

include/util/tchecksum.h

@@ -39,7 +39,7 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin
 }
 
 static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) {
-  return (checksum == (*crc32c)(0, stream, (size_t)ssize));
+  return (checksum != (*crc32c)(0, stream, (size_t)ssize));
 }
 
 static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) {

include/util/tconfig.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define TSDB_CFG_MAX_NUM    119
+#define TSDB_CFG_MAX_NUM    115
 #define TSDB_CFG_PRINT_LEN  23
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41

include/util/tdef.h

@@ -193,7 +193,7 @@ do { \
 #define TSDB_MAX_TAG_CONDITIONS   1024
 
 #define TSDB_AUTH_LEN             16
-#define TSDB_PASSWORD_LEN         64
+#define TSDB_PASSWORD_LEN         32
 #define TSDB_VERSION_LEN          12
 #define TSDB_LABEL_LEN            8 
 
@@ -209,6 +209,8 @@ do { \
 #define TSDB_STEP_NAME_LEN        32
 #define TSDB_STEP_DESC_LEN        128
 
+#define TSDB_DNODE_CONFIG_LEN     128
+
 #define TSDB_MQTT_HOSTNAME_LEN    64
 #define TSDB_MQTT_PORT_LEN        8
 #define TSDB_MQTT_USER_LEN        24

include/util/tutil.h

@@ -29,7 +29,7 @@ int32_t strdequote(char *src);
 int32_t strndequote(char *dst, const char *z, int32_t len);
 int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
-char   *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);
 char   *strtolower(char *dst, const char *src);
 char   *strntolower(char *dst, const char *src, int32_t n);
@@ -60,7 +60,7 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
   tMD5Update(&context, inBuf, (unsigned int)len);
   tMD5Final(&context);
 
-  sprintf(target, "%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x", context.digest[0], context.digest[1], context.digest[2],
+  sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2],
           context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7],
           context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12],
           context.digest[13], context.digest[14], context.digest[15]);

source/client/inc/clientInt.h

@@ -26,6 +26,8 @@ extern "C" {
 #include "tlist.h"
 #include "trpc.h"
 #include "tdef.h"
+#include "tmsgtype.h"
+#include "tep.h"
 
 typedef struct SQueryExecMetric {
   int64_t      start;    // start timestamp
@@ -53,10 +55,11 @@ typedef struct SHeartBeatInfo {
 
 typedef struct SAppInstInfo {
   int64_t           numOfConns;
-  SRpcCorEpSet      mgmtEp;
+  SCorEpSet         mgmtEp;
   SInstanceActivity summary;
   SList            *pConnList;  // STscObj linked list
   char              clusterId[TSDB_CLUSTER_ID_LEN];
+  void             *pTransporter;
 } SAppInstInfo;
 
 typedef struct SAppInfo {
@@ -77,7 +80,7 @@ typedef struct STscObj {
   uint32_t           connId;
   uint64_t           id;       // ref ID returned by taosAddRef
 //  struct SSqlObj    *sqlList;
-//  SRpcObj           *pRpcObj;
+  void              *pTransporter;
   pthread_mutex_t    mutex;     // used to protect the operation on db
   int32_t            numOfReqs; // number of sqlObj from this tscObj
   SAppInstInfo      *pAppInfo;
@@ -102,13 +105,27 @@ typedef struct SRequestObj {
   void            *pInfo;   // sql parse info, generated by parser module
 } SRequestObj;
 
+typedef struct SRequestMsgBody {
+  int32_t  msgType;
+  void    *pData;
+  int32_t  msgLen;
+  uint64_t requestId;
+  uint64_t requestObjRefId;
+} SRequestMsgBody;
+
+extern SAppInfo   appInfo;
 extern int32_t    tscReqRef;
 extern void      *tscQhandle;
 extern int32_t    tscConnRef;
 extern void      *tscRpcCache;
 extern pthread_mutex_t rpcObjMutex;
 
-void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port);
+extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg);
+extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
+
+int   taos_init();
+
+void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo);
 void  destroyTscObj(void* pTscObj);
 
 void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
@@ -119,6 +136,11 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
 void taos_init_imp(void);
 int taos_options_imp(TSDB_OPTION option, const char *str);
 
+void* openTransporter(const char *user, const char *auth);
+
+void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
+void initMsgHandleFp();
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientImpl.c

@@ -1,12 +1,15 @@
-#include "tglobal.h"
+#include <tpagedfile.h>
 #include "clientInt.h"
 #include "tdef.h"
 #include "tep.h"
+#include "tglobal.h"
 #include "tmsgtype.h"
 #include "tref.h"
 #include "tscLog.h"
 
-static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet);
+static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
+static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody);
+static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId);
 
 static bool stringLengthCheck(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -33,9 +36,19 @@ static bool validateDbName(const char* db) {
   return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1);
 }
 
-static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param);
+static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) {
+  char key[512] = {0};
+  snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port);
+  return strdup(key);
+}
+
+static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
 
 TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
+  if (taos_init() != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+
   if (!validateUserName(user)) {
     terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH;
     return NULL;
@@ -52,7 +65,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
     strdequote(tmp);
   }
 
-  char secretEncrypt[64] = {0};
+  char secretEncrypt[32] = {0};
   if (auth == NULL) {
     if (!validatePassword(pass)) {
       terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH;
@@ -64,7 +77,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
     tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt));
   }
 
-  SRpcCorEpSet epSet;
+  SCorEpSet epSet = {0};
   if (ip) {
     if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
       return NULL;
@@ -79,10 +92,22 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
     }
   }
 
-  return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL);
+  char* key = getClusterKey(user, secretEncrypt, ip, port);
+
+  SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
+  if (pInst == NULL) {
+    pInst = calloc(1, sizeof(struct SAppInstInfo));
+
+    pInst->mgmtEp       = epSet;
+    pInst->pTransporter = openTransporter(user, secretEncrypt);
+
+    taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES);
+  }
+
+  return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst);
 }
 
-int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet) {
+int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) {
   pEpSet->version = 0;
 
   // init mgmt ip set
@@ -118,48 +143,148 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pE
   return 0;
 }
 
-STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param) {
+STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
-  if (taos_init() != TSDB_CODE_SUCCESS) {
+  STscObj *pTscObj = createTscObj(user, auth, ip, port, pAppInfo);
-    return NULL;
+  if (NULL == pTscObj) {
-  }
-
-  STscObj *pObj = createTscObj(user, auth, ip, port);
-  if (NULL == pObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    return pObj;
+    return pTscObj;
   }
 
-  //  void *pRpcObj = NULL;
+  SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT);
-  //
-  //  char rpcKey[512] = {0};
-  //  snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port);
-  //  if (tscAcquireRpc(rpcKey, user, auth, &pRpcObj) != 0) {
-  //    terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-  //    return NULL;
-  //  }
-
-  SRequestObj *pRequest = createRequest(pObj, fp, param, TSDB_SQL_CONNECT);
   if (pRequest == NULL) {
-    destroyTscObj(pObj);
+    destroyTscObj(pTscObj);
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-  }
-
-  //    tscBuildAndSendRequest(pRequest, NULL);
-  //    tsem_wait(&pRequest->body.rspSem);
-  if (pRequest->code != TSDB_CODE_SUCCESS) {
-    const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)
-                               ? taos_errstr(pRequest)
-                               : tstrerror(terrno);
-
-    printf("connect failed, reason: %s\n\n", errorMsg);
-
-    taos_free_result(pRequest);
-    taos_close(pObj);
     return NULL;
   }
 
-//  tscDebug("0x%"PRIx64" connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj,
+  SRequestMsgBody body = {0};
-//      pObj->pRpcObj->pDnodeConn);
+  buildConnectMsg(pRequest, &body);
+
+  int64_t transporterId = 0;
+  sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
+
+  tsem_wait(&pRequest->body.rspSem);
+  if (pRequest->code != TSDB_CODE_SUCCESS) {
+    const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
+    printf("failed to connect to server, reason: %s\n\n", errorMsg);
+
     destroyRequest(pRequest);
-  return pObj;
+    taos_close(pTscObj);
+    pTscObj = NULL;
+  } else {
+    tscDebug("%p connection is opening, dnodeConn:%p", pTscObj, pTscObj->pTransporter);
+    destroyRequest(pRequest);
+  }
+
+  return pTscObj;
+}
+
+static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
+  pMsgBody->msgType         = TSDB_MSG_TYPE_CONNECT;
+  pMsgBody->msgLen          = sizeof(SConnectMsg);
+  pMsgBody->requestObjRefId = pRequest->self;
+
+  SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
+  if (pConnect == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  // TODO refactor full_name
+  char *db;  // ugly code to move the space
+
+  STscObj *pObj = pRequest->pTscObj;
+  pthread_mutex_lock(&pObj->mutex);
+  db = strstr(pObj->db, TS_PATH_DELIMITER);
+
+  db = (db == NULL) ? pObj->db : db + 1;
+  tstrncpy(pConnect->db, db, sizeof(pConnect->db));
+  pthread_mutex_unlock(&pObj->mutex);
+
+//  tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion));
+//  tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion));
+
+//  pConnect->pid = htonl(taosGetPId());
+//  taosGetCurrentAPPName(pConnect->appName, NULL);
+  pMsgBody->pData = pConnect;
+  return 0;
+}
+
+int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) {
+  char *pMsg = rpcMallocCont(pBody->msgLen);
+  if (NULL == pMsg) {
+    tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  memcpy(pMsg, pBody->pData, pBody->msgLen);
+  SRpcMsg rpcMsg = {
+      .msgType = pBody->msgType,
+      .pCont   = pMsg,
+      .contLen = pBody->msgLen,
+      .ahandle = (void*) pBody->requestObjRefId,
+      .handle  = NULL,
+      .code    = 0
+  };
+
+  rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId);
+  return TSDB_CODE_SUCCESS;
+}
+
+//
+//int tscBuildAndSendRequest(SRequestObj *pRequest) {
+//  assert(pRequest != NULL);
+//  char name[TSDB_TABLE_FNAME_LEN] = {0};
+//
+//  uint32_t type = 0;
+//  tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pRequest->requestId, taosMsg[pRequest->type], name, type);
+//  if (pRequest->type < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL
+//
+//  } else if (pCmd->command >= TSDB_SQL_LOCAL) {
+//    return (*tscProcessMsgRsp[pCmd->command])(pSql);
+//  }
+//
+//  return buildConnectMsg(pRequest);
+//}
+
+
+void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
+  int64_t requestRefId = (int64_t)pMsg->ahandle;
+
+  SRequestObj *pRequest = (SRequestObj *)taosAcquireRef(tscReqRef, requestRefId);
+  if (pRequest == NULL) {
+    rpcFreeCont(pMsg->pCont);
+    return;
+  }
+
+  assert(pRequest->self == requestRefId);
+  pRequest->metric.rsp = taosGetTimestampMs();
+
+  pRequest->code = pMsg->code;
+
+  STscObj *pTscObj = pRequest->pTscObj;
+  if (pEpSet) {
+    if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, pEpSet)) {
+      updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, pEpSet);
+    }
+  }
+
+  /*
+   * There is not response callback function for submit response.
+   * The actual inserted number of points is the first number.
+   */
+  if (pMsg->code == TSDB_CODE_SUCCESS) {
+    tscDebug("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pRequest->type],
+             tstrerror(pMsg->code), pMsg->contLen);
+    if (handleRequestRspFp[pRequest->type]) {
+      pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen);
+    }
+  } else {
+    tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pRequest->type],
+             tstrerror(pMsg->code), pMsg->contLen);
+  }
+
+  taosReleaseRef(requestRefId, requestRefId);
+  rpcFreeCont(pMsg->pCont);
 }

source/client/src/tscEnv.c

@@ -36,11 +36,13 @@ int32_t    tscConnRef = -1;
 void      *tscQhandle = NULL;
 void      *tscRpcCache= NULL;            // TODO removed from here.
 
+int32_t tsNumOfThreads = 1;
+
 pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently
 volatile int32_t tscInitRes = 0;
 
 static void registerRequest(SRequestObj* pRequest) {
-  STscObj*pTscObj = (STscObj*) taosAcquireRef(tscConnRef, pRequest->pTscObj->id);
+  STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id);
   assert(pTscObj != NULL);
 
   // connection has been released already, abort creating request.
@@ -48,11 +50,14 @@ static void registerRequest(SRequestObj* pRequest) {
 
   int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1);
 
-  SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary;
+  if (pTscObj->pAppInfo) {
+    SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary;
+
     int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1);
     int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1);
-
+    tscDebug("0x%" PRIx64 " new Request from 0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self,
-  tscDebug("0x%"PRIx64" new Request from 0x%"PRIx64", current:%d, app current:%d, total:%d", pRequest->self, pRequest->pTscObj->id, num, currentInst, total);
+             pRequest->pTscObj->id, num, currentInst, total);
+  }
 }
 
 static void deregisterRequest(SRequestObj* pRequest) {
@@ -103,80 +108,52 @@ void tscReleaseRpc(void *param)  {
   taosCacheRelease(tscRpcCache, (void *)&param, false);
 }
 
-void* tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt) {
+// TODO refactor
-#if 0
+void* openTransporter(const char *user, const char *auth) {
-  SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key));
-  pthread_mutex_lock(&rpcObjMutex);
-  if (pRpcObj != NULL) {
-    pthread_mutex_unlock(&rpcObjMutex);
-    return pRpcObj;
-  }
-
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort = 0;
   rpcInit.label = "TSC";
-  rpcInit.numOfThreads = tscNumOfThreads;
+  rpcInit.numOfThreads = tsNumOfThreads;
-  rpcInit.cfp = tscProcessMsgFromServer;
+  rpcInit.cfp = processMsgFromServer;
   rpcInit.sessions = tsMaxConnections;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
   rpcInit.idleTime = tsShellActivityTimer * 1000;
   rpcInit.ckey = "key";
-  rpcInit.spi = 1;
+//  rpcInit.spi = 1;
-  rpcInit.secret = (char *)secretEncrypt;
+  rpcInit.secret = (char *)auth;
 
-  SRpcObj rpcObj = {0};
+  void* pDnodeConn = rpcOpen(&rpcInit);
-  strncpy(rpcObj.key, key, strlen(key));
+  if (pDnodeConn == NULL) {
-  rpcObj.pDnodeConn = rpcOpen(&rpcInit);
-  if (rpcObj.pDnodeConn == NULL) {
-    pthread_mutex_unlock(&rpcObjMutex);
     tscError("failed to init connection to server");
     return NULL;
   }
 
-  pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5);
+  return pDnodeConn;
-  if (pRpcObj == NULL) {
-    rpcClose(rpcObj.pDnodeConn);
-    pthread_mutex_unlock(&rpcObjMutex);
-    return NULL;
-  }
-
-  pthread_mutex_unlock(&rpcObjMutex);
-  return pRpcObj;
-#endif
 }
 
 void destroyTscObj(void *pTscObj) {
   STscObj *pObj = pTscObj;
-//  tfree(pObj->tscCorMgmtEpSet);
 //  tscReleaseRpc(pObj->pRpcObj);
   pthread_mutex_destroy(&pObj->mutex);
   tfree(pObj);
 }
 
-void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port) {
+void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) {
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
-//  char rpcKey[512] = {0};
+  pObj->pAppInfo = pAppInfo;
-//  snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port);
+  if (pAppInfo != NULL) {
-
+    pObj->pTransporter = pAppInfo->pTransporter;
-//  pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet));
+  }
-//  if (pObj->tscCorMgmtEpSet == NULL) {
-//    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-//    free(pObj);
-//    return NULL;
-//  }
-//
-//  memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet));
 
   tstrncpy(pObj->user, user, sizeof(pObj->user));
-  int32_t len = MIN(strlen(auth) + 1, sizeof(pObj->pass));
+  memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
-  tstrncpy(pObj->pass, auth, len);
 
   pthread_mutex_init(&pObj->mutex, NULL);
   pObj->id = taosAddRef(tscConnRef, pObj);
@@ -194,6 +171,8 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
   // TODO generated request uuid
   pRequest->requestId  = 0;
 
+  pRequest->metric.start = taosGetTimestampMs();
+
   pRequest->type       = type;
   pRequest->pTscObj    = pObj;
   pRequest->body.fp    = fp;
@@ -201,6 +180,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
+  return pRequest;
 }
 
 void destroyRequest(void* p) {
@@ -235,6 +215,8 @@ void taos_init_imp(void) {
   }
 
   taosInitNotes();
+  initMsgHandleFp();
+
   rpcInit();
 
   tscDebug("starting to initialize TAOS client ...\nLocal End Point is:%s", tsLocalEp);
@@ -264,6 +246,7 @@ void taos_init_imp(void) {
   taosGetAppName(appInfo.appName, NULL);
   appInfo.pid       = taosGetPId();
   appInfo.startTime = taosGetTimestampMs();
+  appInfo.pInstMap  = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
 
   tscDebug("client is initialized successfully");
 }

source/client/test/clientTests.cpp

@@ -33,6 +33,5 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) {
-  taos_init();
   TAOS* pTaos = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
 }

source/common/src/tep.c

@@ -1,6 +1,6 @@
-#include "os.h"
 #include "tep.h"
 #include "tglobal.h"
+#include "tlockfree.h"
 
 int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
   *port = 0;
@@ -20,3 +20,22 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
   return 0;
 }
 
+bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) {
+  if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) {
+    return false;
+  }
+
+  for (int32_t i = 0; i < s1->numOfEps; i++) {
+    if (s1->port[i] != s2->port[i]
+        || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0)
+      return false;
+  }
+  return true;
+}
+
+void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) {
+  taosCorBeginWrite(&pEpSet->version);
+  pEpSet->epSet = *pNewEpSet;
+  taosCorEndWrite(&pEpSet->version);
+}
+

source/common/src/tglobal.c

@@ -47,7 +47,7 @@ int64_t  tsDnodeStartTime = 0;
 // common
 int32_t tsRpcTimer       = 300;
 int32_t tsRpcMaxTime     = 600;  // seconds;
-int32_t tsRpcForceTcp    = 0;  //disable this, means query, show command use udp protocol as default
+int32_t tsRpcForceTcp    = 1;  //disable this, means query, show command use udp protocol as default
 int32_t tsMaxShellConns  = 50000;
 int32_t tsMaxConnections = 5000;
 int32_t tsShellActivityTimer  = 3;  // second
@@ -157,7 +157,6 @@ int8_t  tsEnableBalance = 1;
 int8_t  tsAlternativeRole = 0;
 int32_t tsBalanceInterval = 300;           // seconds
 int32_t tsOfflineThreshold = 86400 * 10;  // seconds of 10 days
-int32_t tsMnodeEqualVnodeNum = 4;
 int8_t  tsEnableFlowCtrl = 1;
 int8_t  tsEnableSlaveQuery = 1;
 int8_t  tsEnableAdjustMaster = 1;
@@ -1063,17 +1062,6 @@ static void doInitGlobalConfig(void) {
   cfg.maxValue = 10000000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_GB;
-  taosAddConfigOption(cfg);
-
-  // module configs
-  cfg.option = "mnodeEqualVnodeNum";
-  cfg.ptr = &tsMnodeEqualVnodeNum;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 0;
-  cfg.maxValue = 1000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosAddConfigOption(cfg);
 
     // module configs
@@ -1546,7 +1534,7 @@ static void doInitGlobalConfig(void) {
   taosAddConfigOption(cfg);
   assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM);
 #else
-  assert(tsGlobalConfigNum == (TSDB_CFG_MAX_NUM - 5));
+  //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5);
 #endif
 
 }
@@ -1603,12 +1591,6 @@ int32_t taosCheckAndPrintCfg() {
     tsNumOfCores = 1;
   }
 
-  if (tsMaxTablePerVnode < tsMinTablePerVnode) {
-    uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)",
-	   tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode);
-    tsMaxTablePerVnode = tsMinTablePerVnode;
-  }
-
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
   }

source/common/src/ttime.c

@@ -82,18 +82,18 @@ void deltaToUtcInitOnce() {
 }
 
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);
-static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim);
+static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
 static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
 static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec);
 static char* forwardToTimeStringEnd(char* str);
-static bool checkTzPresent(char *str, int32_t len);
+static bool checkTzPresent(const char *str, int32_t len);
 
 static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = {
   parseLocaltime,
   parseLocaltimeDst
 };
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
+int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {
     return parseTimeWithTz(timestr, time, timePrec, 'T');
@@ -104,7 +104,7 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre
   }
 }
 
-bool checkTzPresent(char *str, int32_t len) {
+bool checkTzPresent(const char *str, int32_t len) {
   char *seg = forwardToTimeStringEnd(str);
   int32_t seg_len = len - (int32_t)(seg - str);
 
@@ -237,7 +237,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
  * 2013-04-12T15:52:01+0800
  * 2013-04-12T15:52:01.123+0800
  */
-int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim) {
+int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
 
   int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
                              (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
@@ -432,7 +432,7 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
  * d - Days (24 hours)
  * w - Weeks (7 days)
  */
-int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
+int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
   errno = 0;
   char* endPtr = NULL;
 

source/common/src/tvariant.c

@@ -75,7 +75,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
   return 0;
 }
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
   int32_t ret = 0;
   memset(pVar, 0, sizeof(SVariant));
 

source/dnode/mgmt/daemon/src/daemon.c

@@ -112,7 +112,7 @@ int dmnReadConfig(const char *path) {
     return -1;
   }
 
-  if (taosReadGlobalCfg() != 0) {
+  if (taosReadCfgFromFile() != 0) {
     uError("failed to read global config");
     return -1;
   }
@@ -142,7 +142,6 @@ void dmnInitOption(SDnodeOpt *pOption) {
   pOption->numOfSupportVnodes = 1;
   pOption->numOfSupportQnodes = 1;
   pOption->statusInterval = tsStatusInterval;
-  pOption->mnodeEqualVnodeNum = tsMnodeEqualVnodeNum;
   pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore;
   pOption->ratioOfQueryCores = tsRatioOfQueryCores;
   pOption->maxShellConns = tsMaxShellConns;

source/dnode/mgmt/impl/inc/dndDnode.h

@@ -31,6 +31,7 @@ int32_t dndGetClusterId(SDnode *pDnode);
 void    dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
 void    dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet);
 void    dndSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg);
+void    dndSendStatusMsg(SDnode *pDnode);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/impl/inc/dndInt.h

@@ -58,7 +58,8 @@ typedef struct {
   int32_t    dnodeId;
   int32_t    dropped;
   int32_t    clusterId;
-  uint32_t   rebootTime;
+  int64_t    rebootTime;
+  int8_t     statusSent;
   SEpSet     mnodeEpSet;
   char      *file;
   SHashObj  *dnodeHash;
@@ -111,6 +112,7 @@ typedef struct SDnode {
   EStat       stat;
   SDnodeOpt   opt;
   SDnodeDir   dir;
+  FileFd      lockFd;
   SDnodeMgmt  dmgmt;
   SMnodeMgmt  mmgmt;
   SVnodesMgmt vmgmt;

source/dnode/mgmt/impl/src/dndDnode.c

@@ -282,8 +282,7 @@ PRASE_DNODE_OVER:
     pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
     pMgmt->dnodeEps->num = 1;
     pMgmt->dnodeEps->eps[0].isMnode = 1;   
-    pMgmt->dnodeEps->eps[0].port = pDnode->opt.serverPort;
+    taosGetFqdnPortFromEp(pDnode->opt.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port);
-    tstrncpy(pMgmt->dnodeEps->eps[0].fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN);
   }
 
   dndResetDnodes(pDnode, pMgmt->dnodeEps);
@@ -335,7 +334,7 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
   return 0;
 }
 
-static void dndSendStatusMsg(SDnode *pDnode) {
+void dndSendStatusMsg(SDnode *pDnode) {
   int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
 
   SStatusMsg *pStatus = rpcMallocCont(contLen);
@@ -349,7 +348,7 @@ static void dndSendStatusMsg(SDnode *pDnode) {
   pStatus->sver = htonl(pDnode->opt.sver);
   pStatus->dnodeId = htonl(pMgmt->dnodeId);
   pStatus->clusterId = htonl(pMgmt->clusterId);
-  pStatus->rebootTime = htonl(pMgmt->rebootTime);
+  pStatus->rebootTime = htobe64(pMgmt->rebootTime);
   pStatus->numOfCores = htons(pDnode->opt.numOfCores);
   pStatus->numOfSupportMnodes = htons(pDnode->opt.numOfCores);
   pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfCores);
@@ -357,7 +356,6 @@ static void dndSendStatusMsg(SDnode *pDnode) {
   tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN);
 
   pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval);
-  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pDnode->opt.mnodeEqualVnodeNum);
   pStatus->clusterCfg.checkTime = 0;
   char timestr[32] = "1970-01-01 00:00:00.00";
   (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
@@ -371,6 +369,9 @@ static void dndSendStatusMsg(SDnode *pDnode) {
   contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad);
 
   SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS};
+  pMgmt->statusSent = 1;
+
+  dTrace("pDnode:%p, send status msg to mnode", pDnode);
   dndSendMsgToMnode(pDnode, &rpcMsg);
 }
 
@@ -383,7 +384,7 @@ static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
     pMgmt->dnodeId = pCfg->dnodeId;
     pMgmt->clusterId = pCfg->clusterId;
     pMgmt->dropped = pCfg->dropped;
-    (void)dndWriteDnodes(pDnode);
+    dndWriteDnodes(pDnode);
     taosWUnLockLatch(&pMgmt->latch);
   }
 }
@@ -409,11 +410,16 @@ static void dndUpdateDnodeEps(SDnode *pDnode, SDnodeEps *pDnodeEps) {
 }
 
 static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
+
   if (pEpSet && pEpSet->numOfEps > 0) {
     dndUpdateMnodeEpSet(pDnode, pEpSet);
   }
 
-  if (pMsg->code != TSDB_CODE_SUCCESS) return;
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    pMgmt->statusSent = 0;
+    return;
+  }
 
   SStatusRsp *pRsp = pMsg->pCont;
   SDnodeCfg  *pCfg = &pRsp->dnodeCfg;
@@ -421,7 +427,10 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   pCfg->clusterId = htonl(pCfg->clusterId);
   dndUpdateDnodeCfg(pDnode, pCfg);
 
-  if (pCfg->dropped) return;
+  if (pCfg->dropped) {
+    pMgmt->statusSent = 0;
+    return;
+  }
 
   SDnodeEps *pDnodeEps = &pRsp->dnodeEps;
   pDnodeEps->num = htonl(pDnodeEps->num);
@@ -431,6 +440,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   }
 
   dndUpdateDnodeEps(pDnode, pDnodeEps);
+  pMgmt->statusSent = 0;
 }
 
 static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
@@ -438,13 +448,12 @@ static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { a
 static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); }
 
 static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) {
-  dDebug("config msg is received");
+  dError("config msg is received, but not supported yet");
   SCfgDnodeMsg *pCfg = pMsg->pCont;
 
   int32_t code = TSDB_CODE_OPS_NOT_SUPPORT;
   SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code};
   rpcSendResponse(&rspMsg);
-  rpcFreeCont(pMsg->pCont);
 }
 
 static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) {
@@ -457,18 +466,18 @@ static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) {
 
   SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)};
   rpcSendResponse(&rpcRsp);
-  rpcFreeCont(pMsg->pCont);
 }
 
 static void *dnodeThreadRoutine(void *param) {
   SDnode     *pDnode = param;
+  SDnodeMgmt *pMgmt = &pDnode->dmgmt;
   int32_t     ms = pDnode->opt.statusInterval * 1000;
 
   while (true) {
-    taosMsleep(ms);
     pthread_testcancel();
+    taosMsleep(ms);
 
-    if (dndGetStat(pDnode) == DND_STAT_RUNNING) {
+    if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent) {
       dndSendStatusMsg(pDnode);
     }
   }
@@ -478,7 +487,7 @@ int32_t dndInitDnode(SDnode *pDnode) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
   pMgmt->dnodeId = 0;
-  pMgmt->rebootTime = taosGetTimestampSec();
+  pMgmt->rebootTime = taosGetTimestampMs();
   pMgmt->dropped = 0;
   pMgmt->clusterId = 0;
 
@@ -556,8 +565,10 @@ void dndProcessDnodeReq(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
       dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
       SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
       rpcSendResponse(&rspMsg);
-      rpcFreeCont(pMsg->pCont);
   }
+
+  rpcFreeCont(pMsg->pCont);
+  pMsg->pCont = NULL;
 }
 
 void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
@@ -574,4 +585,7 @@ void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
     default:
       dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
   }
+
+  rpcFreeCont(pMsg->pCont);
+  pMsg->pCont = NULL;
 }

source/dnode/mgmt/impl/src/dndMnode.c

@@ -334,7 +334,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
   pOption->cfg.sver = pDnode->opt.sver;
   pOption->cfg.enableTelem = pDnode->opt.enableTelem;
   pOption->cfg.statusInterval = pDnode->opt.statusInterval;
-  pOption->cfg.mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum;
   pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer;
   pOption->cfg.timezone = pDnode->opt.timezone;
   pOption->cfg.charset = pDnode->opt.charset;
@@ -560,9 +559,12 @@ static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
       break;
   }
 
+  if (pMsg->msgType & 1u) {
     SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
     rpcSendResponse(&rsp);
+  }
   rpcFreeCont(pMsg->pCont);
+  pMsg->pCont = NULL;
   taosFreeQitem(pMsg);
 }
 
@@ -646,9 +648,12 @@ void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) {
 
   SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
   if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) {
+    if (pRpcMsg->msgType & 1u) {
       SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
       rpcSendResponse(&rsp);
+    }
     rpcFreeCont(pRpcMsg->pCont);
+    pRpcMsg->pCont = NULL;
     taosFreeQitem(pMsg);
   }
 }
@@ -657,9 +662,12 @@ void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   SMnodeMgmt *pMgmt = &pDnode->mmgmt;
   SMnode     *pMnode = dndAcquireMnode(pDnode);
   if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) {
+    if (pMsg->msgType & 1u) {
       SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
       rpcSendResponse(&rsp);
+    }
     rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
   }
 
   dndReleaseMnode(pDnode, pMnode);
@@ -669,9 +677,12 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   SMnodeMgmt *pMgmt = &pDnode->mmgmt;
   SMnode     *pMnode = dndAcquireMnode(pDnode);
   if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
+    if (pMsg->msgType & 1u) {
       SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
       rpcSendResponse(&rsp);
+    }
     rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
   }
 
   dndReleaseMnode(pDnode, pMnode);
@@ -681,9 +692,12 @@ void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   SMnodeMgmt *pMgmt = &pDnode->mmgmt;
   SMnode     *pMnode = dndAcquireMnode(pDnode);
   if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) {
+    if (pMsg->msgType & 1u) {
       SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
       rpcSendResponse(&rsp);
+    }
     rpcFreeCont(pMsg->pCont);
+    pMsg->pCont = NULL;
   }
 
   dndReleaseMnode(pDnode, pMnode);

source/dnode/mgmt/impl/src/dndTransport.c

@@ -69,10 +69,10 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_STB_VGROUP] = dndProcessMnodeReadMsg;
   pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg;
@@ -84,12 +84,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq;
 
   // message from mnode to vnode
-  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN_RSP] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN_RSP] = dndProcessMnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN] = dndProcessVnodeWriteMsg;
-  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN_RSP] = dndProcessMnodeWriteMsg;
 
   // message from mnode to dnode
   pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg;
@@ -130,7 +130,7 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
 
   if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
-    dTrace("RPC %p, rsp:%s app:%p is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType], pMsg->ahandle);
+    dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]);
     rpcFreeCont(pMsg->pCont);
     return;
   }
@@ -138,12 +138,11 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
   DndMsgFp fp = pMgmt->msgFp[msgType];
   if (fp != NULL) {
     (*fp)(pDnode, pMsg, pEpSet);
-    dTrace("RPC %p, rsp:%s app:%p is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->ahandle,
+    dTrace("RPC %p, rsp:%s is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF);
-           pMsg->code & 0XFFFF);
   } else {
-    dError("RPC %p, rsp:%s app:%p not processed", pMsg->handle, taosMsg[msgType], pMsg->ahandle);
+    dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
-  }
     rpcFreeCont(pMsg->pCont);
+  }
 }
 
 static int32_t dndInitClient(SDnode *pDnode) {

source/dnode/mgmt/impl/src/dnode.c

@@ -55,30 +55,31 @@ void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) {
   pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
 }
 
-static int32_t dndCheckRunning(char *dataDir) {
+static FileFd dndCheckRunning(char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
 
   FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
   if (fd < 0) {
-    dError("failed to lock file:%s since %s, quit", filepath, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
     return -1;
   }
 
   int32_t ret = taosLockFile(fd);
   if (ret != 0) {
-    dError("failed to lock file:%s since %s, quit", filepath, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
     taosCloseFile(fd);
     return -1;
   }
 
-  return 0;
+  return fd;
 }
 
 static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) {
-  if (dndCheckRunning(pOption->dataDir) != 0) {
+  pDnode->lockFd = dndCheckRunning(pOption->dataDir);
+  if (pDnode->lockFd < 0) {
     return -1;
   }
 
@@ -133,6 +134,12 @@ static void dndCleanupEnv(SDnode *pDnode) {
     tfree(pDnode->dir.dnode);
   }
 
+  if (pDnode->lockFd >= 0) {
+    taosUnLockFile(pDnode->lockFd);
+    taosCloseFile(pDnode->lockFd);
+    pDnode->lockFd = 0;
+  }
+
   taosStopCacheRefreshWorker();
 }
 
@@ -194,13 +201,16 @@ SDnode *dndInit(SDnodeOpt *pOption) {
   }
 
   dndSetStat(pDnode, DND_STAT_RUNNING);
+  dndSendStatusMsg(pDnode);
   dndReportStartup(pDnode, "TDengine", "initialized successfully");
-  dInfo("TDengine is initialized successfully");
+  dInfo("TDengine is initialized successfully, pDnode:%p", pDnode);
 
   return pDnode;
 }
 
 void dndCleanup(SDnode *pDnode) {
+  if (pDnode == NULL) return;
+
   if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
     dError("dnode is shutting down");
     return;

source/dnode/mgmt/impl/test/CMakeLists.txt

@@ -1,5 +1,6 @@
 # add_subdirectory(acct)
 # add_subdirectory(cluster)
+add_subdirectory(dnode)
 # add_subdirectory(profile)
 # add_subdirectory(show)
 add_subdirectory(user)

source/dnode/mgmt/impl/test/acct/acct.cpp

@@ -33,7 +33,7 @@ class DndTestAcct : public ::testing::Test {
   }
 
   static void TearDownTestSuite() {
-    dropServer(pServer);
+    stopServer(pServer);
     dropClient(pClient);
   }
 

source/dnode/mgmt/impl/test/cluster/cluster.cpp

@@ -33,7 +33,7 @@ class DndTestCluster : public ::testing::Test {
   }
 
   static void TearDownTestSuite() {
-    dropServer(pServer);
+    stopServer(pServer);
     dropClient(pClient);
   }
 
@@ -80,7 +80,7 @@ TEST_F(DndTestCluster, ShowCluster) {
 
     EXPECT_NE(pRsp->showId, 0);
     EXPECT_EQ(pMeta->contLen, 0);
-    EXPECT_STREQ(pMeta->tableFname, "show cluster");
+    EXPECT_STREQ(pMeta->tbFname, "show cluster");
     EXPECT_EQ(pMeta->numOfTags, 0);
     EXPECT_EQ(pMeta->precision, 0);
     EXPECT_EQ(pMeta->tableType, 0);

source/dnode/mgmt/impl/test/dnode/CMakeLists.txt

@@ -0,0 +1,29 @@
+add_executable(dndTestDnode "")
+
+target_sources(dndTestDnode
+    PRIVATE
+    "dnode.cpp"
+    "../sut/deploy.cpp"
+)
+
+target_link_libraries(
+    dndTestDnode
+    PUBLIC dnode
+    PUBLIC util
+    PUBLIC os
+    PUBLIC gtest_main
+)
+
+target_include_directories(dndTestDnode
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../sut"
+)
+
+enable_testing()
+
+add_test(
+    NAME dndTestDnode
+    COMMAND dndTestDnode
+)

source/dnode/mgmt/impl/test/dnode/dnode.cpp

@@ -0,0 +1,418 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "deploy.h"
+
+class DndTestDnode : public ::testing::Test {
+ public:
+  static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
+    SServer* pServer = createServer(path, fqdn, port, firstEp);
+    ASSERT(pServer);
+    return pServer;
+  }
+
+  static void SetUpTestSuite() {
+    initLog("/tmp/dndTestDnode");
+
+    const char* fqdn = "localhost";
+    const char* firstEp = "localhost:9521";
+    pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp);
+    pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp);
+    pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp);
+    pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp);
+    pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp);
+    pClient = createClient("root", "taosdata", fqdn, 9521);
+    taosMsleep(300);
+  }
+
+  static void TearDownTestSuite() {
+    stopServer(pServer1);
+    stopServer(pServer2);
+    stopServer(pServer3);
+    stopServer(pServer4);
+    stopServer(pServer5);
+    dropClient(pClient);
+    pServer1 = NULL;
+    pServer2 = NULL;
+    pServer3 = NULL;
+    pServer4 = NULL;
+    pServer5 = NULL;
+    pClient = NULL;
+  }
+
+  static SServer* pServer1;
+  static SServer* pServer2;
+  static SServer* pServer3;
+  static SServer* pServer4;
+  static SServer* pServer5;
+  static SClient* pClient;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) {
+    SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));
+    pShow->type = showType;
+    strcpy(pShow->db, "");
+
+    SRpcMsg showRpcMsg = {0};
+    showRpcMsg.pCont = pShow;
+    showRpcMsg.contLen = sizeof(SShowMsg);
+    showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW;
+
+    sendMsg(pClient, &showRpcMsg);
+    ASSERT_NE(pClient->pRsp, nullptr);
+    ASSERT_EQ(pClient->pRsp->code, 0);
+    ASSERT_NE(pClient->pRsp->pCont, nullptr);
+
+    SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont;
+    ASSERT_NE(pShowRsp, nullptr);
+    pShowRsp->showId = htonl(pShowRsp->showId);
+    pMeta = &pShowRsp->tableMeta;
+    pMeta->numOfTags = htons(pMeta->numOfTags);
+    pMeta->numOfColumns = htons(pMeta->numOfColumns);
+    pMeta->sversion = htons(pMeta->sversion);
+    pMeta->tversion = htons(pMeta->tversion);
+    pMeta->tuid = htobe64(pMeta->tuid);
+    pMeta->suid = htobe64(pMeta->suid);
+
+    showId = pShowRsp->showId;
+
+    EXPECT_NE(pShowRsp->showId, 0);
+    EXPECT_STREQ(pMeta->tbFname, showName);
+    EXPECT_EQ(pMeta->numOfTags, 0);
+    EXPECT_EQ(pMeta->numOfColumns, columns);
+    EXPECT_EQ(pMeta->precision, 0);
+    EXPECT_EQ(pMeta->tableType, 0);
+    EXPECT_EQ(pMeta->update, 0);
+    EXPECT_EQ(pMeta->sversion, 0);
+    EXPECT_EQ(pMeta->tversion, 0);
+    EXPECT_EQ(pMeta->tuid, 0);
+    EXPECT_EQ(pMeta->suid, 0);
+  }
+
+  void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) {
+    SSchema* pSchema = &pMeta->pSchema[index];
+    pSchema->bytes = htons(pSchema->bytes);
+    EXPECT_EQ(pSchema->colId, 0);
+    EXPECT_EQ(pSchema->type, type);
+    EXPECT_EQ(pSchema->bytes, bytes);
+    EXPECT_STREQ(pSchema->name, name);
+  }
+
+  void SendThenCheckShowRetrieveMsg(int32_t rows) {
+    SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg));
+    pRetrieve->showId = htonl(showId);
+    pRetrieve->free = 0;
+
+    SRpcMsg retrieveRpcMsg = {0};
+    retrieveRpcMsg.pCont = pRetrieve;
+    retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg);
+    retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
+
+    sendMsg(pClient, &retrieveRpcMsg);
+
+    ASSERT_NE(pClient->pRsp, nullptr);
+    ASSERT_EQ(pClient->pRsp->code, 0);
+    ASSERT_NE(pClient->pRsp->pCont, nullptr);
+
+    pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont;
+    ASSERT_NE(pRetrieveRsp, nullptr);
+    pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows);
+    pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset);
+    pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds);
+    pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen);
+
+    EXPECT_EQ(pRetrieveRsp->numOfRows, rows);
+    EXPECT_EQ(pRetrieveRsp->offset, 0);
+    EXPECT_EQ(pRetrieveRsp->useconds, 0);
+    // EXPECT_EQ(pRetrieveRsp->completed, completed);
+    EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI);
+    EXPECT_EQ(pRetrieveRsp->compressed, 0);
+    EXPECT_EQ(pRetrieveRsp->reserved, 0);
+    EXPECT_EQ(pRetrieveRsp->compLen, 0);
+
+    pData = pRetrieveRsp->data;
+    pos = 0;
+  }
+
+  void CheckInt16(int16_t val) {
+    int16_t data = *((int16_t*)(pData + pos));
+    pos += sizeof(int16_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckInt64(int64_t val) {
+    int64_t data = *((int64_t*)(pData + pos));
+    pos += sizeof(int64_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckTimestamp() {
+    int64_t data = *((int64_t*)(pData + pos));
+    pos += sizeof(int64_t);
+    EXPECT_GT(data, 0);
+  }
+
+  void CheckBinary(const char* val, int32_t len) {
+    pos += sizeof(VarDataLenT);
+    char* data = (char*)(pData + pos);
+    pos += len;
+    EXPECT_STREQ(data, val);
+  }
+
+  int32_t            showId;
+  STableMetaMsg*     pMeta;
+  SRetrieveTableRsp* pRetrieveRsp;
+  char*              pData;
+  int32_t            pos;
+};
+
+SServer* DndTestDnode::pServer1;
+SServer* DndTestDnode::pServer2;
+SServer* DndTestDnode::pServer3;
+SServer* DndTestDnode::pServer4;
+SServer* DndTestDnode::pServer5;
+SClient* DndTestDnode::pClient;
+
+TEST_F(DndTestDnode, ShowDnode) {
+  SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
+  CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id");
+  CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point");
+  CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes");
+  CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max vnodes");
+  CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status");
+  CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time");
+  CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason");
+
+  SendThenCheckShowRetrieveMsg(1);
+  CheckInt16(1);
+  CheckBinary("localhost:9521", TSDB_EP_LEN);
+  CheckInt16(0);
+  CheckInt16(1);
+  CheckBinary("ready", 10);
+  CheckTimestamp();
+  CheckBinary("", 24);
+}
+
+TEST_F(DndTestDnode, ConfigDnode_01) {
+  SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg));
+  pReq->dnodeId = htonl(1);
+  strcpy(pReq->config, "ddebugflag 131");
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = sizeof(SCfgDnodeMsg);
+  rpcMsg.msgType = TSDB_MSG_TYPE_CONFIG_DNODE;
+
+  sendMsg(pClient, &rpcMsg);
+  SRpcMsg* pMsg = pClient->pRsp;
+  ASSERT_NE(pMsg, nullptr);
+  ASSERT_EQ(pMsg->code, 0);
+}
+
+TEST_F(DndTestDnode, CreateDnode_01) {
+  SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
+  strcpy(pReq->ep, "localhost:9522");
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = sizeof(SCreateDnodeMsg);
+  rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
+
+  sendMsg(pClient, &rpcMsg);
+  SRpcMsg* pMsg = pClient->pRsp;
+  ASSERT_NE(pMsg, nullptr);
+  ASSERT_EQ(pMsg->code, 0);
+
+  taosMsleep(1300);
+  SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
+  SendThenCheckShowRetrieveMsg(2);
+  CheckInt16(1);
+  CheckInt16(2);
+  CheckBinary("localhost:9521", TSDB_EP_LEN);
+  CheckBinary("localhost:9522", TSDB_EP_LEN);
+  CheckInt16(0);
+  CheckInt16(0);
+  CheckInt16(1);
+  CheckInt16(1);
+  CheckBinary("ready", 10);
+  CheckBinary("ready", 10);
+  CheckTimestamp();
+  CheckTimestamp();
+  CheckBinary("", 24);
+  CheckBinary("", 24);
+}
+
+TEST_F(DndTestDnode, DropDnode_01) {
+  SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg));
+  pReq->dnodeId = htonl(2);
+
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.pCont = pReq;
+  rpcMsg.contLen = sizeof(SDropDnodeMsg);
+  rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE;
+
+  sendMsg(pClient, &rpcMsg);
+  SRpcMsg* pMsg = pClient->pRsp;
+  ASSERT_NE(pMsg, nullptr);
+  ASSERT_EQ(pMsg->code, 0);
+
+  taosMsleep(1300);
+  SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
+  SendThenCheckShowRetrieveMsg(1);
+  CheckInt16(1);
+  CheckBinary("localhost:9521", TSDB_EP_LEN);
+  CheckInt16(0);
+  CheckInt16(1);
+  CheckBinary("ready", 10);
+  CheckTimestamp();
+  CheckBinary("", 24);
+}
+
+TEST_F(DndTestDnode, CreateDnode_02) {
+  {
+    SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
+    strcpy(pReq->ep, "localhost:9523");
+
+    SRpcMsg rpcMsg = {0};
+    rpcMsg.pCont = pReq;
+    rpcMsg.contLen = sizeof(SCreateDnodeMsg);
+    rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
+
+    sendMsg(pClient, &rpcMsg);
+    SRpcMsg* pMsg = pClient->pRsp;
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+
+  {
+    SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
+    strcpy(pReq->ep, "localhost:9524");
+
+    SRpcMsg rpcMsg = {0};
+    rpcMsg.pCont = pReq;
+    rpcMsg.contLen = sizeof(SCreateDnodeMsg);
+    rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
+
+    sendMsg(pClient, &rpcMsg);
+    SRpcMsg* pMsg = pClient->pRsp;
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+
+  {
+    SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
+    strcpy(pReq->ep, "localhost:9525");
+
+    SRpcMsg rpcMsg = {0};
+    rpcMsg.pCont = pReq;
+    rpcMsg.contLen = sizeof(SCreateDnodeMsg);
+    rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
+
+    sendMsg(pClient, &rpcMsg);
+    SRpcMsg* pMsg = pClient->pRsp;
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+  }
+
+  taosMsleep(1300);
+  SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
+  SendThenCheckShowRetrieveMsg(4);
+  CheckInt16(1);
+  CheckInt16(3);
+  CheckInt16(4);
+  CheckInt16(5);
+  CheckBinary("localhost:9521", TSDB_EP_LEN);
+  CheckBinary("localhost:9523", TSDB_EP_LEN);
+  CheckBinary("localhost:9524", TSDB_EP_LEN);
+  CheckBinary("localhost:9525", TSDB_EP_LEN);
+  CheckInt16(0);
+  CheckInt16(0);
+  CheckInt16(0);
+  CheckInt16(0);
+  CheckInt16(1);
+  CheckInt16(1);
+  CheckInt16(1);
+  CheckInt16(1);
+  CheckBinary("ready", 10);
+  CheckBinary("ready", 10);
+  CheckBinary("ready", 10);
+  CheckBinary("ready", 10);
+  CheckTimestamp();
+  CheckTimestamp();
+  CheckTimestamp();
+  CheckTimestamp();
+  CheckBinary("", 24);
+  CheckBinary("", 24);
+  CheckBinary("", 24);
+  CheckBinary("", 24);
+}
+
+TEST_F(DndTestDnode, RestartDnode_01) {
+  uInfo("===> stop all server");
+  stopServer(pServer1);
+  stopServer(pServer2);
+  stopServer(pServer3);
+  stopServer(pServer4);
+  stopServer(pServer5);
+  pServer1 = NULL;
+  pServer2 = NULL;
+  pServer3 = NULL;
+  pServer4 = NULL;
+  pServer5 = NULL;
+
+  taosMsleep(3000);  // wait tcp port cleanedup
+  uInfo("===> start all server");
+
+  const char* fqdn = "localhost";
+  const char* firstEp = "localhost:9521";
+  pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp);
+
+  uInfo("===> all server is running");
+
+  // taosMsleep(1300);
+  // SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
+  // SendThenCheckShowRetrieveMsg(4);
+  // CheckInt16(1);
+  // CheckInt16(3);
+  // CheckInt16(4);
+  // CheckInt16(5);
+  // CheckBinary("localhost:9521", TSDB_EP_LEN);
+  // CheckBinary("localhost:9523", TSDB_EP_LEN);
+  // CheckBinary("localhost:9524", TSDB_EP_LEN);
+  // CheckBinary("localhost:9525", TSDB_EP_LEN);
+  // CheckInt16(0);
+  // CheckInt16(0);
+  // CheckInt16(0);
+  // CheckInt16(0);
+  // CheckInt16(1);
+  // CheckInt16(1);
+  // CheckInt16(1);
+  // CheckInt16(1);
+  // CheckBinary("ready", 10);
+  // CheckBinary("ready", 10);
+  // CheckBinary("ready", 10);
+  // CheckBinary("ready", 10);
+  // CheckTimestamp();
+  // CheckTimestamp();
+  // CheckTimestamp();
+  // CheckTimestamp();
+  // CheckBinary("", 24);
+  // CheckBinary("", 24);
+  // CheckBinary("", 24);
+  // CheckBinary("", 24);
+}

source/dnode/mgmt/impl/test/profile/profile.cpp

@@ -33,7 +33,7 @@ class DndTestProfile : public ::testing::Test {
   }
 
   static void TearDownTestSuite() {
-    dropServer(pServer);
+    stopServer(pServer);
     dropClient(pClient);
   }
 
@@ -139,7 +139,7 @@ TEST_F(DndTestProfile, SConnectMsg_03) {
 
     EXPECT_NE(pRsp->showId, 0);
     EXPECT_EQ(pMeta->contLen, 0);
-    EXPECT_STREQ(pMeta->tableFname, "");
+    EXPECT_STREQ(pMeta->tbFname, "");
     EXPECT_EQ(pMeta->numOfTags, 0);
     EXPECT_EQ(pMeta->precision, 0);
     EXPECT_EQ(pMeta->tableType, 0);
@@ -480,7 +480,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) {
 
     EXPECT_NE(pRsp->showId, 0);
     EXPECT_EQ(pMeta->contLen, 0);
-    EXPECT_STREQ(pMeta->tableFname, "");
+    EXPECT_STREQ(pMeta->tbFname, "");
     EXPECT_EQ(pMeta->numOfTags, 0);
     EXPECT_EQ(pMeta->precision, 0);
     EXPECT_EQ(pMeta->tableType, 0);
@@ -667,7 +667,7 @@ TEST_F(DndTestProfile, SKillStreamMsg_03) {
 
     EXPECT_NE(pRsp->showId, 0);
     EXPECT_EQ(pMeta->contLen, 0);
-    EXPECT_STREQ(pMeta->tableFname, "");
+    EXPECT_STREQ(pMeta->tbFname, "");
     EXPECT_EQ(pMeta->numOfTags, 0);
     EXPECT_EQ(pMeta->precision, 0);
     EXPECT_EQ(pMeta->tableType, 0);

source/dnode/mgmt/impl/test/show/show.cpp

@@ -33,7 +33,7 @@ class DndTestShow : public ::testing::Test {
   }
 
   static void TearDownTestSuite() {
-    dropServer(pServer);
+    stopServer(pServer);
     dropClient(pClient);
   }
 
@@ -141,7 +141,7 @@ TEST_F(DndTestShow, SShowMsg_04) {
 
     EXPECT_NE(pRsp->showId, 0);
     EXPECT_EQ(pMeta->contLen, 0);
-    EXPECT_STREQ(pMeta->tableFname, "");
+    EXPECT_STREQ(pMeta->tbFname, "");
     EXPECT_EQ(pMeta->numOfTags, 0);
     EXPECT_EQ(pMeta->precision, 0);
     EXPECT_EQ(pMeta->tableType, 0);

source/dnode/mgmt/impl/test/sut/deploy.cpp

@@ -16,16 +16,17 @@
 #include "deploy.h"
 
 void initLog(const char* path) {
-  dDebugFlag = 0;
+  dDebugFlag = 143;
   vDebugFlag = 0;
-  mDebugFlag = 207;
+  mDebugFlag = 143;
+  cDebugFlag = 0;
   jniDebugFlag = 0;
   tmrDebugFlag = 0;
   sdbDebugFlag = 0;
   httpDebugFlag = 0;
   mqttDebugFlag = 0;
   monDebugFlag = 0;
-  uDebugFlag = 0;
+  uDebugFlag = 143;
   rpcDebugFlag = 0;
   odbcDebugFlag = 0;
   qDebugFlag = 0;
@@ -33,6 +34,10 @@ void initLog(const char* path) {
   sDebugFlag = 0;
   tsdbDebugFlag = 0;
   cqDebugFlag = 0;
+  tscEmbeddedInUtil = 1;
+
+  taosRemoveDir(path);
+  taosMkDir(path);
 
   char temp[PATH_MAX];
   snprintf(temp, PATH_MAX, "%s/taosdlog", path);
@@ -49,14 +54,13 @@ void* runServer(void* param) {
   }
 }
 
-void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port) {
+void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
   pOption->sver = 1;
   pOption->numOfCores = 1;
   pOption->numOfSupportMnodes = 1;
   pOption->numOfSupportVnodes = 1;
   pOption->numOfSupportQnodes = 1;
   pOption->statusInterval = 1;
-  pOption->mnodeEqualVnodeNum = 1;
   pOption->numOfThreadsPerCore = 1;
   pOption->ratioOfQueryCores = 1;
   pOption->maxShellConns = 1000;
@@ -65,16 +69,14 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t
   strcpy(pOption->dataDir, path);
   snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port);
   snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn);
-  snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port);
+  snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp);
 }
 
-SServer* createServer(const char* path, const char* fqdn, uint16_t port) {
+SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
-  taosRemoveDir(path);
   taosMkDir(path);
-  initLog(path);
 
   SDnodeOpt option = {0};
-  initOption(&option, path, fqdn, port);
+  initOption(&option, path, fqdn, port, firstEp);
 
   SDnode* pDnode = dndInit(&option);
   ASSERT(pDnode);
@@ -89,16 +91,28 @@ SServer* createServer(const char* path, const char* fqdn, uint16_t port) {
   return pServer;
 }
 
-void dropServer(SServer* pServer) {
+SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
+  taosRemoveDir(path);
+  return startServer(path, fqdn, port, firstEp);
+}
+
+void stopServer(SServer* pServer) {
+  if (pServer == NULL) return;
   if (pServer->threadId != NULL) {
     taosDestoryThread(pServer->threadId);
   }
+
+  if (pServer->pDnode != NULL) {
+    dndCleanup(pServer->pDnode);
+    pServer->pDnode = NULL;
+  }
 }
 
 void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   SClient* pClient = (SClient*)parent;
   pClient->pRsp = pMsg;
-  // taosMsleep(1000000);
+  uInfo("response:%s from dnode, pCont:%p contLen:%d code:0x%X", taosMsg[pMsg->msgType], pMsg->pCont, pMsg->contLen,
+        pMsg->code);
   tsem_post(&pClient->sem);
 }
 
@@ -144,7 +158,7 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) {
   epSet.inUse = 0;
   epSet.numOfEps = 1;
   epSet.port[0] = pClient->port;
-  strcpy(epSet.fqdn[0], pClient->fqdn);
+  memcpy(epSet.fqdn[0], pClient->fqdn, TSDB_FQDN_LEN);
 
   rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL);
   tsem_wait(&pClient->sem);

source/dnode/mgmt/impl/test/sut/deploy.h

@@ -39,8 +39,10 @@ typedef struct {
   tsem_t   sem;
 } SClient;
 
-SServer* createServer(const char* path, const char* fqdn, uint16_t port);
+void     initLog(const char* path);
-void     dropServer(SServer* pServer);
+SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp);
+SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp);
+void     stopServer(SServer* pServer);
 SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port);
 void     dropClient(SClient* pClient);
 void     sendMsg(SClient* pClient, SRpcMsg* pMsg);

source/dnode/mgmt/impl/test/user/user.cpp

@@ -26,14 +26,15 @@ class DndTestUser : public ::testing::Test {
     const char* path = "/tmp/dndTestUser";
     const char* fqdn = "localhost";
     uint16_t    port = 9524;
+    const char* firstEp = "localhost:9524";
 
-    pServer = createServer(path, fqdn, port);
+    pServer = createServer(path, fqdn, port, firstEp);
     ASSERT(pServer);
     pClient = createClient(user, pass, fqdn, port);
   }
 
   static void TearDownTestSuite() {
-    dropServer(pServer);
+    stopServer(pServer);
     dropClient(pClient);
   }
 
@@ -79,7 +80,7 @@ TEST_F(DndTestUser, ShowUser) {
 
   EXPECT_NE(pShowRsp->showId, 0);
   EXPECT_EQ(pMeta->contLen, 0);
-  EXPECT_STREQ(pMeta->tableFname, "show users");
+  EXPECT_STREQ(pMeta->tbFname, "show users");
   EXPECT_EQ(pMeta->numOfTags, 0);
   EXPECT_EQ(pMeta->precision, 0);
   EXPECT_EQ(pMeta->tableType, 0);

source/dnode/mnode/impl/inc/mndDef.h

@@ -92,7 +92,6 @@ typedef enum {
   DND_REASON_VERSION_NOT_MATCH,
   DND_REASON_DNODE_ID_NOT_MATCH,
   DND_REASON_CLUSTER_ID_NOT_MATCH,
-  DND_REASON_MN_EQUAL_VN_NOT_MATCH,
   DND_REASON_STATUS_INTERVAL_NOT_MATCH,
   DND_REASON_TIME_ZONE_NOT_MATCH,
   DND_REASON_LOCALE_NOT_MATCH,
@@ -125,6 +124,7 @@ typedef struct SDnodeObj {
   int64_t    createdTime;
   int64_t    updateTime;
   int64_t    rebootTime;
+  int64_t    lastAccessTime;
   int32_t    accessTimes;
   int16_t    numOfMnodes;
   int16_t    numOfVnodes;
@@ -241,7 +241,7 @@ typedef struct SVgObj {
   SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
 } SVgObj;
 
-typedef struct SStableObj {
+typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   char     db[TSDB_FULL_DB_NAME_LEN];
   int64_t  createdTime;
@@ -251,9 +251,8 @@ typedef struct SStableObj {
   int32_t  numOfColumns;
   int32_t  numOfTags;
   SRWLatch lock;
-  SSchema *columnSchema;
+  SSchema *pSchema;
-  SSchema *tagSchema;
+} SStbObj;
-} SStableObj;
 
 typedef struct SFuncObj {
   char    name[TSDB_FUNC_NAME_LEN];

source/dnode/mnode/impl/inc/mndInt.h

@@ -88,6 +88,8 @@ void mndSendMsgToMnode(SMnode *pMnode, SRpcMsg *pMsg);
 void mndSendRedirectMsg(SMnode *pMnode, SRpcMsg *pMsg);
 void mndSetMsgHandle(SMnode *pMnode, int32_t msgType, MndMsgFp fp);
 
+uint64_t mndGenerateUid(char *name, int32_t len) ;
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndStb.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_MND_STABLE_H_
+#ifndef _TD_MND_STB_H_
-#define _TD_MND_STABLE_H_
+#define _TD_MND_STB_H_
 
 #include "mndInt.h"
 
@@ -22,11 +22,11 @@
 extern "C" {
 #endif
 
-int32_t mndInitStable(SMnode *pMnode);
+int32_t mndInitStb(SMnode *pMnode);
-void    mndCleanupStable(SMnode *pMnode);
+void    mndCleanupStb(SMnode *pMnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MND_STABLE_H_*/
+#endif /*_TD_MND_STB_H_*/

source/dnode/mnode/impl/src/mndCluster.c

@@ -159,7 +159,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
   cols++;
 
   pMeta->numOfColumns = htons(cols);
-  strcpy(pMeta->tableFname, "show cluster");
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/mnode/impl/src/mndDb.c

@@ -767,7 +767,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe
 
   pShow->numOfRows = sdbGetSize(pSdb, SDB_DB);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -22,10 +22,13 @@
 #include "tutil.h"
 
 #define TSDB_DNODE_VER 1
+#define TSDB_DNODE_RESERVE_SIZE 64
 #define TSDB_CONFIG_OPTION_LEN 16
 #define TSDB_CONIIG_VALUE_LEN 48
 #define TSDB_CONFIG_NUMBER 8
 
+static int32_t id = 2;
+
 static const char *offlineReason[] = {
     "",
     "status msg timeout",
@@ -33,7 +36,6 @@ static const char *offlineReason[] = {
     "version not match",
     "dnodeId not match",
     "clusterId not match",
-    "mnEqualVn not match",
     "interval not match",
     "timezone not match",
     "locale not match",
@@ -117,6 +119,7 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
   SDB_SET_INT64(pRaw, dataPos, pDnode->updateTime)
   SDB_SET_INT16(pRaw, dataPos, pDnode->port)
   SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN)
+  SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE)
   SDB_SET_DATALEN(pRaw, dataPos);
 
   return pRaw;
@@ -142,28 +145,28 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, pRow, dataPos, &pDnode->updateTime)
   SDB_GET_INT16(pRaw, pRow, dataPos, &pDnode->port)
   SDB_GET_BINARY(pRaw, pRow, dataPos, pDnode->fqdn, TSDB_FQDN_LEN)
+  SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_DNODE_RESERVE_SIZE)
 
   return pRow;
 }
 
-static void mnodeResetDnode(SDnodeObj *pDnode) {
+static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) {
+  mTrace("dnode:%d, perform insert action", pDnode->id);
+
   pDnode->rebootTime = 0;
+  pDnode->lastAccessTime = 0;
   pDnode->accessTimes = 0;
-  pDnode->numOfCores = 0;
   pDnode->numOfMnodes = 0;
   pDnode->numOfVnodes = 0;
   pDnode->numOfQnodes = 0;
   pDnode->numOfSupportMnodes = 0;
   pDnode->numOfSupportVnodes = 0;
   pDnode->numOfSupportQnodes = 0;
+  pDnode->numOfCores = 0;
   pDnode->status = DND_STATUS_OFFLINE;
   pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED;
   snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port);
-}
 
-static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) {
-  mTrace("dnode:%d, perform insert action", pDnode->id);
-  mnodeResetDnode(pDnode);
   return 0;
 }
 
@@ -174,11 +177,6 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) {
 
 static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) {
   mTrace("dnode:%d, perform update action", pOldDnode->id);
-  pOldDnode->id = pNewDnode->id;
-  pOldDnode->createdTime = pNewDnode->createdTime;
-  pOldDnode->updateTime = pNewDnode->updateTime;
-  pOldDnode->port = pNewDnode->port;
-  memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN);
   return 0;
 }
 
@@ -232,6 +230,7 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) {
     if (pIter == NULL) break;
     if (i >= numOfEps) {
       sdbCancelFetch(pSdb, pIter);
+      sdbRelease(pSdb, pDnode);
       break;
     }
 
@@ -244,20 +243,15 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) {
       pEp->isMnode = 1;
     }
     i++;
+    sdbRelease(pSdb, pDnode);
   }
 
   pEps->num = htonl(i);
 }
 
 static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
-  if (pCfg->mnodeEqualVnodeNum != pMnode->cfg.mnodeEqualVnodeNum) {
-    mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg inconsistent", pCfg->mnodeEqualVnodeNum,
-           pMnode->cfg.mnodeEqualVnodeNum);
-    return DND_REASON_MN_EQUAL_VN_NOT_MATCH;
-  }
-
   if (pCfg->statusInterval != pMnode->cfg.statusInterval) {
-    mError("\"statusInterval\"[%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval);
+    mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval);
     return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
   }
 
@@ -265,18 +259,18 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
   char    timestr[32] = "1970-01-01 00:00:00.00";
   (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
   if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) {
-    mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone,
+    mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone,
            pCfg->checkTime, checkTime);
     return DND_REASON_TIME_ZONE_NOT_MATCH;
   }
 
   if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) {
-    mError("\"locale\"[%s - %s]  cfg parameters inconsistent", pCfg->locale, pMnode->cfg.locale);
+    mError("locale [%s - %s]  cfg inconsistent", pCfg->locale, pMnode->cfg.locale);
     return DND_REASON_LOCALE_NOT_MATCH;
   }
 
   if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) {
-    mError("\"charset\"[%s - %s] cfg parameters inconsistent.", pCfg->charset, pMnode->cfg.charset);
+    mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset);
     return DND_REASON_CHARSET_NOT_MATCH;
   }
 
@@ -287,14 +281,12 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) {
   pStatus->sver = htonl(pStatus->sver);
   pStatus->dnodeId = htonl(pStatus->dnodeId);
   pStatus->clusterId = htonl(pStatus->clusterId);
-  pStatus->rebootTime = htonl(pStatus->rebootTime);
+  pStatus->rebootTime = htobe64(pStatus->rebootTime);
   pStatus->numOfCores = htons(pStatus->numOfCores);
   pStatus->numOfSupportMnodes = htons(pStatus->numOfSupportMnodes);
   pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes);
   pStatus->numOfSupportQnodes = htons(pStatus->numOfSupportQnodes);
-
   pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval);
-  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pStatus->clusterCfg.mnodeEqualVnodeNum);
   pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime);
 }
 
@@ -308,7 +300,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
     pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp);
     if (pDnode == NULL) {
       mDebug("dnode:%s, not created yet", pStatus->dnodeEp);
-      return TSDB_CODE_MND_DNODE_NOT_EXIST;
+      terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+      return -1;
     }
   } else {
     pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId);
@@ -319,7 +312,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
       }
       mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
       mndReleaseDnode(pMnode, pDnode);
-      return TSDB_CODE_MND_DNODE_NOT_EXIST;
+      terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+      return -1;
     }
   }
 
@@ -329,7 +323,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
     }
     mndReleaseDnode(pMnode, pDnode);
     mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver);
-    return TSDB_CODE_MND_INVALID_MSG_VERSION;
+    terrno = TSDB_CODE_MND_INVALID_MSG_VERSION;
+    return -1;
   }
 
   if (pStatus->dnodeId == 0) {
@@ -341,7 +336,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
       }
       mError("dnode:%d, clusterId %d not match exist %d", pDnode->id, pStatus->clusterId, pMnode->clusterId);
       mndReleaseDnode(pMnode, pDnode);
-      return TSDB_CODE_MND_INVALID_CLUSTER_ID;
+      terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID;
+      return -1;
     } else {
       pDnode->accessTimes++;
       mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes);
@@ -355,7 +351,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
       pDnode->offlineReason = ret;
       mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]);
       mndReleaseDnode(pMnode, pDnode);
-      return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT;
+      terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG;
+      return -1;
     }
 
     mInfo("dnode:%d, from offline to online", pDnode->id);
@@ -366,6 +363,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
   pDnode->numOfSupportMnodes = pStatus->numOfSupportMnodes;
   pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes;
   pDnode->numOfSupportQnodes = pStatus->numOfSupportQnodes;
+  pDnode->lastAccessTime = taosGetTimestampMs();
   pDnode->status = DND_STATUS_READY;
 
   int32_t     numOfEps = mndGetDnodeSize(pMnode);
@@ -373,7 +371,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
   SStatusRsp *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     mndReleaseDnode(pMnode, pDnode);
-    return TSDB_CODE_OUT_OF_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
   }
 
   pRsp->dnodeCfg.dnodeId = htonl(pDnode->id);
@@ -390,13 +389,13 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) {
 
 static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) {
   SDnodeObj dnodeObj = {0};
-  dnodeObj.id = 1;  // todo
+  dnodeObj.id = id++;
   dnodeObj.createdTime = taosGetTimestampMs();
   dnodeObj.updateTime = dnodeObj.createdTime;
   taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port);
 
   if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
     mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
     return terrno;
   }
@@ -449,7 +448,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
   mDebug("dnode:%s, start to create", pCreate->ep);
 
   if (pCreate->ep[0] == 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
     mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
     return -1;
   }
@@ -457,7 +456,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
   SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep);
   if (pDnode != NULL) {
     mError("dnode:%d, already exist", pDnode->id);
-    sdbRelease(pMnode->pSdb, pDnode);
+    mndReleaseDnode(pMnode, pDnode);
     terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
     return -1;
   }
@@ -478,7 +477,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode)
     mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr());
     return -1;
   }
-  mDebug("trans:%d, used to drop user:%d", pTrans->id, pDnode->id);
+  mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
 
   SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode);
   if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
@@ -522,26 +521,26 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) {
   mDebug("dnode:%d, start to drop", pDrop->dnodeId);
 
   if (pDrop->dnodeId <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_MND_INVALID_DNODE_ID;
     mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr());
     return -1;
   }
 
   SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId);
   if (pDnode == NULL) {
-    mError("dnode:%d, not exist", pDrop->dnodeId);
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
-    return -1;
-  }
-
-  int32_t code = mndDropDnode(pMnode, pMsg, pDnode);
-
-  if (code != 0) {
     mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr());
     return -1;
   }
 
-  sdbRelease(pMnode->pSdb, pDnode);
+  int32_t code = mndDropDnode(pMnode, pMsg, pDnode);
+  if (code != 0) {
+    mndReleaseDnode(pMnode, pDnode);
+    mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr());
+    return -1;
+  }
+
+  mndReleaseDnode(pMnode, pDnode);
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
@@ -553,7 +552,7 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) {
   SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId);
   if (pDnode == NULL) {
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
-    mError("dnode:%d, failed to cfg since %s ", pCfg->dnodeId, terrstr());
+    mError("dnode:%d, failed to config since %s ", pCfg->dnodeId, terrstr());
     return -1;
   }
 
@@ -562,17 +561,22 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) {
 
   SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg));
   pCfgDnode->dnodeId = htonl(pCfg->dnodeId);
-  memcpy(pCfgDnode->config, pCfg->config, 128);
+  memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN);
 
-  SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg)};
+  SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN,
+                    .pCont = pCfgDnode,
+                    .contLen = sizeof(SCfgDnodeMsg),
+                    .ahandle = pMsg->rpcMsg.ahandle};
 
-  mInfo("dnode:%d, is configured", pCfg->dnodeId);
+  mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config);
   mndSendMsgToDnode(pMnode, &epSet, &rpcMsg);
 
   return 0;
 }
 
-static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp is received"); }
+static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) {
+  mInfo("app:%p config rsp from dnode", pMsg->rpcMsg.ahandle);
+}
 
 static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {
   int32_t  cols = 0;
@@ -600,8 +604,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
 
   pShow->numOfRows = TSDB_CONFIG_NUMBER;
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  pShow->pIter = NULL;
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
 
   return 0;
 }
@@ -676,7 +679,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
 
   pShow->bytes[cols] = 2;
   pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "cores");
+  strcpy(pSchema[cols].name, "max vnodes");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -708,7 +711,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
 
   pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }
@@ -740,7 +743,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDnode->numOfCores;
+    *(int16_t *)pWrite = pDnode->numOfSupportVnodes;
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -753,7 +756,11 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    if (pDnode->status == DND_STATUS_READY) {
+      STR_TO_VARSTR(pWrite, "");
+    } else {
       STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]);
+    }
     cols++;
 
     numOfRows++;

source/dnode/mnode/impl/src/mndFunc.c

@@ -429,7 +429,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p
 
   pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, "show funcs");
+  strcpy(pMeta->tbFname, "show funcs");
 
   return 0;
 }

source/dnode/mnode/impl/src/mndMnode.c

@@ -405,7 +405,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
 
   pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }

source/dnode/mnode/impl/src/mndShow.c

@@ -16,6 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "mndShow.h"
 
+#define SHOW_STEP_SIZE 100
+
 static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg);
 static void      mndFreeShowObj(SShowObj *pShow);
 static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId);
@@ -211,7 +213,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
   }
 
   /* return no more than 100 tables in one round trip */
-  if (rowsToRead > 100) rowsToRead = 100;
+  if (rowsToRead > SHOW_STEP_SIZE) rowsToRead = SHOW_STEP_SIZE;
 
   /*
    * the actual number of table may be larger than the value of pShow->numOfRows, if a query is
@@ -220,7 +222,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
   if (rowsToRead < 0) rowsToRead = 0;
   size = pShow->rowSize * rowsToRead;
 
-  size += 100;
+  size += SHOW_STEP_SIZE;
   SRetrieveTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {
     mndReleaseShowObj(pShow, false);
@@ -270,7 +272,7 @@ char *mndShowStr(int32_t showType) {
       return "show mnodes";
     case TSDB_MGMT_TABLE_VGROUP:
       return "show vgroups";
-    case TSDB_MGMT_TABLE_STABLE:
+    case TSDB_MGMT_TABLE_STB:
       return "show stables";
     case TSDB_MGMT_TABLE_MODULE:
       return "show modules";

source/dnode/mnode/impl/src/mndStable.c

@@ -1,427 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "mndStable.h"
-#include "mndDnode.h"
-#include "mndMnode.h"
-#include "mndShow.h"
-#include "mndTrans.h"
-#include "mndUser.h"
-#include "mndDb.h"
-#include "tname.h"
-
-#define TSDB_STABLE_VER_NUM 1
-#define TSDB_STABLE_RESERVE_SIZE 64
-
-static SSdbRaw *mndStableActionEncode(SStableObj *pStb);
-static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw);
-static int32_t  mndStableActionInsert(SSdb *pSdb, SStableObj *pStb);
-static int32_t  mndStableActionDelete(SSdb *pSdb, SStableObj *pStb);
-static int32_t  mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb);
-static int32_t  mndProcessCreateStableMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessAlterStableMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessDropStableMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessCreateStableInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessAlterStableInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessDropStableInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessStableMetaMsg(SMnodeMsg *pMsg);
-static int32_t  mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
-static int32_t  mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
-static void     mndCancelGetNextStable(SMnode *pMnode, void *pIter);
-
-int32_t mndInitStable(SMnode *pMnode) {
-  SSdbTable table = {.sdbType = SDB_STABLE,
-                     .keyType = SDB_KEY_BINARY,
-                     .encodeFp = (SdbEncodeFp)mndStableActionEncode,
-                     .decodeFp = (SdbDecodeFp)mndStableActionDecode,
-                     .insertFp = (SdbInsertFp)mndStableActionInsert,
-                     .updateFp = (SdbUpdateFp)mndStableActionUpdate,
-                     .deleteFp = (SdbDeleteFp)mndStableActionDelete};
-
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE, mndProcessCreateStableMsg);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE, mndProcessAlterStableMsg);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE, mndProcessDropStableMsg);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP, mndProcessCreateStableInRsp);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP, mndProcessAlterStableInRsp);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE_IN_RSP, mndProcessDropStableInRsp);
-  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStableMetaMsg);
-
-  mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndGetStableMeta);
-  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndRetrieveStables);
-  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndCancelGetNextStable);
-
-  return sdbSetTable(pMnode->pSdb, table);
-}
-
-void mndCleanupStable(SMnode *pMnode) {}
-
-static SSdbRaw *mndStableActionEncode(SStableObj *pStb) {
-  int32_t  size = sizeof(SStableObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema);
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size);
-  if (pRaw == NULL) return NULL;
-
-  int32_t dataPos = 0;
-  SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
-  SDB_SET_INT64(pRaw, dataPos, pStb->createdTime)
-  SDB_SET_INT64(pRaw, dataPos, pStb->updateTime)
-  SDB_SET_INT64(pRaw, dataPos, pStb->uid)
-  SDB_SET_INT64(pRaw, dataPos, pStb->version)
-  SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns)
-  SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags)
-
-  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
-    SSchema *pSchema = &pStb->columnSchema[i];
-    SDB_SET_INT8(pRaw, dataPos, pSchema->type);
-    SDB_SET_INT32(pRaw, dataPos, pSchema->colId);
-    SDB_SET_INT32(pRaw, dataPos, pSchema->bytes);
-    SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
-  }
-
-  for (int32_t i = 0; i < pStb->numOfTags; ++i) {
-    SSchema *pSchema = &pStb->tagSchema[i];
-    SDB_SET_INT8(pRaw, dataPos, pSchema->type);
-    SDB_SET_INT32(pRaw, dataPos, pSchema->colId);
-    SDB_SET_INT32(pRaw, dataPos, pSchema->bytes);
-    SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
-  }
-
-  SDB_SET_RESERVE(pRaw, dataPos, TSDB_STABLE_RESERVE_SIZE)
-  SDB_SET_DATALEN(pRaw, dataPos);
-
-  return pRaw;
-}
-
-static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) {
-  int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
-
-  if (sver != TSDB_STABLE_VER_NUM) {
-    mError("failed to decode stable since %s", terrstr());
-    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
-    return NULL;
-  }
-
-  int32_t     size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema);
-  SSdbRow    *pRow = sdbAllocRow(size);
-  SStableObj *pStb = sdbGetRowObj(pRow);
-  if (pStb == NULL) return NULL;
-
-  int32_t dataPos = 0;
-  SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
-  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime)
-  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime)
-  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags)
-
-  pStb->columnSchema = calloc(pStb->numOfColumns, sizeof(SSchema));
-  pStb->tagSchema = calloc(pStb->numOfTags, sizeof(SSchema));
-
-  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
-    SSchema *pSchema = &pStb->columnSchema[i];
-    SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type);
-    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId);
-    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes);
-    SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
-  }
-
-  for (int32_t i = 0; i < pStb->numOfTags; ++i) {
-    SSchema *pSchema = &pStb->tagSchema[i];
-    SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type);
-    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId);
-    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes);
-    SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
-  }
-
-  SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STABLE_RESERVE_SIZE)
-
-  return pRow;
-}
-
-static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb) {
-  mTrace("stable:%s, perform insert action", pStb->name);
-  return 0;
-}
-
-static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb) {
-  mTrace("stable:%s, perform delete action", pStb->name);
-  return 0;
-}
-
-static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb) {
-  mTrace("stable:%s, perform update action", pOldStb->name);
-  atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime);
-  atomic_exchange_32(&pOldStb->version, pNewStb->version);
-
-  taosWLockLatch(&pOldStb->lock);
-  int32_t numOfTags = pNewStb->numOfTags;
-  int32_t tagSize = numOfTags * sizeof(SSchema);
-  int32_t numOfColumns = pNewStb->numOfColumns;
-  int32_t columnSize = numOfColumns * sizeof(SSchema);
-
-  if (pOldStb->numOfTags < numOfTags) {
-    pOldStb->tagSchema = malloc(tagSize);
-  }
-  if (pOldStb->numOfColumns < numOfColumns) {
-    pOldStb->columnSchema = malloc(columnSize);
-  }
-
-  memcpy(pOldStb->tagSchema, pNewStb->tagSchema, tagSize);
-  memcpy(pOldStb->columnSchema, pNewStb->columnSchema, columnSize);
-  taosWUnLockLatch(&pOldStb->lock);
-  return 0;
-}
-
-SStableObj *mndAcquireStb(SMnode *pMnode, char *stbName) {
-  SSdb *pSdb = pMnode->pSdb;
-  return sdbAcquire(pSdb, SDB_STABLE, stbName);
-}
-
-void mndReleaseStb(SMnode *pMnode, SStableObj *pStb) {
-  SSdb *pSdb = pMnode->pSdb;
-  sdbRelease(pSdb, pStb);
-}
-
-static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; }
-
-static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; }
-
-static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; }
-
-static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg) { return 0; }
-
-static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; }
-
-static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; }
-
-static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) {
-  SName name = {0};
-  tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-
-  char db[TSDB_TABLE_FNAME_LEN] = {0};
-  tNameGetFullDbName(&name, db);
-
-  return mndAcquireDb(pMnode, db);
-}
-
-static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) {
-  SMnode         *pMnode = pMsg->pMnode;
-  SStableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
-
-  mDebug("stable:%s, start to retrieve meta", pInfo->name);
-
-  SDbObj *pDb = mndGetDbByStbName(pMnode, pInfo->name);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    mError("stable:%s, failed to retrieve meta since %s", pInfo->name, terrstr());
-    return -1;
-  }
-
-  SStableObj *pStb = mndAcquireStb(pMnode, pInfo->name);
-  if (pStb == NULL) {
-    mndReleaseDb(pMnode, pDb);
-    terrno = TSDB_CODE_MND_INVALID_TABLE_NAME;
-    mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr());
-    return -1;
-  }
-
-  int32_t        contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema);
-  STableMetaMsg *pMeta = rpcMallocCont(contLen);
-  if (pMeta == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr());
-    return -1;
-  }
-
-  memcpy(pMeta->stableFname, pStb->name, TSDB_TABLE_FNAME_LEN);
-  pMeta->numOfTags = htonl(pStb->numOfTags);
-  pMeta->numOfColumns = htonl(pStb->numOfColumns);
-  pMeta->precision = pDb->cfg.precision;
-  pMeta->tableType = TSDB_SUPER_TABLE;
-  pMeta->update = pDb->cfg.update;
-  pMeta->sversion = htonl(pStb->version);
-  pMeta->suid = htonl(pStb->uid);
-
-  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
-    SSchema *pSchema = &pMeta->pSchema[i];
-    SSchema *pColumn = &pStb->columnSchema[i];
-    memcpy(pSchema->name, pColumn->name, TSDB_COL_NAME_LEN);
-    pSchema->type = pColumn->type;
-    pSchema->colId = htonl(pColumn->colId);
-    pSchema->bytes = htonl(pColumn->bytes);
-  }
-
-  for (int32_t i = 0; i < pStb->numOfTags; ++i) {
-    SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns];
-    SSchema *pTag = &pStb->tagSchema[i];
-    memcpy(pSchema->name, pTag->name, TSDB_COL_NAME_LEN);
-    pSchema->type = pTag->type;
-    pSchema->colId = htons(pTag->colId);
-    pSchema->bytes = htonl(pTag->bytes);
-  }
-
-  pMsg->pCont = pMeta;
-  pMsg->contLen = contLen;
-
-  mDebug("stable:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags);
-  return 0;
-}
-
-static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) {
-  SSdb *pSdb = pMnode->pSdb;
-
-  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    return -1;
-  }
-
-  int32_t numOfStables = 0;
-  void   *pIter = NULL;
-  while (1) {
-    SStableObj *pStb = NULL;
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb);
-    if (pIter == NULL) break;
-
-    if (strcmp(pStb->db, dbName) == 0) {
-      numOfStables++;
-    }
-
-    sdbRelease(pSdb, pStb);
-  }
-
-  *pNumOfStables = numOfStables;
-  return 0;
-}
-
-static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {
-  SMnode *pMnode = pMsg->pMnode;
-  SSdb   *pSdb = pMnode->pSdb;
-
-  if (mndGetNumOfStables(pMnode, pShow->db, &pShow->numOfRows) != 0) {
-    return -1;
-  }
-
-  int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
-
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 8;
-  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "create time");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "columns");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-  strcpy(pSchema[cols].name, "tags");
-  pSchema[cols].bytes = htons(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htons(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
-
-  return 0;
-}
-
-static void mnodeExtractTableName(char* tableId, char* name) {
-  int pos = -1;
-  int num = 0;
-  for (pos = 0; tableId[pos] != 0; ++pos) {
-    if (tableId[pos] == '.') num++;
-    if (num == 2) break;
-  }
-
-  if (num == 2) {
-    strcpy(name, tableId + pos + 1);
-  }
-}
-
-static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode     *pMnode = pMsg->pMnode;
-  SSdb       *pSdb = pMnode->pSdb;
-  int32_t     numOfRows = 0;
-  SStableObj *pStb = NULL;
-  int32_t     cols = 0;
-  char       *pWrite;
-  char        prefix[64] = {0};
-
-  tstrncpy(prefix, pShow->db, 64);
-  strcat(prefix, TS_PATH_DELIMITER);
-  int32_t prefixLen = (int32_t)strlen(prefix);
-
-  while (numOfRows < rows) {
-    pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStb);
-    if (pShow->pIter == NULL) break;
-
-    if (strncmp(pStb->name, prefix, prefixLen) != 0) {
-      sdbRelease(pSdb, pStb);
-      continue;
-    }
-
-    cols = 0;
-
-    char stableName[TSDB_TABLE_FNAME_LEN] = {0};
-    memcpy(stableName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen);
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, stableName);
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int64_t *)pWrite = pStb->createdTime;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pStb->numOfColumns;
-    cols++;
-
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pStb->numOfTags;
-    cols++;
-
-    numOfRows++;
-    sdbRelease(pSdb, pStb);
-  }
-
-  pShow->numOfReads += numOfRows;
-  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
-  return numOfRows;
-}
-
-static void mndCancelGetNextStable(SMnode *pMnode, void *pIter) {
-  SSdb *pSdb = pMnode->pSdb;
-  sdbCancelFetch(pSdb, pIter);
-}

source/dnode/mnode/impl/src/mndStb.c

@@ -0,0 +1,675 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mndStb.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndShow.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "tname.h"
+
+#define TSDB_STB_VER_NUM 1
+#define TSDB_STB_RESERVE_SIZE 64
+
+static SSdbRaw *mndStbActionEncode(SStbObj *pStb);
+static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
+static int32_t  mndStbActionInsert(SSdb *pSdb, SStbObj *pStb);
+static int32_t  mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
+static int32_t  mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb);
+static int32_t  mndProcessCreateStbMsg(SMnodeMsg *pMsg);
+static int32_t  mndProcessAlterStbMsg(SMnodeMsg *pMsg);
+static int32_t  mndProcessDropStbMsg(SMnodeMsg *pMsg);
+static int32_t  mndProcessCreateStbInRsp(SMnodeMsg *pMsg);
+static int32_t  mndProcessAlterStbInRsp(SMnodeMsg *pMsg);
+static int32_t  mndProcessDropStbInRsp(SMnodeMsg *pMsg);
+static int32_t  mndProcessStbMetaMsg(SMnodeMsg *pMsg);
+static int32_t  mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
+static int32_t  mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
+static void     mndCancelGetNextStb(SMnode *pMnode, void *pIter);
+
+int32_t mndInitStb(SMnode *pMnode) {
+  SSdbTable table = {.sdbType = SDB_STB,
+                     .keyType = SDB_KEY_BINARY,
+                     .encodeFp = (SdbEncodeFp)mndStbActionEncode,
+                     .decodeFp = (SdbDecodeFp)mndStbActionDecode,
+                     .insertFp = (SdbInsertFp)mndStbActionInsert,
+                     .updateFp = (SdbUpdateFp)mndStbActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndStbActionDelete};
+
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB, mndProcessCreateStbMsg);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB, mndProcessAlterStbMsg);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB, mndProcessDropStbMsg);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB_IN_RSP, mndProcessCreateStbInRsp);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB_IN_RSP, mndProcessAlterStbInRsp);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB_IN_RSP, mndProcessDropStbInRsp);
+  mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStbMetaMsg);
+
+  mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta);
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb);
+
+  return sdbSetTable(pMnode->pSdb, table);
+}
+
+void mndCleanupStb(SMnode *pMnode) {}
+
+static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
+  int32_t  size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema);
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size);
+  if (pRaw == NULL) return NULL;
+
+  int32_t dataPos = 0;
+  SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
+  SDB_SET_INT64(pRaw, dataPos, pStb->createdTime)
+  SDB_SET_INT64(pRaw, dataPos, pStb->updateTime)
+  SDB_SET_INT64(pRaw, dataPos, pStb->uid)
+  SDB_SET_INT64(pRaw, dataPos, pStb->version)
+  SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns)
+  SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags)
+
+  int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
+  for (int32_t i = 0; i < totalCols; ++i) {
+    SSchema *pSchema = &pStb->pSchema[i];
+    SDB_SET_INT8(pRaw, dataPos, pSchema->type);
+    SDB_SET_INT32(pRaw, dataPos, pSchema->colId);
+    SDB_SET_INT32(pRaw, dataPos, pSchema->bytes);
+    SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
+  }
+
+  SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE)
+  SDB_SET_DATALEN(pRaw, dataPos);
+
+  return pRaw;
+}
+
+static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
+
+  if (sver != TSDB_STB_VER_NUM) {
+    mError("failed to decode stable since %s", terrstr());
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    return NULL;
+  }
+
+  int32_t  size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema);
+  SSdbRow *pRow = sdbAllocRow(size);
+  SStbObj *pStb = sdbGetRowObj(pRow);
+  if (pStb == NULL) return NULL;
+
+  int32_t dataPos = 0;
+  SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
+  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime)
+  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime)
+  SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags)
+
+  int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
+  pStb->pSchema = calloc(totalCols, sizeof(SSchema));
+
+  for (int32_t i = 0; i < totalCols; ++i) {
+    SSchema *pSchema = &pStb->pSchema[i];
+    SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type);
+    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId);
+    SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes);
+    SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN);
+  }
+
+  SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STB_RESERVE_SIZE)
+
+  return pRow;
+}
+
+static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
+  mTrace("stb:%s, perform insert action", pStb->name);
+  return 0;
+}
+
+static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
+  mTrace("stb:%s, perform delete action", pStb->name);
+  return 0;
+}
+
+static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb) {
+  mTrace("stb:%s, perform update action", pOldStb->name);
+  atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime);
+  atomic_exchange_32(&pOldStb->version, pNewStb->version);
+
+  taosWLockLatch(&pOldStb->lock);
+  int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns;
+  int32_t totalSize = totalCols * sizeof(SSchema);
+
+  if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) {
+    pOldStb->pSchema = malloc(totalSize);
+  }
+
+  memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize);
+  taosWUnLockLatch(&pOldStb->lock);
+  return 0;
+}
+
+SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) {
+  SSdb *pSdb = pMnode->pSdb;
+  return sdbAcquire(pSdb, SDB_STB, stbName);
+}
+
+void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbRelease(pSdb, pStb);
+}
+
+static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) {
+  SName name = {0};
+  tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+
+  char db[TSDB_TABLE_FNAME_LEN] = {0};
+  tNameGetFullDbName(&name, db);
+
+  return mndAcquireDb(pMnode, db);
+}
+
+static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) {
+  pCreate->numOfColumns = htonl(pCreate->numOfColumns);
+  pCreate->numOfTags = htonl(pCreate->numOfTags);
+  int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags;
+  for (int32_t i = 0; i < totalCols; ++i) {
+    SSchema *pSchema = &pCreate->pSchema[i];
+    pSchema->colId = htonl(pSchema->colId);
+    pSchema->bytes = htonl(pSchema->bytes);
+  }
+
+  if (pCreate->igExists < 0 || pCreate->igExists > 1) {
+    terrno = TSDB_CODE_MND_STB_INVALID_IGEXIST;
+    return -1;
+  }
+
+  if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfColumns > TSDB_MAX_COLUMNS) {
+    terrno = TSDB_CODE_MND_STB_INVALID_COLS_NUM;
+    return -1;
+  }
+
+  if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) {
+    terrno = TSDB_CODE_MND_STB_INVALID_TAGS_NUM;
+    return -1;
+  }
+
+  int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS);
+  for (int32_t i = 0; i < totalCols; ++i) {
+    SSchema *pSchema = &pCreate->pSchema[i];
+    if (pSchema->type <= 0) {
+      terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE;
+      return -1;
+    }
+    if (pSchema->colId < 0 || pSchema->colId >= maxColId) {
+      terrno = TSDB_CODE_MND_STB_INVALID_COL_ID;
+      return -1;
+    }
+    if (pSchema->bytes <= 0) {
+      terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES;
+      return -1;
+    }
+    if (pSchema->name[0] == 0) {
+      terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) {
+  SStbObj stbObj = {0};
+  tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
+  tstrncpy(stbObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN);
+  stbObj.createdTime = taosGetTimestampMs();
+  stbObj.updateTime = stbObj.createdTime;
+  stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  stbObj.version = 1;
+  stbObj.numOfColumns = pCreate->numOfColumns;
+  stbObj.numOfTags = pCreate->numOfTags;
+
+  int32_t totalCols = stbObj.numOfColumns + stbObj.numOfTags;
+  int32_t totalSize = totalCols * sizeof(SSchema);
+  stbObj.pSchema = malloc(totalSize);
+  if (stbObj.pSchema == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  memcpy(stbObj.pSchema, pCreate->pSchema, totalSize);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
+  if (pTrans == NULL) {
+    mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+  mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
+
+  SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj);
+  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
+
+  SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj);
+  if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
+    mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+  SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj);
+  if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
+    mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
+
+  if (mndTransPrepare(pTrans) != 0) {
+    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return 0;
+}
+
+static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) {
+  SMnode        *pMnode = pMsg->pMnode;
+  SCreateStbMsg *pCreate = pMsg->rpcMsg.pCont;
+
+  mDebug("stb:%s, start to create", pCreate->name);
+
+  if (mndCheckStbMsg(pCreate) != 0) {
+    mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+
+  SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name);
+  if (pStb != NULL) {
+    sdbRelease(pMnode->pSdb, pStb);
+    if (pCreate->igExists) {
+      mDebug("stb:%s, already exist, ignore exist is set", pCreate->name);
+      return 0;
+    } else {
+      terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
+      mError("db:%s, failed to create since %s", pCreate->name, terrstr());
+      return -1;
+    }
+  }
+
+  SDbObj *pDb = mndAcquireDbByStb(pMnode, pCreate->name);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+
+  int32_t code = mndCreateStb(pMnode, pMsg, pCreate, pDb);
+  mndReleaseDb(pMnode, pDb);
+
+  if (code != 0) {
+    terrno = code;
+    mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; }
+
+static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) {
+  SSchema *pSchema = &pAlter->schema;
+  pSchema->colId = htonl(pSchema->colId);
+  pSchema->bytes = htonl(pSchema->bytes);
+
+  if (pSchema->type <= 0) {
+    terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE;
+    return -1;
+  }
+  if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) {
+    terrno = TSDB_CODE_MND_STB_INVALID_COL_ID;
+    return -1;
+  }
+  if (pSchema->bytes <= 0) {
+    terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES;
+    return -1;
+  }
+  if (pSchema->name[0] == 0) {
+    terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME;
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; }
+
+static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) {
+  SMnode       *pMnode = pMsg->pMnode;
+  SAlterStbMsg *pAlter = pMsg->rpcMsg.pCont;
+
+  mDebug("stb:%s, start to alter", pAlter->name);
+
+  if (mndCheckAlterStbMsg(pAlter) != 0) {
+    mError("stb:%s, failed to alter since %s", pAlter->name, terrstr());
+    return -1;
+  }
+
+  SStbObj *pStb = mndAcquireStb(pMnode, pAlter->name);
+  if (pStb == NULL) {
+    terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+    mError("stb:%s, failed to alter since %s", pAlter->name, terrstr());
+    return -1;
+  }
+
+  SStbObj stbObj = {0};
+  memcpy(&stbObj, pStb, sizeof(SStbObj));
+
+  int32_t code = mndUpdateStb(pMnode, pMsg, pStb, &stbObj);
+  mndReleaseStb(pMnode, pStb);
+
+  if (code != 0) {
+    mError("stb:%s, failed to alter since %s", pAlter->name, tstrerror(code));
+    return code;
+  }
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; }
+
+static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) {
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
+  if (pTrans == NULL) {
+    mError("stb:%s, failed to drop since %s", pStb->name, terrstr());
+    return -1;
+  }
+  mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
+
+  SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
+  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
+
+  SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
+  if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
+    mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
+
+  SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
+  if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
+    mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
+
+  if (mndTransPrepare(pTrans) != 0) {
+    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return 0;
+}
+
+static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) {
+  SMnode      *pMnode = pMsg->pMnode;
+  SDropStbMsg *pDrop = pMsg->rpcMsg.pCont;
+
+  mDebug("stb:%s, start to drop", pDrop->name);
+
+  SStbObj *pStb = mndAcquireStb(pMnode, pDrop->name);
+  if (pStb == NULL) {
+    if (pDrop->igNotExists) {
+      mDebug("stb:%s, not exist, ignore not exist is set", pDrop->name);
+      return 0;
+    } else {
+      terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+      mError("stb:%s, failed to drop since %s", pDrop->name, terrstr());
+      return -1;
+    }
+  }
+
+  int32_t code = mndDropStb(pMnode, pMsg, pStb);
+  mndReleaseStb(pMnode, pStb);
+
+  if (code != 0) {
+    terrno = code;
+    mError("stb:%s, failed to drop since %s", pDrop->name, terrstr());
+    return -1;
+  }
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; }
+
+static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
+  SMnode      *pMnode = pMsg->pMnode;
+  SStbInfoMsg *pInfo = pMsg->rpcMsg.pCont;
+
+  mDebug("stb:%s, start to retrieve meta", pInfo->name);
+
+  SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->name);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    mError("stb:%s, failed to retrieve meta since %s", pInfo->name, terrstr());
+    return -1;
+  }
+
+  SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name);
+  if (pStb == NULL) {
+    mndReleaseDb(pMnode, pDb);
+    terrno = TSDB_CODE_MND_INVALID_TABLE_NAME;
+    mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr());
+    return -1;
+  }
+
+  int32_t        contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema);
+  STableMetaMsg *pMeta = rpcMallocCont(contLen);
+  if (pMeta == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr());
+    return -1;
+  }
+
+  memcpy(pMeta->stbFname, pStb->name, TSDB_TABLE_FNAME_LEN);
+  pMeta->numOfTags = htonl(pStb->numOfTags);
+  pMeta->numOfColumns = htonl(pStb->numOfColumns);
+  pMeta->precision = pDb->cfg.precision;
+  pMeta->tableType = TSDB_SUPER_TABLE;
+  pMeta->update = pDb->cfg.update;
+  pMeta->sversion = htonl(pStb->version);
+  pMeta->suid = htonl(pStb->uid);
+
+  for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
+    SSchema *pSchema = &pMeta->pSchema[i];
+    SSchema *pSrcSchema = &pStb->pSchema[i];
+    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
+    pSchema->type = pSrcSchema->type;
+    pSchema->colId = htonl(pSrcSchema->colId);
+    pSchema->bytes = htonl(pSrcSchema->bytes);
+  }
+
+  pMsg->pCont = pMeta;
+  pMsg->contLen = contLen;
+
+  mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags);
+  return 0;
+}
+
+static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) {
+  SSdb *pSdb = pMnode->pSdb;
+
+  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    return -1;
+  }
+
+  int32_t numOfStbs = 0;
+  void   *pIter = NULL;
+  while (1) {
+    SStbObj *pStb = NULL;
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb);
+    if (pIter == NULL) break;
+
+    if (strcmp(pStb->db, dbName) == 0) {
+      numOfStbs++;
+    }
+
+    sdbRelease(pSdb, pStb);
+  }
+
+  *pNumOfStbs = numOfStbs;
+  return 0;
+}
+
+static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {
+  SMnode *pMnode = pMsg->pMnode;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) {
+    return -1;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->pSchema;
+
+  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "name");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 8;
+  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
+  strcpy(pSchema[cols].name, "create time");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "columns");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "tags");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
+
+  return 0;
+}
+
+static void mnodeExtractTableName(char *tableId, char *name) {
+  int pos = -1;
+  int num = 0;
+  for (pos = 0; tableId[pos] != 0; ++pos) {
+    if (tableId[pos] == '.') num++;
+    if (num == 2) break;
+  }
+
+  if (num == 2) {
+    strcpy(name, tableId + pos + 1);
+  }
+}
+
+static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
+  SMnode  *pMnode = pMsg->pMnode;
+  SSdb    *pSdb = pMnode->pSdb;
+  int32_t  numOfRows = 0;
+  SStbObj *pStb = NULL;
+  int32_t  cols = 0;
+  char    *pWrite;
+  char     prefix[64] = {0};
+
+  tstrncpy(prefix, pShow->db, 64);
+  strcat(prefix, TS_PATH_DELIMITER);
+  int32_t prefixLen = (int32_t)strlen(prefix);
+
+  while (numOfRows < rows) {
+    pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb);
+    if (pShow->pIter == NULL) break;
+
+    if (strncmp(pStb->name, prefix, prefixLen) != 0) {
+      sdbRelease(pSdb, pStb);
+      continue;
+    }
+
+    cols = 0;
+
+    char stbName[TSDB_TABLE_FNAME_LEN] = {0};
+    memcpy(stbName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen);
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, stbName);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t *)pWrite = pStb->createdTime;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pStb->numOfColumns;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pStb->numOfTags;
+    cols++;
+
+    numOfRows++;
+    sdbRelease(pSdb, pStb);
+  }
+
+  pShow->numOfReads += numOfRows;
+  mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  return numOfRows;
+}
+
+static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndUser.c

@@ -171,7 +171,7 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
 static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) {
   mTrace("user:%s, perform update action", pOldUser->user);
   memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN);
-  memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN);
+  memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN);
   memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN);
   pOldUser->createdTime = pNewUser->createdTime;
   pOldUser->updateTime = pNewUser->updateTime;
@@ -489,7 +489,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p
 
   pShow->numOfRows = sdbGetSize(pSdb, SDB_USER);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -45,7 +45,7 @@ static void    mndCancelGetNextVnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitVgroup(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_VGROUP,
-                     .keyType = SDB_KEY_BINARY,
+                     .keyType = SDB_KEY_INT32,
                      .encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
                      .decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
                      .insertFp = (SdbInsertFp)mndVgroupActionInsert,
@@ -238,7 +238,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
   }
 
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }
@@ -337,7 +337,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
   pShow->replica = dnodeId;
   pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tableFname, mndShowStr(pShow->type));
+  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
 
   return 0;
 }

source/dnode/mnode/impl/src/mnode.c

@@ -24,7 +24,7 @@
 #include "mndMnode.h"
 #include "mndProfile.h"
 #include "mndShow.h"
-#include "mndStable.h"
+#include "mndStb.h"
 #include "mndSync.h"
 #include "mndTelem.h"
 #include "mndTrans.h"
@@ -131,7 +131,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
-  if (mndAllocStep(pMnode, "mnode-stable", mndInitStable, mndCleanupStable) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
   if (pMnode->clusterId <= 0) {
     if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1;
@@ -205,7 +205,6 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
   pMnode->cfg.sver = pOption->cfg.sver;
   pMnode->cfg.enableTelem = pOption->cfg.enableTelem;
   pMnode->cfg.statusInterval = pOption->cfg.statusInterval;
-  pMnode->cfg.mnodeEqualVnodeNum = pOption->cfg.mnodeEqualVnodeNum;
   pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer;
   pMnode->cfg.timezone = strdup(pOption->cfg.timezone);
   pMnode->cfg.locale = strdup(pOption->cfg.locale);
@@ -215,7 +214,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
 
   if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL ||
       pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 ||
-      pMnode->cfg.statusInterval < 1 || pOption->cfg.mnodeEqualVnodeNum < 0) {
+      pMnode->cfg.statusInterval < 1) {
     terrno = TSDB_CODE_MND_INVALID_OPTIONS;
     return -1;
   }
@@ -337,8 +336,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
   }
 
   SRpcConnInfo connInfo = {0};
-  if (rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
+  if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
-    mndCleanupMsg(pMsg);
+    taosFreeQitem(pMsg);
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
     mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr());
     return NULL;
@@ -355,6 +354,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
 
 void mndCleanupMsg(SMnodeMsg *pMsg) {
   mTrace("msg:%p, app:%p is destroyed, RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle);
+  rpcFreeCont(pMsg->rpcMsg.pCont);
+  pMsg->rpcMsg.pCont = NULL;
   taosFreeQitem(pMsg);
 }
 
@@ -368,7 +369,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) {
   int32_t code = 0;
   int32_t msgType = pMsg->rpcMsg.msgType;
   void   *ahandle = pMsg->rpcMsg.ahandle;
-  bool    isReq = (msgType % 2 == 1);
+  bool    isReq = (msgType & 1U);
 
   mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]);
 
@@ -430,3 +431,10 @@ void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
 void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
 
 void mndProcessApplyMsg(SMnodeMsg *pMsg) {}
+
+uint64_t mndGenerateUid(char *name, int32_t len) {
+  int64_t  us = taosGetTimestampUs();
+  int32_t  hashval = MurmurHash3_32(name, len);
+  uint64_t x = (us & 0x000000FFFFFFFFFF) << 24;
+  return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
+}

source/libs/index/inc/index_fst.h

@@ -26,10 +26,24 @@ extern "C" {
 #include "index_fst_counting_writer.h"
 #include "index_fst_automation.h"
 
-
-typedef struct FstNode FstNode;
 #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) 
 
+typedef struct Fst     Fst;
+typedef struct FstNode FstNode;
+
+typedef enum { Included, Excluded, Unbounded} FstBound; 
+
+typedef struct FstBoundWithData {
+  FstSlice data; 
+  FstBound type;
+} FstBoundWithData;
+
+typedef struct FstStreamBuilder {
+  Fst *fst; 
+  AutomationCtx *aut;
+  FstBoundWithData *min;  
+  FstBoundWithData *max;
+} FstStreamBuilder, FstStreamWithStateBuilder;
 
 typedef struct FstRange {
   uint64_t start;
@@ -39,16 +53,9 @@ typedef struct FstRange {
 
 typedef enum {GE, GT, LE, LT} RangeType;
 typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State;
-
 typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType;
 
 
-typedef enum { Included, Excluded, Unbounded} FstBound; 
-typedef struct FstBoundWithData {
-  FstSlice data; 
-  FstBound type;
-} FstBoundWithData;
-
 FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data);
 bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
 bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); 
@@ -60,8 +67,6 @@ typedef struct FstOutput {
   Output out;
 } FstOutput;
 
- 
-
 /*
  * 
  * UnFinished node and helper function
@@ -275,6 +280,9 @@ FstNode*     fstGetRoot(Fst *fst);
 FstType      fstGetType(Fst *fst); 
 CompiledAddr fstGetRootAddr(Fst *fst);
 Output       fstEmptyFinalOutput(Fst *fst, bool *null);
+FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
+FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
+
 bool         fstVerify(Fst *fst);
 
 
@@ -292,12 +300,12 @@ void streamStateDestroy(void *s);
 
 typedef struct StreamWithState {
   Fst           *fst;
-  Automation *aut; 
+  AutomationCtx *aut; 
   SArray        *inp;
   FstOutput     emptyOutput;
   SArray        *stack; // <StreamState>
   FstBoundWithData *endAt;
-} StreamWithState ;
+} StreamWithState;
 
 typedef struct StreamWithStateResult {
   FstSlice data;  
@@ -310,19 +318,13 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
 void swsResultDestroy(StreamWithStateResult *result);
 
 typedef void* (*StreamCallback)(void *);
-StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ;
+StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ;
 void streamWithStateDestroy(StreamWithState *sws);
 bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);           
+
 StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
 
-typedef struct FstStreamBuilder {
+FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); 
-  Fst *fst; 
-  Automation *aut;
-  FstBoundWithData *min;  
-  FstBoundWithData *max;
-} FstStreamBuilder;
-
-FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); 
 // set up bound range
 // refator, simple code by marco 
 

source/libs/index/inc/index_fst_automation.h

@@ -19,33 +19,50 @@
 extern "C" {
 #endif
 
+#include "index_fst_util.h"
 typedef struct AutomationCtx AutomationCtx;
 
+typedef enum AutomationType {
+  AUTOMATION_PREFIX,
+  AUTMMATION_MATCH 
+} AutomationType;
+
 typedef struct StartWith {
   AutomationCtx  *autoSelf;
 } StartWith;
 
 typedef struct Complement {
   AutomationCtx *autoSelf;
-  
 } Complement;
 
 // automation 
 typedef struct AutomationCtx {
-// automation interface
+  AutomationType type; 
   void *data;
 } AutomationCtx;
 
-typedef struct Automation {
-  void* (*start)() ; 
-  bool (*isMatch)(void *);
-  bool (*canMatch)(void *data);
-  bool (*willAlwaysMatch)(void *state); 
-  void* (*accept)(void *state, uint8_t byte);
-  void* (*acceptEof)(void *state);
-  void *data;
-} Automation; 
 
+
+typedef enum StartWithStateKind { Done, Running } StartWithStateKind; 
+
+typedef struct StartWithStateValue {
+  StartWithStateKind kind;
+  void *value;
+} StartWithStateValue;
+
+typedef struct AutomationFunc {
+  void* (*start)(AutomationCtx *ctx) ; 
+  bool (*isMatch)(AutomationCtx *ctx, void *);
+  bool (*canMatch)(AutomationCtx *ctx, void *data);
+  bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); 
+  void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte);
+  void* (*acceptEof)(AutomationCtx *ct, void *state);
+} AutomationFunc; 
+
+AutomationCtx *automCtxCreate(void *data, AutomationType type);
+void automCtxDestroy(AutomationCtx *ctx);
+
+extern AutomationFunc automFuncs[]; 
 #ifdef __cplusplus
 }
 #endif

source/libs/index/src/index_fst.c

@@ -1072,7 +1072,6 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
     tOut += trn.out; 
     root = fstGetNode(fst, trn.addr);
     taosArrayPush(nodes, &root);
-    //fstNodeDestroy(root);
   }
   if (!FST_NODE_IS_FINAL(root)) {
     return false;
@@ -1091,6 +1090,12 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
   
   return true; 
 }
+FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) {
+  return fstStreamBuilderCreate(fst, ctx);
+}
+FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) {
+  return fstStreamBuilderCreate(fst, ctx);
+}
 
 FstNode *fstGetRoot(Fst *fst) {
   if (fst->root != NULL) {
@@ -1177,7 +1182,7 @@ void fstBoundDestroy(FstBoundWithData *bound) {
   free(bound);
 }
 
-StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) {
+StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) {
   StreamWithState *sws = calloc(1, sizeof(StreamWithState));
   if (sws == NULL) { return NULL; } 
 
@@ -1204,6 +1209,8 @@ void streamWithStateDestroy(StreamWithState *sws) {
 }
 
 bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
+
+  AutomationCtx *aut = sws->aut;
   if (fstBoundWithDataIsEmpty(min)) {
     if (fstBoundWithDataIsIncluded(min)) {
        sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null));
@@ -1211,7 +1218,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
     StreamState s = {.node     = fstGetRoot(sws->fst),
                      .trans    = 0,
                      .out      = {.null = false, .out = 0},
-                     .autState = sws->aut->start()}; // auto.start callback 
+                     .autState = automFuncs[aut->type].start(aut)}; // auto.start callback 
     taosArrayPush(sws->stack, &s);
     return true;
   } 
@@ -1229,7 +1236,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
 
   FstNode *node = fstGetRoot(sws->fst); 
   Output  out = 0;
-  void*   autState = sws->aut->start();  
+  //void*   autState = sws->aut->start();  
+  void* autState = automFuncs[aut->type].start(aut);
 
   int32_t len; 
   uint8_t *data = fstSliceData(key, &len);  
@@ -1241,7 +1249,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
       FstTransition trn;
       fstNodeGetTransitionAt(node, res, &trn);   
       void *preState = autState;
-      autState = sws->aut->accept(preState, b);
+      // autState = sws->aut->accept(preState, b);
+      autState = automFuncs[aut->type].accept(aut, preState, b);
       taosArrayPush(sws->inp, &b);
       StreamState s = {.node     = node, 
                        .trans    = res + 1, 
@@ -1298,6 +1307,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
 }          
 
 StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) {
+  AutomationCtx *aut = sws->aut;
   FstOutput output = sws->emptyOutput; 
   if (output.null == false) {
     FstSlice emptySlice = fstSliceCreate(NULL, 0);   
@@ -1306,15 +1316,15 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
       sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); 
       return NULL;
     }
-    void* start = sws->aut->start();
+    void *start = automFuncs[aut->type].start(aut);
-    if (sws->aut->isMatch(start)) { 
+    if (automFuncs[aut->type].isMatch(aut, start)) {
       FstSlice s = fstSliceCreate(NULL, 0);
       return swsResultCreate(&s, output, callback(start));
     }
   }
   while (taosArrayGetSize(sws->stack) > 0) {
     StreamState *p = (StreamState *)taosArrayPop(sws->stack);     
-    if (p->trans >= FST_NODE_LEN(p->node) || !sws->aut->canMatch(p->autState)) {
+    if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) {
       if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
         taosArrayPop(sws->inp);
       }
@@ -1324,16 +1334,18 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
     FstTransition trn; 
     fstNodeGetTransitionAt(p->node, p->trans, &trn);
     Output out = p->out.out + trn.out;
-    void* nextState = sws->aut->accept(p->autState, trn.inp);
+    void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp);
     void* tState = callback(nextState);
-    bool isMatch = sws->aut->isMatch(nextState);
+    bool isMatch = automFuncs[aut->type].isMatch(aut, nextState);
+    //bool isMatch = sws->aut->isMatch(nextState);
     FstNode *nextNode = fstGetNode(sws->fst, trn.addr); 
     taosArrayPush(sws->inp, &(trn.inp)); 
 
     if (FST_NODE_IS_FINAL(nextNode)) {
-      void *eofState = sws->aut->acceptEof(nextState); 
+      //void *eofState = sws->aut->acceptEof(nextState); 
+      void *eofState = automFuncs[aut->type].acceptEof(aut, nextState);
       if (eofState != NULL) {
-        isMatch = sws->aut->isMatch(eofState); 
+        isMatch = automFuncs[aut->type].isMatch(aut, eofState);
       }
     } 
     StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};  
@@ -1391,7 +1403,7 @@ void streamStateDestroy(void *s) {
   //free(s->autoState);
 }
 
-FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) {
+FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) {
   FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder));
   if (NULL == b) { return NULL; }
 
@@ -1434,3 +1446,5 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang
 
 
 
+
+

source/libs/index/src/index_fst_automation.c

@@ -13,3 +13,94 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "index_fst_automation.h"
+
+
+// prefix query, impl later
+static void* prefixStart(AutomationCtx *ctx) {  
+  StartWithStateValue *data = (StartWithStateValue *)(ctx->data);
+  return data;
+};
+static bool prefixIsMatch(AutomationCtx *ctx, void *data) {
+  return true;
+} 
+static bool prefixCanMatch(AutomationCtx *ctx, void *data) {
+  return true;
+}
+static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) {
+  return true;
+}
+static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
+  return NULL;
+}
+static void* prefixAcceptEof(AutomationCtx *ctx, void *state) {
+  return NULL;
+}
+
+// pattern query, impl later
+
+static void* patternStart(AutomationCtx *ctx) {
+  return NULL;
+}
+static bool patternIsMatch(AutomationCtx *ctx, void *data) {
+  return true;
+} 
+static bool patternCanMatch(AutomationCtx *ctx, void *data) {
+  return true;
+} 
+static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) {
+  return true;
+}
+
+static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
+  return NULL;
+}
+
+static void* patternAcceptEof(AutomationCtx *ctx, void *state) {
+  return NULL;
+}
+
+AutomationFunc automFuncs[]  = {{
+    prefixStart,        
+    prefixIsMatch, 
+    prefixCanMatch,
+    prefixWillAlwaysMatch,
+    prefixAccept,
+    prefixAcceptEof
+  },  
+  {
+    patternStart,
+    patternIsMatch,
+    patternCanMatch,
+    patternWillAlwaysMatch,
+    patternAccept,
+    patternAcceptEof
+  }
+  // add more search type
+};
+
+AutomationCtx* automCtxCreate(void *data, AutomationType type) {
+  AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx));
+  if (ctx == NULL) { return NULL; }
+
+  if (type == AUTOMATION_PREFIX) {
+    StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue));   
+    swsv->kind  = Done; 
+    swsv->value = NULL; 
+    ctx->data = (void *)swsv;
+  } else if (type == AUTMMATION_MATCH) {
+      
+  } else {
+    // add more search type
+  }
+
+  ctx->type = type;
+  return ctx; 
+} 
+void automCtxDestroy(AutomationCtx *ctx) {
+  if (ctx->type == AUTOMATION_PREFIX) {
+    free(ctx->data);
+  } else if (ctx->type == AUTMMATION_MATCH) {
+  }
+  free(ctx);
+}

source/libs/index/test/indexTests.cpp

@@ -65,6 +65,7 @@ class FstReadMemory {
     
    ~FstReadMemory() {
     fstCountingWriterDestroy(_w);
+    fstDestroy(_fst);
     fstSliceDestroy(&_s);
   } 
   
@@ -129,10 +130,12 @@ class FstReadMemory {
 //}
 
 
+#define L 100
+#define M 100
+#define N 100
 
 int Performance_fstWriteRecords(FstWriter *b) {
   std::string str("aa"); 
-  int L = 100, M = 100, N = 10;
   for (int i = 0; i < L; i++) {
     str[0] = 'a' + i;
     str.resize(2); 
@@ -150,22 +153,29 @@ int Performance_fstWriteRecords(FstWriter *b) {
 }
 
 void Performance_fstReadRecords(FstReadMemory *m) {
-   std::string str("a");
+  std::string str("aa");
-   for (int i = 0; i < 50; i++) {
+  for (int i = 0; i < M; i++) {
-     //std::string str("aa"); 
+    str[0] = 'a' + i;
+    str.resize(2); 
+    for(int j = 0; j < N; j++) {
+      str[1] = 'a' + j;
+      str.resize(2);
+      for (int k = 0; k < L; k++) {
         str.push_back('a');
-     uint64_t out, cost;
+        uint64_t val, cost; 
-     bool ok = m->GetWithTimeCostUs(str, &out, &cost); 
+        if (m->GetWithTimeCostUs(str, &val, &cost)) {
-     if (ok == true) {
+          printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
-      printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost);
         } else {
-      printf("failed to get(%s)\n", str.c_str());
+          printf("failed to get key: %s\n", str.c_str());
+        }
+      }
     } 
   }
 }
 void checkFstPerf() {
   FstWriter *fw = new FstWriter;
   int64_t s = taosGetTimestampUs();
+
   int num = Performance_fstWriteRecords(fw);
   int64_t e = taosGetTimestampUs();
   printf("write %d record cost %" PRId64"us\n", num,  e - s);
@@ -173,13 +183,11 @@ void checkFstPerf() {
 
   FstReadMemory *m = new FstReadMemory(1024 * 64);
   if (m->init()) {
-    uint64_t val;
+    printf("success to init fst read");  
-    if(m->Get("aaaaaaa", &val)) {
-      std::cout << "succes to Get val: " << val << std::endl;
-    } else {
-      std::cout << "failed to Get " << std::endl;
-    } 
   }  
+  Performance_fstReadRecords(m); 
+   
+  delete m;
 } 
 
 

source/libs/parser/inc/astGenerator.h

@@ -121,7 +121,7 @@ typedef struct SRelationInfo {
 
 typedef struct SCreatedTableInfo {
   SToken             name;        // table name token
-  SToken             stableName;  // super table name token , for using clause
+  SToken             stbName;     // super table name token , for using clause
   SArray            *pTagNames;   // create by using super table, tag name
   SArray            *pTagVals;    // create by using super table, tag value
   char              *fullname;    // table full name

source/libs/parser/inc/dataBlockMgt.h

@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DATABLOCKMGT_H
+#define TDENGINE_DATABLOCKMGT_H
+
+#include "catalog.h"
+#include "os.h"
+#include "ttypes.h"
+#include "tname.h"
+
+#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
+
+typedef enum EOrderStatus {
+  ORDER_STATUS_UNKNOWN = 0,
+  ORDER_STATUS_ORDERED = 1,
+  ORDER_STATUS_DISORDERED = 2,
+} EOrderStatus;
+
+typedef enum EValStat {
+  VAL_STAT_HAS = 0x0,    // 0 means has val
+  VAL_STAT_NONE = 0x01,  // 1 means no val
+} EValStat;
+
+typedef enum ERowCompareStat {
+  ROW_COMPARE_NO_NEED = 0,
+  ROW_COMPARE_NEED = 1,
+} ERowCompareStat;
+
+typedef struct SBoundColumn {
+  int32_t offset;   // all column offset value
+  int32_t toffset;  // first part offset for SDataRow TODO: get offset from STSchema on future
+  uint8_t valStat;  // EValStat. denote if current column bound or not(0 means has val, 1 means no val)
+} SBoundColumn;
+
+typedef struct {
+  uint16_t schemaColIdx;
+  uint16_t boundIdx;
+  uint16_t finalIdx;
+} SBoundIdxInfo;
+
+typedef struct SParsedDataColInfo {
+  int16_t        numOfCols;
+  int16_t        numOfBound;
+  uint16_t       flen;        // TODO: get from STSchema
+  uint16_t       allNullLen;  // TODO: get from STSchema
+  uint16_t       extendedVarLen;
+  int32_t        *boundedColumns;  // bound column idx according to schema
+  SBoundColumn   *cols;
+  SBoundIdxInfo  *colIdxInfo;
+  int8_t         orderStatus;  // bound columns
+} SParsedDataColInfo;
+
+typedef struct SMemRowInfo {
+  int32_t dataLen;  // len of SDataRow
+  int32_t kvLen;    // len of SKVRow
+} SMemRowInfo;
+
+typedef struct {
+  uint8_t      memRowType;   // default is 0, that is SDataRow 
+  uint8_t      compareStat;  // 0 no need, 1 need compare
+  TDRowTLenT   kvRowInitLen;
+  SMemRowInfo *rowInfo;
+} SMemRowBuilder;
+
+typedef struct SParamInfo {
+  int32_t  idx;
+  uint8_t  type;
+  uint8_t  timePrec;
+  int16_t  bytes;
+  uint32_t offset;
+} SParamInfo;
+
+typedef struct STableDataBlocks {
+  SName       tableName;
+  int8_t      tsSource;     // where does the UNIX timestamp come from, server or client
+  bool        ordered;      // if current rows are ordered or not
+  int64_t     vgId;         // virtual group id
+  int64_t     prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
+  int32_t     numOfTables;  // number of tables in current submit block
+  int32_t     rowSize;      // row size for current table
+  uint32_t    nAllocSize;
+  uint32_t    headerSize;   // header for table info (uid, tid, submit metadata)
+  uint32_t    size;
+  STableMeta *pTableMeta;   // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
+  char       *pData;
+  bool        cloned;
+  STagData    tagData; 
+  
+  SParsedDataColInfo boundColumnInfo;
+
+  // for parameter ('?') binding
+  uint32_t       numOfAllocedParams;
+  uint32_t       numOfParams;
+  SParamInfo *   params;
+  SMemRowBuilder rowBuilder;
+} STableDataBlocks;
+
+static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
+  memRowSetType(row, memRowType);
+  if (isDataRowT(memRowType)) {
+    dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion);
+    dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen));
+  } else {
+    ASSERT(nBoundCols > 0);
+    memRowSetKvVersion(row, pBlock->pTableMeta->sversion);
+    kvRowSetNCols(memRowKvBody(row), nBoundCols);
+    kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
+  }
+}
+
+static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
+  ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize);
+  return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
+}
+
+// Applicable to consume by one row
+static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
+                                                 int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen,
+                                                 uint8_t compareStat) {
+  tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
+  if (compareStat == ROW_COMPARE_NEED) {
+    tdGetColAppendDeltaLen(value, colType, dataLen, kvLen);
+  }
+}
+
+static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd,
+                                                int32_t idx, int32_t *toffset) {
+  int32_t schemaIdx = 0;
+  if (IS_DATA_COL_ORDERED(spd)) {
+    schemaIdx = spd->boundedColumns[idx];
+    if (isDataRowT(memRowType)) {
+      *toffset = (spd->cols + schemaIdx)->toffset;  // the offset of firstPart
+    } else {
+      *toffset = idx * sizeof(SColIdx);  // the offset of SColIdx
+    }
+  } else {
+    ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
+    schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
+    if (isDataRowT(memRowType)) {
+      *toffset = (spd->cols + schemaIdx)->toffset;
+    } else {
+      *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);
+    }
+  }
+}
+
+static FORCE_INLINE void convertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
+  if (isDataRow(row)) {
+    if (kvLen < (dataLen * KVRatioConvert)) {
+      memRowSetConvert(row);
+    }
+  } else if (kvLen > dataLen) {
+    memRowSetConvert(row);
+  }
+}
+
+static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
+  pBlocks->tid = pTableMeta->suid;
+  pBlocks->uid = pTableMeta->uid;
+  pBlocks->sversion = pTableMeta->sversion;
+
+  if (pBlocks->numOfRows + numOfRows >= INT16_MAX) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  } else {
+    pBlocks->numOfRows += numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
+int32_t schemaIdxCompar(const void *lhs, const void *rhs);
+int32_t boundIdxCompar(const void *lhs, const void *rhs);
+void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols);
+void destroyBoundColumnInfo(SParsedDataColInfo* pColList);
+int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen);
+int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
+int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
+    SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList);
+int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap);
+
+#endif  // TDENGINE_DATABLOCKMGT_H

source/libs/parser/inc/insertParser.h

@@ -16,4 +16,16 @@
 #ifndef TDENGINE_INSERTPARSER_H
 #define TDENGINE_INSERTPARSER_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "parser.h"
+
+int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // TDENGINE_INSERTPARSER_H

source/libs/parser/inc/parserInt.h

@@ -26,14 +26,6 @@ extern "C" {
 
 struct SSqlNode;
 
-typedef struct SInsertStmtInfo {
-  SHashObj *pTableBlockHashList;     // data block for each table
-  SArray   *pDataBlocks;             // SArray<STableDataBlocks*>. Merged submit block for each vgroup
-  int8_t    schemaAttached;          // denote if submit block is built with table schema or not
-  uint8_t   payloadType;             // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t  insertType;              // insert data from [file|sql statement| bound statement]
-  char     *sql;                     // current sql statement position
-} SInsertStmtInfo;
 
 typedef struct SInternalField {
   TAOS_FIELD      field;

source/libs/parser/inc/parserUtil.h

@@ -46,7 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
 
 int32_t parserValidateIdToken(SToken* pToken);
 int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
-int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo,  const char* sourceStr);
+int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr);
 
 STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo);
 
@@ -61,6 +61,8 @@ void cleanupColumnCond(SArray** pCond);
 uint32_t convertRelationalOperator(SToken *pToken);
 int32_t    getExprFunctionId(SExprInfo *pExprInfo);
 
+STableMeta* tableMetaDup(const STableMeta* pTableMeta);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/parser/inc/sql.y

@@ -112,13 +112,13 @@ cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y).         {
 }
 
 cmd ::= SHOW dbPrefix(X) STABLES.      {
-    setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &X, 0);
+    setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &X, 0);
 }
 
 cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y).      {
     SToken token;
     tSetDbName(&token, &X);
-    setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &Y);
+    setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &Y);
 }
 
 cmd ::= SHOW dbPrefix(X) VGROUPS.    {

source/libs/parser/inc/ttoken.h

@@ -44,7 +44,7 @@ typedef struct SToken {
  * @param tokenType
  * @return
  */
-uint32_t tGetToken(char *z, uint32_t *tokenType);
+uint32_t tGetToken(const char *z, uint32_t *tokenType);
 
 /**
  * enhanced tokenizer for sql string.
@@ -54,7 +54,7 @@ uint32_t tGetToken(char *z, uint32_t *tokenType);
  * @param isPrevOptr
  * @return
  */
-SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr);
+SToken tStrGetToken(const char *str, int32_t *i, bool isPrevOptr);
 
 /**
  * check if it is a keyword or not

source/libs/parser/src/astGenerator.c

@@ -191,7 +191,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
   pExpr->type = SQL_NODE_EXPR;
 
   if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) {
-    char* endPos   = pRight->exprToken.z + pRight->exprToken.n;
+    const char* endPos   = pRight->exprToken.z + pRight->exprToken.n;
     pExpr->exprToken.z = pLeft->exprToken.z;
     pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z);
     pExpr->exprToken.type = pLeft->exprToken.type;
@@ -634,7 +634,7 @@ SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames,
   info.name       = *pToken;
   info.pTagNames  = pTagNames;
   info.pTagVals   = pTagVals;
-  info.stableName = *pTableName;
+  info.stbName    = *pTableName;
   info.igExist    = (igExists->n > 0)? 1:0;
 
   return info;

source/libs/parser/src/dataBlockMgt.c

@@ -0,0 +1,665 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dataBlockMgt.h"
+
+#include "catalog.h"
+#include "parserUtil.h"
+#include "queryInfoUtil.h"
+#include "taosmsg.h"
+
+#define IS_RAW_PAYLOAD(t) \
+  (((int)(t)) == PAYLOAD_TYPE_RAW)  // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+
+typedef struct SBlockKeyTuple {
+  TSKEY skey;
+  void* payloadAddr;
+} SBlockKeyTuple;
+
+typedef struct SBlockKeyInfo {
+  int32_t         maxBytesAlloc;
+  SBlockKeyTuple* pKeyTuple;
+} SBlockKeyInfo;
+
+static int32_t rowDataCompar(const void *lhs, const void *rhs) {
+  TSKEY left = *(TSKEY *)lhs;
+  TSKEY right = *(TSKEY *)rhs;
+
+  if (left == right) {
+    return 0;
+  } else {
+    return left > right ? 1 : -1;
+  }
+}
+
+void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) {
+  pColList->numOfCols = numOfCols;
+  pColList->numOfBound = numOfCols;
+  pColList->orderStatus = ORDER_STATUS_ORDERED;  // default is ORDERED for non-bound mode
+  pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t));
+  pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn));
+  pColList->colIdxInfo = NULL;
+  pColList->flen = 0;
+  pColList->allNullLen = 0;
+
+  int32_t nVar = 0;
+  for (int32_t i = 0; i < pColList->numOfCols; ++i) {
+    uint8_t type = pSchema[i].type;
+    if (i > 0) {
+      pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes;
+      pColList->cols[i].toffset = pColList->flen;
+    }
+    pColList->flen += TYPE_BYTES[type];
+    switch (type) {
+      case TSDB_DATA_TYPE_BINARY:
+        pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
+        ++nVar;
+        break;
+      case TSDB_DATA_TYPE_NCHAR:
+        pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+        ++nVar;
+        break;
+      default:
+        break;
+    }
+    pColList->boundedColumns[i] = pSchema[i].colId;
+  }
+  pColList->allNullLen += pColList->flen;
+  pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT));
+}
+
+int32_t schemaIdxCompar(const void *lhs, const void *rhs) {
+  uint16_t left = *(uint16_t *)lhs;
+  uint16_t right = *(uint16_t *)rhs;
+
+  if (left == right) {
+    return 0;
+  } else {
+    return left > right ? 1 : -1;
+  }
+}
+
+int32_t boundIdxCompar(const void *lhs, const void *rhs) {
+  uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t));
+  uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t));
+
+  if (left == right) {
+    return 0;
+  } else {
+    return left > right ? 1 : -1;
+  }
+}
+
+void destroyBoundColumnInfo(SParsedDataColInfo* pColList) {
+  tfree(pColList->boundedColumns);
+  tfree(pColList->cols);
+  tfree(pColList->colIdxInfo);
+}
+
+static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name,
+                           const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) {
+  STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks));
+  if (dataBuf == NULL) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  dataBuf->nAllocSize = (uint32_t)defaultSize;
+  dataBuf->headerSize = startOffset;
+
+  // the header size will always be the startOffset value, reserved for the subumit block header
+  if (dataBuf->nAllocSize <= dataBuf->headerSize) {
+    dataBuf->nAllocSize = dataBuf->headerSize * 2;
+  }
+
+  //dataBuf->pData = calloc(1, dataBuf->nAllocSize);
+  dataBuf->pData = malloc(dataBuf->nAllocSize);
+  if (dataBuf->pData == NULL) {
+    tfree(dataBuf);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  memset(dataBuf->pData, 0, sizeof(SSubmitBlk));
+
+  //Here we keep the tableMeta to avoid it to be remove by other threads.
+  dataBuf->pTableMeta = tableMetaDup(pTableMeta);
+
+  SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo;
+  SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta);
+  setBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns);
+
+  dataBuf->ordered  = true;
+  dataBuf->prevTS   = INT64_MIN;
+  dataBuf->rowSize  = rowSize;
+  dataBuf->size     = startOffset;
+  dataBuf->tsSource = -1;
+  dataBuf->vgId     = dataBuf->pTableMeta->vgId;
+
+  tNameAssign(&dataBuf->tableName, name);
+
+  assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL);
+
+  *dataBlocks = dataBuf;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
+                                SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks,
+                                SArray* pBlockList) {
+  *dataBlocks = NULL;
+  STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id));
+  if (t1 != NULL) {
+    *dataBlocks = *t1;
+  }
+
+  if (*dataBlocks == NULL) {
+    int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+
+    taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES);
+    if (pBlockList) {
+      taosArrayPush(pBlockList, dataBlocks);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t getRowExpandSize(STableMeta* pTableMeta) {
+  int32_t  result = TD_MEM_ROW_DATA_HEAD_SIZE;
+  int32_t  columns = getNumOfColumns(pTableMeta);
+  SSchema* pSchema = getTableColumnSchema(pTableMeta);
+  for (int32_t i = 0; i < columns; i++) {
+    if (IS_VAR_DATA_TYPE((pSchema + i)->type)) {
+      result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
+    }
+  }
+  return result;
+}
+
+/**
+ * TODO: Move to tdataformat.h and refactor when STSchema available.
+ *    - fetch flen and toffset from STSChema and remove param spd
+ */
+static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, SParsedDataColInfo *spd) {
+  ASSERT(isKvRow(src));
+  SKVRow   kvRow = memRowKvBody(src);
+  SDataRow dataRow = memRowDataBody(dest);
+
+  memRowSetType(dest, SMEM_ROW_DATA);
+  dataRowSetVersion(dataRow, memRowKvVersion(src));
+  dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen));
+
+  int32_t kvIdx = 0;
+  for (int i = 0; i < nCols; ++i) {
+    SSchema *schema = pSchema + i;
+    void *   val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx);
+    tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type,
+                       (spd->cols + i)->toffset);
+  }
+}
+
+// TODO: Move to tdataformat.h and refactor when STSchema available.
+static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, SParsedDataColInfo *spd) {
+  ASSERT(isDataRow(src));
+
+  SDataRow dataRow = memRowDataBody(src);
+  SKVRow   kvRow = memRowKvBody(dest);
+
+  memRowSetType(dest, SMEM_ROW_KV);
+  memRowSetKvVersion(kvRow, dataRowVersion(dataRow));
+  kvRowSetNCols(kvRow, nBoundCols);
+  kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
+
+  int32_t toffset = 0, kvOffset = 0;
+  for (int i = 0; i < nCols; ++i) {
+    if ((spd->cols + i)->valStat == VAL_STAT_HAS) {
+      SSchema *schema = pSchema + i;
+      toffset = (spd->cols + i)->toffset;
+      void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE);
+      tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset);
+      kvOffset += sizeof(SColIdx);
+    }
+  }
+}
+
+// TODO: Move to tdataformat.h and refactor when STSchema available.
+static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) {
+  STableMeta *        pTableMeta = pBlock->pTableMeta;
+  STableComInfo       tinfo = getTableInfo(pTableMeta);
+  SSchema *           pSchema = getTableColumnSchema(pTableMeta);
+  SParsedDataColInfo *spd = &pBlock->boundColumnInfo;
+
+  ASSERT(dest != src);
+
+  if (isDataRow(src)) {
+    // TODO: Can we use pBlock -> numOfParam directly?
+    ASSERT(spd->numOfBound > 0);
+    convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd);
+  } else {
+    convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd);
+  }
+}
+
+void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) {
+  if (pDataBlock == NULL) {
+    return;
+  }
+
+  tfree(pDataBlock->pData);
+
+  if (removeMeta) {
+    char name[TSDB_TABLE_FNAME_LEN] = {0};
+    tNameExtractFullName(&pDataBlock->tableName, name);
+
+    // taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
+  }
+
+  if (!pDataBlock->cloned) {
+    tfree(pDataBlock->params);
+
+    // free the refcount for metermeta
+    if (pDataBlock->pTableMeta != NULL) {
+      tfree(pDataBlock->pTableMeta);
+    }
+
+    destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
+  }
+
+  tfree(pDataBlock);
+}
+
+void* destroyBlockArrayList(SArray* pDataBlockList) {
+  if (pDataBlockList == NULL) {
+    return NULL;
+  }
+
+  size_t size = taosArrayGetSize(pDataBlockList);
+  for (int32_t i = 0; i < size; i++) {
+    void* d = taosArrayGetP(pDataBlockList, i);
+    destroyDataBlock(d, false);
+  }
+
+  taosArrayDestroy(pDataBlockList);
+  return NULL;
+}
+
+// data block is disordered, sort it in ascending order
+void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) {
+  SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData;
+
+  // size is less than the total size, since duplicated rows may be removed yet.
+  assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size);
+
+  if (!dataBuf->ordered) {
+    char *pBlockData = pBlocks->data;
+    qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar);
+
+    int32_t i = 0;
+    int32_t j = 1;
+
+    while (j < pBlocks->numOfRows) {
+      TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i);
+      TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j);
+
+      if (ti == tj) {
+        ++j;
+        continue;
+      }
+
+      int32_t nextPos = (++i);
+      if (nextPos != j) {
+        memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize);
+      }
+
+      ++j;
+    }
+
+    dataBuf->ordered = true;
+
+    pBlocks->numOfRows = i + 1;
+    dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows;
+  }
+
+  dataBuf->prevTS = INT64_MIN;
+}
+
+// data block is disordered, sort it in ascending order
+int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) {
+  SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData;
+  int16_t     nRows = pBlocks->numOfRows;
+
+  // size is less than the total size, since duplicated rows may be removed yet.
+
+  // allocate memory
+  size_t nAlloc = nRows * sizeof(SBlockKeyTuple);
+  if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) {
+    char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc);
+    if (tmp == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp;
+    pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc;
+  }
+  memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
+
+  int32_t         extendedRowSize = getExtendedRowSize(dataBuf);
+  SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+  char *          pBlockData = pBlocks->data;
+  int             n = 0;
+  while (n < nRows) {
+    pBlkKeyTuple->skey = memRowKey(pBlockData);
+    pBlkKeyTuple->payloadAddr = pBlockData;
+
+    // next loop
+    pBlockData += extendedRowSize;
+    ++pBlkKeyTuple;
+    ++n;
+  }
+
+  if (!dataBuf->ordered) {
+    pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+    qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar);
+
+    pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+    int32_t i = 0;
+    int32_t j = 1;
+    while (j < nRows) {
+      TSKEY ti = (pBlkKeyTuple + i)->skey;
+      TSKEY tj = (pBlkKeyTuple + j)->skey;
+
+      if (ti == tj) {
+        ++j;
+        continue;
+      }
+
+      int32_t nextPos = (++i);
+      if (nextPos != j) {
+        memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple));
+      }
+      ++j;
+    }
+
+    dataBuf->ordered = true;
+    pBlocks->numOfRows = i + 1;
+  }
+
+  dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize;
+  dataBuf->prevTS = INT64_MIN;
+
+  return 0;
+}
+
+// Erase the empty space reserved for binary data
+static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) {
+  // TODO: optimize this function, handle the case while binary is not presented
+  STableMeta*     pTableMeta = pTableDataBlock->pTableMeta;
+  STableComInfo   tinfo = getTableInfo(pTableMeta);
+  SSchema*        pSchema = getTableColumnSchema(pTableMeta);
+
+  SSubmitBlk* pBlock = pDataBlock;
+  memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
+  pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk);
+
+  int32_t flen = 0;  // original total length of row
+
+  // schema needs to be included into the submit data block
+  if (schemaAttached) {
+    int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta);
+    for(int32_t j = 0; j < numOfCols; ++j) {
+      STColumn* pCol = (STColumn*) pDataBlock;
+      pCol->colId = htons(pSchema[j].colId);
+      pCol->type  = pSchema[j].type;
+      pCol->bytes = htons(pSchema[j].bytes);
+      pCol->offset = 0;
+
+      pDataBlock = (char*)pDataBlock + sizeof(STColumn);
+      flen += TYPE_BYTES[pSchema[j].type];
+    }
+
+    int32_t schemaSize = sizeof(STColumn) * numOfCols;
+    pBlock->schemaLen = schemaSize;
+  } else {
+    if (isRawPayload) {
+      for (int32_t j = 0; j < tinfo.numOfColumns; ++j) {
+        flen += TYPE_BYTES[pSchema[j].type];
+      }
+    }
+    pBlock->schemaLen = 0;
+  }
+
+  char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
+  pBlock->dataLen = 0;
+  int32_t numOfRows = htons(pBlock->numOfRows);
+
+  if (isRawPayload) {
+    for (int32_t i = 0; i < numOfRows; ++i) {
+      SMemRow memRow = (SMemRow)pDataBlock;
+      memRowSetType(memRow, SMEM_ROW_DATA);
+      SDataRow trow = memRowDataBody(memRow);
+      dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
+      dataRowSetVersion(trow, pTableMeta->sversion);
+
+      int toffset = 0;
+      for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
+        tdAppendColVal(trow, p, pSchema[j].type, toffset);
+        toffset += TYPE_BYTES[pSchema[j].type];
+        p += pSchema[j].bytes;
+      }
+
+      pDataBlock = (char*)pDataBlock + memRowTLen(memRow);
+      pBlock->dataLen += memRowTLen(memRow);
+    }
+  } else {
+    for (int32_t i = 0; i < numOfRows; ++i) {
+      char* payload = (blkKeyTuple + i)->payloadAddr;
+      if (isNeedConvertRow(payload)) {
+        convertSMemRow(pDataBlock, payload, pTableDataBlock);
+        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
+        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+        pBlock->dataLen += rowTLen;
+      } else {
+        TDRowTLenT rowTLen = memRowTLen(payload);
+        memcpy(pDataBlock, payload, rowTLen);
+        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+        pBlock->dataLen += rowTLen;
+      }
+    }
+  }
+
+  int32_t len = pBlock->dataLen + pBlock->schemaLen;
+  pBlock->dataLen = htonl(pBlock->dataLen);
+  pBlock->schemaLen = htonl(pBlock->schemaLen);
+
+  return len;
+}
+
+static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) {
+  // todo
+}
+
+int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) {
+  const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
+  int       code = 0;
+  bool      isRawPayload = IS_RAW_PAYLOAD(payloadType);
+  SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  SArray*   pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
+
+  STableDataBlocks** p = taosHashIterate(pHashObj, NULL);
+  STableDataBlocks* pOneTableBlock = *p;
+  SBlockKeyInfo blkKeyInfo = {0};  // share by pOneTableBlock
+  while (pOneTableBlock) {
+    SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
+    if (pBlocks->numOfRows > 0) {
+      STableDataBlocks* dataBuf = NULL;
+      int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE,
+                                  INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList);
+      if (ret != TSDB_CODE_SUCCESS) {
+        taosHashCleanup(pVnodeDataBlockHashList);
+        destroyBlockArrayList(pVnodeDataBlockList);
+        tfree(blkKeyInfo.pKeyTuple);
+        return ret;
+      }
+
+      // the maximum expanded size in byte when a row-wise data is converted to SDataRow format
+      int32_t           expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
+      int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
+                         sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta);
+
+      if (dataBuf->nAllocSize < destSize) {
+        dataBuf->nAllocSize = (uint32_t)(destSize * 1.5);
+        char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize);
+        if (tmp != NULL) {
+          dataBuf->pData = tmp;
+        } else {  // failed to allocate memory, free already allocated memory and return error code
+          taosHashCleanup(pVnodeDataBlockHashList);
+          destroyBlockArrayList(pVnodeDataBlockList);
+          tfree(dataBuf->pData);
+          tfree(blkKeyInfo.pKeyTuple);
+          return TSDB_CODE_TSC_OUT_OF_MEMORY;
+        }
+      }
+
+      if (isRawPayload) {
+        sortRemoveDataBlockDupRowsRaw(pOneTableBlock);
+      } else {
+        if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) {
+          taosHashCleanup(pVnodeDataBlockHashList);
+          destroyBlockArrayList(pVnodeDataBlockList);
+          tfree(dataBuf->pData);
+          tfree(blkKeyInfo.pKeyTuple);
+          return code;
+        }
+        ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0);
+      }
+
+      int32_t len = pBlocks->numOfRows *
+                        (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) +
+                    sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta);
+
+      pBlocks->tid = htonl(pBlocks->tid);
+      pBlocks->uid = htobe64(pBlocks->uid);
+      pBlocks->sversion = htonl(pBlocks->sversion);
+      pBlocks->numOfRows = htons(pBlocks->numOfRows);
+      pBlocks->schemaLen = 0;
+
+      // erase the empty space reserved for binary data
+      int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload);
+      assert(finalLen <= len);
+
+      dataBuf->size += (finalLen + sizeof(SSubmitBlk));
+      assert(dataBuf->size <= dataBuf->nAllocSize);
+
+      // the length does not include the SSubmitBlk structure
+      pBlocks->dataLen = htonl(finalLen);
+      dataBuf->numOfTables += 1;
+
+      pBlocks->numOfRows = 0;
+    }
+
+    p = taosHashIterate(pHashObj, p);
+    if (p == NULL) {
+      break;
+    }
+
+    pOneTableBlock = *p;
+  }
+
+  extractTableNameList(pHashObj, freeBlockMap);
+
+  // free the table data blocks;
+  // pInsertParam->pDataBlocks = pVnodeDataBlockList;
+  taosHashCleanup(pVnodeDataBlockHashList);
+  tfree(blkKeyInfo.pKeyTuple);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) {
+  size_t    remain = pDataBlock->nAllocSize - pDataBlock->size;
+  const int factor = 5;
+  uint32_t nAllocSizeOld = pDataBlock->nAllocSize;
+  
+  // expand the allocated size
+  if (remain < rowSize * factor) {
+    while (remain < rowSize * factor) {
+      pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5);
+      remain = pDataBlock->nAllocSize - pDataBlock->size;
+    }
+
+    char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize);
+    if (tmp != NULL) {
+      pDataBlock->pData = tmp;
+      memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size);
+    } else {
+      // do nothing, if allocate more memory failed
+      pDataBlock->nAllocSize = nAllocSizeOld;
+      *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize;
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+  }
+
+  *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen) {
+  ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols));
+  if (nRows > 0) {
+    // already init(bind multiple rows by single column)
+    if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // default compareStat is  ROW_COMPARE_NO_NEED
+  if (nBoundCols == 0) {  // file input
+    pBuilder->memRowType = SMEM_ROW_DATA;
+    return TSDB_CODE_SUCCESS;
+  } else {
+    float boundRatio = ((float)nBoundCols / (float)nCols);
+
+    if (boundRatio < KVRatioKV) {
+      pBuilder->memRowType = SMEM_ROW_KV;
+      return TSDB_CODE_SUCCESS;
+    } else if (boundRatio > KVRatioData) {
+      pBuilder->memRowType = SMEM_ROW_DATA;
+      return TSDB_CODE_SUCCESS;
+    }
+    pBuilder->compareStat = ROW_COMPARE_NEED;
+
+    if (boundRatio < KVRatioPredict) {
+      pBuilder->memRowType = SMEM_ROW_KV;
+    } else {
+      pBuilder->memRowType = SMEM_ROW_DATA;
+    }
+  }
+
+  pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
+
+  if (nRows > 0) {
+    pBuilder->rowInfo = calloc(nRows, sizeof(SMemRowInfo));
+    if (pBuilder->rowInfo == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+
+    for (int i = 0; i < nRows; ++i) {
+      (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
+      (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}

source/libs/parser/src/insertParser.c

@@ -0,0 +1,885 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "insertParser.h"
+
+#include "dataBlockMgt.h"
+#include "parserInt.h"
+#include "parserUtil.h"
+#include "queryInfoUtil.h"
+#include "tglobal.h"
+#include "ttime.h"
+#include "ttoken.h"
+#include "ttypes.h"
+
+#define NEXT_TOKEN(pSql, sToken) \
+  do { \
+    int32_t index = 0; \
+    sToken = tStrGetToken(pSql, &index, false); \
+    pSql += index; \
+  } while (0)
+
+#define CHECK_CODE(expr) \
+  do { \
+    int32_t code = expr; \
+    if (TSDB_CODE_SUCCESS != code) { \
+      terrno = code; \
+      return terrno; \
+    } \
+  } while (0)
+
+#define CHECK_CODE_1(expr, destroy) \
+  do { \
+    int32_t code = expr; \
+    if (TSDB_CODE_SUCCESS != code) { \
+      (void)destroy; \
+      terrno = code; \
+      return terrno; \
+    } \
+  } while (0)
+
+#define CHECK_CODE_2(expr, destroy1, destroy2) \
+  do { \
+    int32_t code = expr; \
+    if (TSDB_CODE_SUCCESS != code) { \
+      (void)destroy1; \
+      (void)destroy2; \
+      terrno = code; \
+      return terrno; \
+    } \
+  } while (0)
+
+enum {
+  TSDB_USE_SERVER_TS = 0,
+  TSDB_USE_CLI_TS = 1,
+};
+
+typedef struct SInsertParseContext {
+  SParseContext* pComCxt;
+  const char* pSql;
+  SMsgBuf msg;
+  struct SCatalog* pCatalog;
+  SMetaData meta;               // need release
+  const STableMeta* pTableMeta;
+  SHashObj* pTableBlockHashObj; // data block for each table. need release
+  int32_t totalNum;
+  SInsertStmtInfo* pOutput;
+} SInsertParseContext;
+
+static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
+static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
+
+static bool isNullStr(SToken *pToken) {
+  return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) &&
+                                       (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
+}
+
+static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) {
+  errno = 0;
+  *value = strtold(pToken->z, endPtr);
+
+  // not a valid integer number, return error
+  if ((*endPtr - pToken->z) != pToken->n) {
+    return TK_ILLEGAL;
+  }
+
+  return pToken->type;
+}
+
+static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) {
+  errno = 0;
+  int32_t ret = 0;
+
+  char* endPtr = NULL;
+  if (type == TK_FLOAT) {
+    double v = strtod(z, &endPtr);
+    if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) {
+      ret = -1;
+    } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) {
+      ret = -1;
+    } else {
+      *value = (int64_t) round(v);
+    }
+
+    errno = 0;
+    return ret;
+  }
+
+  int32_t radix = 10;
+  if (type == TK_HEX) {
+    radix = 16;
+  } else if (type == TK_BIN) {
+    radix = 2;
+  }
+
+  // the string may be overflow according to errno
+  if (!issigned) {
+    const char *p = z;
+    while(*p != 0 && *p == ' ') p++;   
+    if (*p != 0 && *p == '-') { return -1;}
+
+    *value = strtoull(z, &endPtr, radix);
+  } else {
+    *value = strtoll(z, &endPtr, radix);
+  }
+
+  // not a valid integer number, return error
+  if (endPtr - z != n || errno == ERANGE) {
+    ret = -1;
+  }
+
+  errno = 0;
+  return ret;
+}
+
+static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) {
+  *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo));
+  if (NULL == *pInsertInfo) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t skipInsertInto(SInsertParseContext* pCxt) {
+  SToken sToken;
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  if (TK_INSERT != sToken.type) {
+    return buildSyntaxErrMsg(&pCxt->msg, "keyword INSERT is expected", sToken.z);
+  }
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  if (TK_INTO != sToken.type) {
+    return buildSyntaxErrMsg(&pCxt->msg, "keyword INTO is expected", sToken.z);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) {
+  if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) {
+    return buildInvalidOperationMsg(&pCxt->msg, "invalid table name");
+  }
+
+  SName name = {0};
+  strndequote(name.tname, pStname->z, pStname->n);
+  taosArrayPush(tableNameList, &name);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SMetaReq* pMetaReq) {
+  pMetaReq->pTableName = taosArrayInit(4, sizeof(SName));
+  return buildTableName(pCxt, pStname, pMetaReq->pTableName);
+}
+
+static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
+  SMetaReq req;
+  CHECK_CODE(buildMetaReq(pCxt, pTname, &req));
+  CHECK_CODE(catalogGetMetaData(pCxt->pCatalog, &req, &pCxt->meta));
+  pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0);
+  return TSDB_CODE_SUCCESS;
+}
+
+// todo speedup by using hash list
+static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) {
+  while (start < end) {
+    if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) {
+      return start;
+    }
+    ++start;
+  }
+  return -1;
+}
+
+static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) {
+  // once the data block is disordered, we do NOT keep previous timestamp any more
+  if (!pDataBlocks->ordered) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  TSKEY k = *(TSKEY *)start;
+
+  if (k == INT64_MIN) {
+    if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) {
+      return -1;
+    } else if (pDataBlocks->tsSource == -1) {
+      pDataBlocks->tsSource = TSDB_USE_SERVER_TS;
+    }
+  } else {
+    if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) {
+      return -1;  // client time/server time can not be mixed
+    } else if (pDataBlocks->tsSource == -1) {
+      pDataBlocks->tsSource = TSDB_USE_CLI_TS;
+    }
+  }
+
+  if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) {
+    pDataBlocks->ordered = false;
+  }
+
+  pDataBlocks->prevTS = k;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec, int64_t *time) {
+  int32_t   index = 0;
+  SToken sToken;
+  int64_t   interval;
+  int64_t   useconds = 0;
+  const char* pTokenEnd = pCxt->pSql;
+
+  if (pToken->type == TK_NOW) {
+    useconds = taosGetTimestamp(timePrec);
+  } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) {
+    // do nothing
+  } else if (pToken->type == TK_INTEGER) {
+    useconds = taosStr2int64(pToken->z);
+  } else {
+    // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
+    if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
+      return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp format", pToken->z);
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  for (int k = pToken->n; pToken->z[k] != '\0'; k++) {
+    if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue;
+    if (pToken->z[k] == ',') {
+      pCxt->pSql = pTokenEnd;
+      *time = useconds;
+      return 0;
+    }
+
+    break;
+  }
+
+  /*
+   * time expression:
+   * e.g., now+12a, now-5h
+   */
+  SToken valueToken;
+  index = 0;
+  sToken = tStrGetToken(pTokenEnd, &index, false);
+  pTokenEnd += index;
+
+  if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) {
+    index = 0;
+    valueToken = tStrGetToken(pTokenEnd, &index, false);
+    pTokenEnd += index;
+
+    if (valueToken.n < 2) {
+      return buildSyntaxErrMsg(&pCxt->msg, "value expected in timestamp", sToken.z);
+    }
+
+    char unit = 0;
+    if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+
+    if (sToken.type == TK_PLUS) {
+      useconds += interval;
+    } else {
+      useconds = useconds - interval;
+    }
+
+    pCxt->pSql = pTokenEnd;
+  }
+
+  *time = useconds;
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param);
+
+typedef struct SKvParam {
+  char buf[TSDB_MAX_TAGS_LEN];
+  SKVRowBuilder* builder;
+  SSchema* schema;
+} SKvParam;
+
+static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) {
+  SKvParam* pa = (SKvParam*)param;
+  if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
+    STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
+    tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf);
+  } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
+    // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
+    int32_t output = 0;
+    if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
+      return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
+    }
+    varDataSetLen(pa->buf, output);
+    tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf);
+  } else {
+    tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, value);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef struct SMemParam {
+  SMemRow row;
+  SSchema* schema;
+  int32_t toffset;
+  uint8_t compareStat;
+  int32_t dataLen;
+  int32_t kvLen;
+} SMemParam;
+
+static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *param) {
+  SMemParam* pa = (SMemParam*)param;
+  if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
+    char *rowEnd = memRowEnd(pa->row);
+    STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len);
+    appendMemRowColValEx(pa->row, rowEnd, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+  } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
+    // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
+    int32_t output = 0;
+    char *  rowEnd = memRowEnd(pa->row);
+    if (!taosMbsToUcs4(value, len, (char *)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
+      return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
+    }
+    varDataSetLen(rowEnd, output);
+    appendMemRowColValEx(pa->row, rowEnd, false, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+  } else {
+    appendMemRowColValEx(pa->row, value, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE int32_t checkAndTrimValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, char* tmpTokenBuf) {
+  int16_t type = pToken->type;
+  if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
+        type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) ||
+      (pToken->n == 0) || (type == TK_RP)) {
+    return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pToken->z);
+  }
+
+  if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) {
+    return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z);
+  }
+
+  // Remove quotation marks
+  if (TK_STRING == type) {
+    if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
+      return buildSyntaxErrMsg(&pCxt->msg, "too long string", pToken->z);
+    }
+    // delete escape character: \\, \', \"
+    char delim = pToken->z[0];
+    int32_t cnt = 0;
+    int32_t j = 0;
+    for (uint32_t k = 1; k < pToken->n - 1; ++k) {
+      if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
+        tmpTokenBuf[j] = pToken->z[k + 1];
+        cnt++;
+        j++;
+        k++;
+        continue;
+      }
+      tmpTokenBuf[j] = pToken->z[k];
+      j++;
+    }
+    tmpTokenBuf[j] = 0;
+    pToken->z = tmpTokenBuf;
+    pToken->n -= 2 + cnt;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, FRowAppend func, void* param) {
+  int64_t iv;
+  int32_t ret;
+  char *  endptr = NULL;
+
+  CHECK_CODE(checkAndTrimValue(pCxt, pToken, pSchema, tmpTokenBuf));
+
+  if (isNullStr(pToken)) {
+    if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
+      int64_t tmpVal = 0;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    return func(getNullValue(pSchema->type), 0, param);
+  }
+
+  switch (pSchema->type) {
+    case TSDB_DATA_TYPE_BOOL: {
+      if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
+        if (strncmp(pToken->z, "true", pToken->n) == 0) {
+          return func(&TRUE_VALUE, pSchema->bytes, param);
+        } else if (strncmp(pToken->z, "false", pToken->n) == 0) {
+          return func(&FALSE_VALUE, pSchema->bytes, param);
+        } else {
+          return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z);
+        }
+      } else if (pToken->type == TK_INTEGER) {
+        return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
+      } else if (pToken->type == TK_FLOAT) {
+        return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param);
+      } else {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z);
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z);
+      } else if (!IS_VALID_TINYINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z);
+      }
+      uint8_t tmpVal = (uint8_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_UTINYINT:{
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z);
+      } else if (!IS_VALID_UTINYINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z);
+      }
+      uint8_t tmpVal = (uint8_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z);
+      } else if (!IS_VALID_SMALLINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z);
+      }
+      int16_t tmpVal = (int16_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z);
+      } else if (!IS_VALID_USMALLINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z);
+      }
+      uint16_t tmpVal = (uint16_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_INT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z);
+      } else if (!IS_VALID_INT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z);
+      }
+      int32_t tmpVal = (int32_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z);
+      } else if (!IS_VALID_UINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z);
+      }
+      uint32_t tmpVal = (uint32_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z);
+      } else if (!IS_VALID_BIGINT(iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z);
+      }
+      return func(&iv, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z);
+      } else if (!IS_VALID_UBIGINT((uint64_t)iv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z);
+      }
+      uint64_t tmpVal = (uint64_t)iv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      double dv;
+      if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
+      }
+      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
+      }
+      float tmpVal = (float)dv;
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double dv;
+      if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
+      }
+      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
+      }
+      return func(&dv, pSchema->bytes, param);
+    }
+    case TSDB_DATA_TYPE_BINARY: {
+      // too long values will return invalid sql, not be truncated automatically
+      if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {  // todo refactor
+        return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z);
+      }
+      return func(pToken->z, pToken->n, param);
+    }
+    case TSDB_DATA_TYPE_NCHAR: {
+      return func(pToken->z, pToken->n, param);
+    }
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      int64_t tmpVal;
+      if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) {
+        return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z);
+      }
+      return func(&tmpVal, pSchema->bytes, param);
+    }
+  }
+
+  return TSDB_CODE_FAILED;
+}
+
+// pSql -> tag1_name, ...)
+static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) {
+  int32_t nCols = pColList->numOfCols;
+
+  pColList->numOfBound = 0; 
+  memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols);
+  for (int32_t i = 0; i < nCols; ++i) {
+    pColList->cols[i].valStat = VAL_STAT_NONE;
+  }
+
+  SToken sToken;
+  bool    isOrdered = true;
+  int32_t lastColIdx = -1;  // last column found
+  while (1) {
+    NEXT_TOKEN(pCxt->pSql, sToken);
+
+    if (TK_RP == sToken.type) {
+      break;
+    }
+
+    int32_t t = lastColIdx + 1;
+    int32_t index = findCol(&sToken, t, nCols, pSchema);
+    if (index < 0 && t > 0) {
+      index = findCol(&sToken, 0, t, pSchema);
+      isOrdered = false;
+    }
+    if (index < 0) {
+      return buildSyntaxErrMsg(&pCxt->msg, "invalid column/tag name", sToken.z);
+    }
+    if (pColList->cols[index].valStat == VAL_STAT_HAS) {
+      return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z);
+    }
+    lastColIdx = index;
+    pColList->cols[index].valStat = VAL_STAT_HAS;
+    pColList->boundedColumns[pColList->numOfBound] = index;
+    ++pColList->numOfBound;
+  }
+
+  pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED;
+
+  if (!isOrdered) {
+    pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo));
+    if (NULL == pColList->colIdxInfo) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    SBoundIdxInfo* pColIdx = pColList->colIdxInfo;
+    for (uint16_t i = 0; i < pColList->numOfBound; ++i) {
+      pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i];
+      pColIdx[i].boundIdx = i;
+    }
+    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
+    for (uint16_t i = 0; i < pColList->numOfBound; ++i) {
+      pColIdx[i].finalIdx = i;
+    }
+    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
+  }
+
+  memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+// pSql -> tag1_value, ...)
+static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) {
+  SKVRowBuilder kvRowBuilder = {0};
+  if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
+    destroyBoundColumnInfo(pSpd);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  SKvParam param = {.builder = &kvRowBuilder};
+  SToken sToken;
+  char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"
+  for (int i = 0; i < pSpd->numOfBound; ++i) {
+    NEXT_TOKEN(pCxt->pSql, sToken);
+    SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]];
+    param.schema = pSchema;
+    CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, &param), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd));
+  }
+
+  destroyBoundColumnInfo(pSpd);
+  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+  tdDestroyKVRowBuilder(&kvRowBuilder);
+  if (NULL == row) {
+    return buildInvalidOperationMsg(&pCxt->msg, "tag value expected");
+  }
+  tdSortKVRowByColIdx(row);
+
+  // todo construct payload
+
+  tfree(row);
+}
+
+// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
+static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) {
+  SToken sToken;
+
+  // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  CHECK_CODE(getTableMeta(pCxt, &sToken));
+  if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
+    return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
+  }
+
+  SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta);
+  SParsedDataColInfo spd = {0};
+  setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta));
+
+  // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  if (TK_LP == sToken.type) {
+    CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd));
+    NEXT_TOKEN(pCxt->pSql, sToken);
+  }
+
+  if (TK_TAGS != sToken.type) {
+    return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z);
+  }
+  // pSql -> (tag1_value, ...)
+  NEXT_TOKEN(pCxt->pSql, sToken);
+  if (TK_LP != sToken.type) {
+    return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
+  }
+  CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) {  
+  SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo;
+  SMemRowBuilder* pBuilder = &pDataBlocks->rowBuilder;
+  char *row = pDataBlocks->pData + pDataBlocks->size;  // skip the SSubmitBlk header
+  initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound);
+
+  bool isParseBindParam = false;
+  SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta);
+  SMemParam param = {.row = row};
+  SToken sToken = {0};
+  // 1. set the parsed value from sql string
+  for (int i = 0; i < spd->numOfBound; ++i) {
+    NEXT_TOKEN(pCxt->pSql, sToken);
+    // todo bind param
+    SSchema *pSchema = &schema[spd->boundedColumns[i]]; 
+    param.schema = pSchema;
+    param.compareStat = pBuilder->compareStat;
+    getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &param.toffset);
+    CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, &param));
+
+    if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
+      TSKEY tsKey = memRowKey(row);
+      if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) {
+        buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z);
+        return TSDB_CODE_TSC_INVALID_TIME_STAMP;
+      }
+    }
+  }
+
+  if (!isParseBindParam) {
+    // 2. check and set convert flag
+    if (pBuilder->compareStat == ROW_COMPARE_NEED) {
+      convertMemRow(row, spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE, pBuilder->kvRowInitLen);
+    }
+
+    // 3. set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) {
+      SDataRow dataRow = memRowDataBody(row);
+      for (int32_t i = 0; i < spd->numOfCols; ++i) {
+        if (spd->cols[i].valStat == VAL_STAT_NONE) {
+          tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset);
+        }
+      }
+    }
+  }
+
+  *len = getExtendedRowSize(pDataBlocks);
+  return TSDB_CODE_SUCCESS;
+}
+
+// pSql -> (field1_value, ...) [(field1_value2, ...) ...]
+static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) {
+  STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta);
+  int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
+  CHECK_CODE(initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, pDataBlock->boundColumnInfo.allNullLen));
+
+  (*numOfRows) = 0;
+  char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"
+  SToken sToken;
+  while (1) {
+    NEXT_TOKEN(pCxt->pSql, sToken);
+    if (TK_LP != sToken.type) {
+      break;
+    }
+
+    if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) {
+      int32_t tSize;
+      CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize));
+      ASSERT(tSize >= maxRows);
+      maxRows = tSize;
+    }
+
+    int32_t len = 0;
+    CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf));
+    pDataBlock->size += len;
+
+    NEXT_TOKEN(pCxt->pSql, sToken);
+    if (TK_RP != sToken.type) {
+      return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z);
+    }
+
+    (*numOfRows)++;
+  }
+
+  if (0 == (*numOfRows)) {
+    return  buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) {  
+  int32_t maxNumOfRows;
+  CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows));
+
+  int32_t numOfRows = 0;
+  CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows));
+
+  for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) {
+    SParamInfo *param = dataBuf->params + i;
+    if (param->idx == -1) {
+      // param->idx = pInsertParam->numOfParams++;
+      param->offset -= sizeof(SSubmitBlk);
+    }
+  }
+
+  SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData);
+  if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) {
+    return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767");
+  }
+
+  dataBuf->numOfTables = 1;
+  pCxt->totalNum += numOfRows;
+  return TSDB_CODE_SUCCESS;
+}
+
+//   tb_name
+//       [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
+//       [(field1_name, ...)]
+//       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
+//   [...];
+static int32_t parseInsertBody(SInsertParseContext* pCxt) {
+  while (1) {
+    SToken sToken;
+    // pSql -> tb_name ...
+    NEXT_TOKEN(pCxt->pSql, sToken);
+
+    // no data in the sql string anymore.
+    if (sToken.n == 0) {
+      if (0 == pCxt->totalNum) {
+        return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");;
+      }
+      break;
+    }
+
+    SToken tbnameToken = sToken;
+    NEXT_TOKEN(pCxt->pSql, sToken);
+
+    // USING cluase 
+    if (TK_USING == sToken.type) {
+      CHECK_CODE(parseUsingClause(pCxt, &tbnameToken));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    } else {
+      CHECK_CODE(getTableMeta(pCxt, &sToken));
+    }
+
+    STableDataBlocks *dataBuf = NULL;
+    CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
+        sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, NULL/* tbname */, pCxt->pTableMeta, &dataBuf, NULL));
+
+    if (TK_LP == sToken.type) {
+      // pSql -> field1_name, ...)
+      CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo));
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    }
+
+    if (TK_VALUES == sToken.type) {
+      // pSql -> (field1_value, ...) [(field1_value2, ...) ...]
+      CHECK_CODE(parseValuesClause(pCxt, dataBuf));
+      continue;
+    }
+
+    // FILE csv_file_path
+    if (TK_FILE == sToken.type) {
+      // pSql -> csv_file_path
+      NEXT_TOKEN(pCxt->pSql, sToken);
+      if (0 == sToken.n || (TK_STRING != sToken.type && TK_ID != sToken.type)) {
+        return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z);
+      }
+      // todo
+      continue;
+    }
+
+    return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z);
+  }
+  // merge according to vgId
+  if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) {
+    CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true));
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+// INSERT INTO
+//   tb_name
+//       [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
+//       [(field1_name, ...)]
+//       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
+//   [...];
+int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
+  CHECK_CODE(createInsertStmtInfo(pInfo));
+
+  SInsertParseContext context = {
+    .pComCxt = pContext,
+    .pSql = pContext->pSql,
+    .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
+    .pCatalog = getCatalogHandle(pContext->pEpSet),
+    .pTableMeta = NULL,
+    .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false),
+    .totalNum = 0,
+    .pOutput = *pInfo
+  };
+
+  if (NULL == context.pTableBlockHashObj) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  CHECK_CODE(skipInsertInto(&context));
+  CHECK_CODE(parseInsertBody(&context));
+
+  return TSDB_CODE_SUCCESS;
+}

source/libs/parser/src/parser.c

@@ -18,6 +18,7 @@
 #include "parserUtil.h"
 #include "ttoken.h"
 #include "function.h"
+#include "insertParser.h"
 
 bool qIsInsertSql(const char* pStr, size_t length) {
   int32_t index = 0;
@@ -46,8 +47,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo**
   return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen);
 }
 
-int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen) {
+int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
-  return 0;
+  return parseInsertSql(pContext, pInfo);
 }
 
 int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) {
@@ -173,7 +174,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet
       assert(t != NULL);
 
       if (t->n >= TSDB_FUNC_NAME_LEN) {
-        return buildSyntaxErrMsg(msg, msgBufLen, "too long function name", t->z);
+        return buildSyntaxErrMsg(&msgBuf, "too long function name", t->z);
       }
 
       // Let's assume that it is an UDF/UDAF, if it is not a built-in function.

source/libs/parser/src/parserUtil.c

@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "parserUtil.h"
+
 #include "taosmsg.h"
 #include "parser.h"
 #include "taoserror.h"
@@ -18,7 +34,6 @@ typedef struct STableFilterCond {
 
 static STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
                                         SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
-STableMeta* tableMetaDup(STableMeta* pTableMeta);
 
 int32_t parserValidateIdToken(SToken* pToken) {
   if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
@@ -87,7 +102,7 @@ int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
   return TSDB_CODE_TSC_INVALID_OPERATION;
 }
 
-int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo,  const char* sourceStr) {
+int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr) {
   const char* msgFormat1 = "syntax error near \'%s\'";
   const char* msgFormat2 = "syntax error near \'%s\' (%s)";
   const char* msgFormat3 = "%s";
@@ -95,7 +110,7 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn
   const char* prefix = "syntax error";
   if (sourceStr == NULL) {
     assert(additionalInfo != NULL);
-    snprintf(dst, dstBufLen, msgFormat1, additionalInfo);
+    snprintf(pBuf->buf, pBuf->len, msgFormat1, additionalInfo);
     return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
   }
 
@@ -103,10 +118,10 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn
   strncpy(buf, sourceStr, tListLen(buf) - 1);
 
   if (additionalInfo != NULL) {
-    snprintf(dst, dstBufLen, msgFormat2, buf, additionalInfo);
+    snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo);
   } else {
     const char* msgFormat = (0 == strncmp(sourceStr, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1;
-    snprintf(dst, dstBufLen, msgFormat, buf);
+    snprintf(pBuf->buf, pBuf->len, msgFormat, buf);
   }
 
   return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
@@ -1490,7 +1505,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
   return pTableMeta;
 }
 
-uint32_t getTableMetaSize(STableMeta* pTableMeta) {
+uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
 
   int32_t totalCols = 0;
@@ -1505,7 +1520,7 @@ uint32_t getTableMetaMaxSize() {
   return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema);
 }
 
-STableMeta* tableMetaDup(STableMeta* pTableMeta) {
+STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
   size_t size = getTableMetaSize(pTableMeta);
 

source/libs/parser/src/sql.c

@@ -2312,14 +2312,14 @@ static void yy_reduce(
         break;
       case 26: /* cmd ::= SHOW dbPrefix STABLES */
 {
-    setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &yymsp[-1].minor.yy0, 0);
+    setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &yymsp[-1].minor.yy0, 0);
 }
         break;
       case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */
 {
     SToken token;
     tSetDbName(&token, &yymsp[-3].minor.yy0);
-    setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &yymsp[0].minor.yy0);
+    setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &yymsp[0].minor.yy0);
 }
         break;
       case 28: /* cmd ::= SHOW dbPrefix VGROUPS */

source/libs/parser/src/ttokenizer.c

@@ -284,7 +284,7 @@ static int32_t tKeywordCode(const char* z, int n) {
  * Return the length of the token that begins at z[0].
  * Store the token type in *type before returning.
  */
-uint32_t tGetToken(char* z, uint32_t* tokenId) {
+uint32_t tGetToken(const char* z, uint32_t* tokenId) {
   uint32_t i;
   switch (*z) {
     case ' ':
@@ -595,7 +595,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) {
   return ntoken;
 }
 
-SToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
+SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) {
   SToken t0 = {0};
 
   // here we reach the end of sql string, null-terminated string
@@ -689,13 +689,12 @@ void taosCleanupKeywordsTable() {
 }
 
 SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) {
-  assert(pToken != NULL && buf != NULL);
+  assert(pToken != NULL && buf != NULL && len > pToken->n);
+  
+  strncpy(buf, pToken->z, pToken->n);
+  buf[pToken->n] = 0;
+
   SToken token = *pToken;
   token.z = buf;
-
-  assert(len > token.n);
-  strncpy(token.z, pToken->z, pToken->n);
-  token.z[token.n] = 0;
-
   return token;
 }

source/libs/parser/test/insertTest.cpp

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include "insertParser.h"
+#include "mockCatalog.h"
+
+using namespace std;
+using namespace testing;
+
+namespace {
+  string toString(int32_t code) {
+    return tstrerror(code);
+  }
+}
+
+// syntax:
+// INSERT INTO
+//   tb_name
+//       [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
+//       [(field1_name, ...)]
+//       VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
+//   [...];
+class InsertTest : public Test {
+protected:
+  void bind(const char* sql) {
+    reset();
+    cxt.pSql = sql;
+    cxt.sqlLen = strlen(sql);
+  }
+
+  int32_t run() {
+    code = parseInsertSql(&cxt, &res);
+    if (code != TSDB_CODE_SUCCESS) {
+      cout << "code:" << toString(code) << ", msg:" << errMagBuf << endl;
+    }
+    return code;
+  }
+
+  SInsertStmtInfo* reslut() {
+    return res;
+  }
+
+private:
+  static const int max_err_len = 1024;
+
+  void reset() {
+    memset(&cxt, 0, sizeof(cxt));
+    memset(errMagBuf, 0, max_err_len);
+    cxt.pMsg = errMagBuf;
+    cxt.msgLen = max_err_len;
+    code = TSDB_CODE_SUCCESS;
+    res = nullptr;
+  }
+
+  char errMagBuf[max_err_len];
+  SParseContext cxt;
+  int32_t code;
+  SInsertStmtInfo* res;
+};
+
+// INSERT INTO tb_name VALUES (field1_value, ...)
+TEST_F(InsertTest, simpleTest) {
+  bind("insert into .. values (...)");
+  ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
+  SInsertStmtInfo* res = reslut();
+  // todo check
+}
+
+TEST_F(InsertTest, toleranceTest) {
+  bind("insert into");
+  ASSERT_NE(run(), TSDB_CODE_SUCCESS);
+  bind("insert into t");
+  ASSERT_NE(run(), TSDB_CODE_SUCCESS);
+}

source/libs/parser/test/mockCatalog.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mockCatalog.h"
+
+#include <iostream>
+
+void generateMetaData(MockCatalogService* mcs) {
+  {
+    ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, MockCatalogService::numOfDataTypes)
+        .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP);
+    for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) {
+      if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) {
+        continue;
+      }
+      builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type);
+    }
+    builder.done();
+  }
+  {
+    ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, MockCatalogService::numOfDataTypes, 2)
+        .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(2).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP);
+    for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) {
+      if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) {
+        continue;
+      }
+      builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type);
+    }
+    builder.done();
+  }
+  mcs->showTables();
+}
+
+struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
+  return mockCatalogService->getCatalogHandle(pMgmtEps);
+}
+
+int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+  return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData);
+}

source/libs/parser/test/mockCatalog.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MOCK_CATALOG_H
+#define MOCK_CATALOG_H
+
+#include "mockCatalogService.h"
+
+void generateMetaData(MockCatalogService* mcs);
+
+// mock
+struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps);
+int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData);
+
+#endif  // MOCK_CATALOG_H

source/libs/parser/test/mockCatalogService.cpp

@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mockCatalogService.h"
+
+#include <iomanip>
+#include <iostream>
+#include <map>
+
+#include "ttypes.h"
+
+std::unique_ptr<MockCatalogService> mockCatalogService;
+
+class TableBuilder : public ITableBuilder {
+public:
+  virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) {
+    assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns);
+    SSchema* col = meta_->schema + colIndex_;
+    col->type = type;
+    col->colId = colIndex_++;
+    col->bytes = bytes;
+    strcpy(col->name, name.c_str());
+    return *this;
+  }
+
+  virtual TableBuilder& setVgid(int16_t vgid) {
+    meta_->vgId = vgid;
+    return *this;
+  }
+
+  virtual TableBuilder& setPrecision(uint8_t precision) {
+    meta_->tableInfo.precision = precision;
+    return *this;
+  }
+
+  virtual void done() {
+    meta_->tableInfo.rowSize = rowsize_;
+  }
+
+private:
+  friend class MockCatalogServiceImpl;
+
+  static std::unique_ptr<TableBuilder> createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+    STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags));
+    if (nullptr == meta) {
+      throw std::bad_alloc();
+    }
+    meta->tableType = tableType;
+    meta->tableInfo.numOfTags = numOfTags;
+    meta->tableInfo.numOfColumns = numOfColumns;
+    return std::unique_ptr<TableBuilder>(new TableBuilder(meta));
+  }
+
+  TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) {
+  }
+
+  STableMeta* table() {
+    return meta_;
+  }
+
+  int32_t colIndex_;
+  int32_t rowsize_;
+  STableMeta* meta_;
+};
+
+class MockCatalogServiceImpl {
+public:
+  static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]);
+
+  MockCatalogServiceImpl() {
+  }
+
+  struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
+    return (struct SCatalog*)0x01;
+  }
+
+  int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+    return 0;
+  }
+
+  TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+    builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags);
+    meta_[db][tbname].reset(builder_->table());
+    meta_[db][tbname]->uid = id_++;
+    return *(builder_.get());
+  }
+
+  void showTables() const {
+    // number of forward fills
+    #define NOF(n) ((n) / 2)
+    // number of backward fills
+    #define NOB(n) ((n) % 2 ? (n) / 2 + 1 : (n) / 2)
+    // center aligned
+    #define CA(n, s) std::setw(NOF((n) - (s).length())) << "" << (s) << std::setw(NOB((n) - (s).length())) << "" << "|"
+    // string field length
+    #define SFL 20
+    // string field header
+    #define SH(h) CA(SFL, std::string(h))
+    // string field
+    #define SF(n) CA(SFL, n)
+    // integer field length
+    #define IFL 10
+    // integer field header
+    #define IH(i) CA(IFL, std::string(i))
+    // integer field
+    #define IF(i) CA(IFL, std::to_string(i))
+    // split line
+    #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ')
+
+    for (const auto& db : meta_) {
+      std::cout << SH("Database") << SH("Table") << SH("Type") << SH("Precision") << IH(std::string("Vgid")) << std::endl;
+      std::cout << SL(4, 1) << std::endl;
+      for (const auto& table : db.second) {
+        std::cout << SF(db.first) << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl;
+        // int16_t numOfFields = table.second->tableInfo.numOfTags + table.second->tableInfo.numOfColumns;
+        // for (int16_t i = 0; i < numOfFields; ++i) {
+        //   const SSchema* schema = table.second->schema + i;
+        //   std::cout << schema->name << " " << schema->type << " " << schema->bytes << std::endl;
+        // }
+      }
+    }
+  }
+
+private:
+  std::string ttToString(int8_t tableType) const {
+    switch (tableType) {
+      case TSDB_SUPER_TABLE:
+        return "super table";
+      case TSDB_CHILD_TABLE:
+        return "child table";
+      case TSDB_NORMAL_TABLE:
+        return "normal table";
+      default:
+        return "unknown";
+    }
+  }
+
+  std::string pToString(uint8_t precision) const {
+    switch (precision) {
+      case TSDB_TIME_PRECISION_MILLI:
+        return "millisecond";
+      case TSDB_TIME_PRECISION_MICRO:
+        return "microsecond";
+      case TSDB_TIME_PRECISION_NANO:
+        return "nanosecond";
+      default:
+        return "unknown";
+    }
+  }
+
+  uint64_t id_;
+  std::unique_ptr<TableBuilder> builder_;
+  std::map<std::string, std::map<std::string, std::shared_ptr<STableMeta> > > meta_;
+};
+
+MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {
+}
+
+MockCatalogService::~MockCatalogService() {
+}
+
+struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) {
+  return impl_->getCatalogHandle(pMgmtEps);
+}
+
+int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
+  return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData);
+}
+
+ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) {
+  return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags);
+}
+
+void MockCatalogService::showTables() const {
+  impl_->showTables();
+}

source/libs/parser/test/mockCatalogService.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MOCK_CATALOG_SERVICE_H
+#define MOCK_CATALOG_SERVICE_H
+
+#include <memory>
+#include <string>
+
+#include "catalog.h"
+
+class ITableBuilder {
+public:
+  ITableBuilder& addTag(const std::string& name, int8_t type) {
+    return addColumn(name, type, tDataTypes[type].bytes);
+  }
+
+  ITableBuilder& addTag(const std::string& name, int8_t type, int32_t bytes) {
+    return addColumn(name, type, bytes);
+  }
+
+  ITableBuilder& addColumn(const std::string& name, int8_t type) {
+    return addColumn(name, type, tDataTypes[type].bytes);
+  }
+
+  virtual ITableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) = 0;
+  virtual ITableBuilder& setVgid(int16_t vgid) = 0;
+  virtual ITableBuilder& setPrecision(uint8_t precision) = 0;
+  virtual void done() = 0;
+};
+
+class MockCatalogServiceImpl;
+
+class MockCatalogService {
+public:
+  static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]);
+
+  MockCatalogService();
+  ~MockCatalogService();
+  struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps);
+  int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData);
+  ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0);
+  void showTables() const;
+
+private:
+  std::unique_ptr<MockCatalogServiceImpl> impl_;
+};
+
+extern std::unique_ptr<MockCatalogService> mockCatalogService;
+
+#endif  // MOCK_CATALOG_SERVICE_H

source/libs/parser/test/parserMain.cpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "mockCatalog.h"
+
+class ParserEnv : public testing::Environment {
+public:
+  virtual void SetUp() {
+    mockCatalogService.reset(new MockCatalogService());
+    generateMetaData(mockCatalogService.get());
+  }
+
+  virtual void TearDown() {
+    mockCatalogService.reset();
+  }
+
+  ParserEnv() {}
+  virtual ~ParserEnv() {}
+};
+
+int main(int argc, char* argv[]) {
+	testing::AddGlobalTestEnvironment(new ParserEnv());
+	testing::InitGoogleTest(&argc, argv);
+	return RUN_ALL_TESTS();
+}

source/libs/parser/test/tokenizerTest.cpp

@@ -79,11 +79,6 @@ static void _init_tvariant_nchar(SVariant* t) {
   t->nLen = twcslen(t->wpz);
 }
 
-int main(int argc, char** argv) {
-  testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
 TEST(testCase, validateToken_test) {
   char t01[] = "abc";
   EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);

source/libs/transport/src/rpcMain.c

@@ -126,6 +126,8 @@ typedef struct SRpcConn {
   SRpcReqContext *pContext; // request context
 } SRpcConn;
 
+static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT;
+
 int tsRpcMaxUdpSize = 15000;  // bytes
 int tsProgressTimer = 100;
 // not configurable
@@ -220,9 +222,9 @@ static void rpcFree(void *p) {
   free(p);
 }
 
-int32_t rpcInit(void) {
+static void rpcInitImp(void) {
-  tsProgressTimer = tsRpcTimer/2; 
+  tsProgressTimer = tsRpcTimer / 2;
-  tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
+  tsRpcMaxRetry = tsRpcMaxTime * 1000 / tsProgressTimer;
   tsRpcHeadSize = RPC_MSG_OVERHEAD;
   tsRpcOverhead = sizeof(SRpcReqContext);
 
@@ -231,6 +233,11 @@ int32_t rpcInit(void) {
   return 0;
 }
 
+int32_t rpcInit(void) {
+  pthread_once(&tsRpcInitOnce, rpcInitImp);
+  return 0;
+}
+
 void rpcCleanup(void) {
   taosCloseRef(tsRpcRefId);
   tsRpcRefId = -1;
@@ -406,7 +413,7 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t
   // for TDengine, all the query, show commands shall have TCP connection
   char type = pMsg->msgType;
   if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE
-    || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STABLE_VGROUP
+    || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STB_VGROUP
     || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META
     || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE)
     pContext->connType = RPC_CONN_TCPC;

source/libs/wal/inc/walInt.h

@@ -18,6 +18,7 @@
 
 #include "wal.h"
 #include "compare.h"
+#include "tchecksum.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -32,6 +33,11 @@ typedef struct WalFileInfo {
   int64_t fileSize;
 } WalFileInfo;
 
+typedef struct WalIdxEntry {
+  int64_t ver;
+  int64_t offset;
+} WalIdxEntry;
+
 static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) {
   WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft;
   WalFileInfo* pInfoRight = (WalFileInfo*)pRight;
@@ -79,6 +85,26 @@ static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) {
   return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer);
 }
 
+static inline int walValidHeadCksum(SWalHead* pHead) {
+  return taosCheckChecksum((uint8_t*)&pHead->head, sizeof(SWalReadHead), pHead->cksumHead);
+}
+
+static inline int walValidBodyCksum(SWalHead* pHead) {
+  return taosCheckChecksum((uint8_t*)pHead->head.cont, pHead->head.len, pHead->cksumBody);
+}
+
+static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) {
+  return walValidHeadCksum(pHead) && walValidBodyCksum(pHead);
+}
+
+static inline uint32_t walCalcHeadCksum(SWalHead *pHead) {
+  return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead));
+}
+
+static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) {
+  return taosCalcChecksum(0, (uint8_t*)body, len);
+}
+
 int walReadMeta(SWal* pWal);
 int walWriteMeta(SWal* pWal);
 int walRollFileInfo(SWal* pWal);
@@ -87,6 +113,10 @@ char* walMetaSerialize(SWal* pWal);
 int walMetaDeserialize(SWal* pWal, const char* bytes);
 //meta section end
 
+//seek section
+int walChangeFile(SWal *pWal, int64_t ver);
+//seek section end
+
 int64_t walGetSeq();
 int walSeekVer(SWal *pWal, int64_t ver);
 int walRoll(SWal *pWal);

source/libs/wal/src/walMeta.c

@@ -24,6 +24,18 @@
 #include <libgen.h>
 #include <regex.h>
 
+int64_t walGetFirstVer(SWal *pWal) {
+  return pWal->firstVersion;
+}
+
+int64_t walGetSnaphostVer(SWal *pWal) {
+  return pWal->snapshotVersion;
+}
+
+int64_t walGetLastVer(SWal *pWal) {
+  return pWal->lastVersion;
+}
+
 int walRollFileInfo(SWal* pWal) {
   int64_t ts = taosGetTimestampSec();
 
@@ -34,6 +46,7 @@ int walRollFileInfo(SWal* pWal) {
     pInfo->closeTs = ts;
   }
 
+  //TODO: change to emplace back
   WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo));
   if(pNewInfo == NULL) {
     return -1;
@@ -44,12 +57,13 @@ int walRollFileInfo(SWal* pWal) {
   pNewInfo->closeTs = -1;
   pNewInfo->fileSize = 0;
   taosArrayPush(pWal->fileInfoSet, pNewInfo);
+  free(pNewInfo);
   return 0;
 }
 
 char* walMetaSerialize(SWal* pWal) {
   char buf[30];
-  if(pWal == NULL || pWal->fileInfoSet == NULL) return 0;
+  ASSERT(pWal->fileInfoSet);
   int sz = pWal->fileInfoSet->size;
   cJSON* pRoot = cJSON_CreateObject();
   cJSON* pMeta = cJSON_CreateObject();
@@ -91,7 +105,9 @@ char* walMetaSerialize(SWal* pWal) {
     sprintf(buf, "%" PRId64, pInfo->fileSize);
     cJSON_AddStringToObject(pField, "fileSize", buf);
   }
-  return cJSON_Print(pRoot);
+  char* serialized = cJSON_Print(pRoot);
+  cJSON_Delete(pRoot);
+  return serialized;
 }
 
 int walMetaDeserialize(SWal* pWal, const char* bytes) {
@@ -111,7 +127,8 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
   pFiles = cJSON_GetObjectItem(pRoot, "files");
   int sz = cJSON_GetArraySize(pFiles);
   //deserialize
-  SArray* pArray = taosArrayInit(sz, sizeof(WalFileInfo));
+  SArray* pArray = pWal->fileInfoSet;
+  taosArrayEnsureCap(pArray, sz);
   WalFileInfo *pData = pArray->pData;
   for(int i = 0; i < sz; i++) {
     cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
@@ -129,6 +146,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
   }
   taosArraySetSize(pArray, sz);
   pWal->fileInfoSet = pArray;
+  cJSON_Delete(pRoot);
   return 0;
 }
 
@@ -159,6 +177,8 @@ static int walFindCurMetaVer(SWal* pWal) {
       break;
     }
   }
+  closedir(dir);
+  regfree(&walMetaRegexPattern);
   return metaVer;
 }
 
@@ -183,6 +203,7 @@ int walWriteMeta(SWal* pWal) {
     walBuildMetaName(pWal, metaVer, fnameStr);
     remove(fnameStr);
   }
+  free(serialized);
   return 0;
 }
 
@@ -203,6 +224,7 @@ int walReadMeta(SWal* pWal) {
   if(buf == NULL) {
     return -1;
   }
+  memset(buf, 0, size+5);
   int tfd = tfOpenRead(fnameStr);
   if(tfRead(tfd, buf, size) != size) {
     free(buf);

source/libs/wal/src/walMgmt.c

@@ -80,23 +80,36 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return NULL;
   }
+  memset(pWal, 0, sizeof(SWal));
   pWal->writeLogTfd = -1;
   pWal->writeIdxTfd = -1;
+  pWal->writeCur = -1;
 
   //set config
   pWal->vgId = pCfg->vgId;
   pWal->fsyncPeriod = pCfg->fsyncPeriod;
   pWal->rollPeriod = pCfg->rollPeriod;
   pWal->segSize = pCfg->segSize;
+  pWal->retentionSize = pCfg->retentionSize;
+  pWal->retentionPeriod = pCfg->retentionPeriod;
   pWal->level = pCfg->walLevel;
 
-  //init status
+  //init version info
+  pWal->firstVersion = -1;
+  pWal->commitVersion = -1;
+  pWal->snapshotVersion = -1;
   pWal->lastVersion = -1;
+
+  pWal->snapshottingVer = -1;
+
+  pWal->totSize = 0;
+
+  //init status
   pWal->lastRollSeq = -1;
 
   //init write buffer
   memset(&pWal->head, 0, sizeof(SWalHead));
-  pWal->head.sver = 0;
+  pWal->head.head.sver = 0;
 
   tstrncpy(pWal->path, path, sizeof(pWal->path));
   pthread_mutex_init(&pWal->mutex, NULL);
@@ -114,6 +127,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
     walFreeObj(pWal);
     return NULL;
   }
+  walReadMeta(pWal);
 
   wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod);
 
@@ -145,9 +159,12 @@ void walClose(SWal *pWal) {
 
   pthread_mutex_lock(&pWal->mutex);
   tfClose(pWal->writeLogTfd);
+  pWal->writeLogTfd = -1;
   tfClose(pWal->writeIdxTfd);
-  /*taosArrayDestroy(pWal->fileInfoSet);*/
+  pWal->writeIdxTfd = -1;
-  /*pWal->fileInfoSet = NULL;*/
+  walWriteMeta(pWal);
+  taosArrayDestroy(pWal->fileInfoSet);
+  pWal->fileInfoSet = NULL;
   pthread_mutex_unlock(&pWal->mutex);
   taosRemoveRef(tsWal.refSetId, pWal->refId);
 }
@@ -157,7 +174,7 @@ static int32_t walInitObj(SWal *pWal) {
     wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
-  pWal->fileInfoSet = taosArrayInit(0, sizeof(WalFileInfo));
+  pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo));
   if(pWal->fileInfoSet == NULL) {
     wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
@@ -241,8 +258,9 @@ static int32_t walCreateThread() {
 static void walStopThread() {
   atomic_store_8(&tsWal.stop, 1);
 
-  if (taosCheckPthreadValid(tsWal.thread)) {
+  if (tsWal.thread != NULL && taosCheckPthreadValid(tsWal.thread)) {
     pthread_join(tsWal.thread, NULL);
+    tsWal.thread = NULL;
   }
 
   wDebug("wal thread is stopped");

source/libs/wal/src/walRead.c

@@ -15,19 +15,6 @@
 
 #include "walInt.h"
 #include "tfile.h"
-#include "tchecksum.h"
-
-static inline int walValidHeadCksum(SWalHead* pHead) {
-  return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead);
-}
-
-static inline int walValidBodyCksum(SWalHead* pHead) {
-  return taosCheckChecksum((uint8_t*)pHead->cont, pHead->len, pHead->cksumBody);
-}
-
-static int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) {
-  return walValidHeadCksum(pHead) && walValidBodyCksum(pHead);
-}
 
 int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
   int code;
@@ -49,13 +36,13 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
   if(walValidHeadCksum(*ppHead) != 0) {
     return -1;
   }
-  void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->len);
+  void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
   if(ptr == NULL) {
     free(*ppHead);
     *ppHead = NULL;
     return -1;
   }
-  if(tfRead(pWal->writeLogTfd, (*ppHead)->cont, (*ppHead)->len) != (*ppHead)->len) {
+  if(tfRead(pWal->writeLogTfd, (*ppHead)->head.cont, (*ppHead)->head.len) != (*ppHead)->head.len) {
     return -1;
   }
   //TODO: endian compatibility processing after read
@@ -69,18 +56,3 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
 int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {
   return 0;
 }
-
-int64_t walGetFirstVer(SWal *pWal) {
-  if (pWal == NULL) return 0;
-  return pWal->firstVersion;
-}
-
-int64_t walGetSnaphostVer(SWal *pWal) {
-  if (pWal == NULL) return 0;
-  return pWal->snapshotVersion;
-}
-
-int64_t walGetLastVer(SWal *pWal) {
-  if (pWal == NULL) return 0;
-  return pWal->lastVersion;
-}

source/libs/wal/src/walSeek.c

@@ -43,12 +43,35 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
   if (code != 0) {
     return -1;
   }
-  /*pWal->curLogOffset = readBuf[1];*/
-  pWal->curVersion = ver;
   return code;
 }
 
-static int walChangeFile(SWal *pWal, int64_t ver) {
+int walChangeFileToLast(SWal *pWal) {
+  int64_t idxTfd, logTfd;
+  WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
+  ASSERT(pRet != NULL);
+  int64_t fileFirstVer = pRet->firstVer;
+
+  char fnameStr[WAL_FILE_LEN];
+  walBuildIdxName(pWal, fileFirstVer, fnameStr);
+  idxTfd = tfOpenReadWrite(fnameStr);
+  if(idxTfd < 0) {
+    return -1;
+  }
+  walBuildLogName(pWal, fileFirstVer, fnameStr);
+  logTfd = tfOpenReadWrite(fnameStr);
+  if(logTfd < 0) {
+    return -1;
+  }
+  //switch file
+  pWal->writeIdxTfd = idxTfd;
+  pWal->writeLogTfd = logTfd;
+  //change status
+  pWal->curStatus = WAL_CUR_FILE_WRITABLE;
+  return 0;
+}
+
+int walChangeFile(SWal *pWal, int64_t ver) {
   int code = 0;
   int64_t idxTfd, logTfd;
   char fnameStr[WAL_FILE_LEN];
@@ -86,21 +109,21 @@ static int walChangeFile(SWal *pWal, int64_t ver) {
   return code;
 }
 
+int walGetVerOffset(SWal* pWal, int64_t ver) {
+  int code;
+  return 0;
+}
+
 int walSeekVer(SWal *pWal, int64_t ver) {
   int code;
-  if((!(pWal->curStatus & WAL_CUR_FAILED)) && ver == pWal->curVersion) {
+  if(ver == pWal->lastVersion) {
     return 0;
   }
-  if(ver > pWal->lastVersion) {
+  if(ver > pWal->lastVersion || ver < pWal->firstVersion) {
-    //TODO: some records are skipped
-    return -1;
-  }
-  if(ver < pWal->firstVersion) {
-    //TODO: try to seek pruned log
     return -1;
   }
   if(ver < pWal->snapshotVersion) {
-    //TODO: seek snapshotted log, invalid in some cases
+    //TODO: set flag to prevent roll back
   }
   if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
     code = walChangeFile(pWal, ver);

source/libs/wal/src/walWrite.c

@@ -21,65 +21,6 @@
 #include "tfile.h"
 #include "walInt.h"
 
-static void walFtruncate(SWal *pWal, int64_t ver);
-
-int32_t walCommit(SWal *pWal, int64_t ver) {
-  ASSERT(pWal->snapshotVersion <= pWal->commitVersion);
-  ASSERT(pWal->commitVersion <= pWal->lastVersion);
-  ASSERT(ver >= pWal->commitVersion);
-  ASSERT(ver <= pWal->lastVersion);
-  pWal->commitVersion = ver;
-  return 0;
-}
-
-int32_t walRollback(SWal *pWal, int64_t ver) {
-  //TODO: ftruncate
-  ASSERT(ver > pWal->commitVersion);
-  ASSERT(ver <= pWal->lastVersion);
-  //seek position
-  walSeekVer(pWal, ver); 
-  walFtruncate(pWal, ver);
-  return 0;
-}
-
-int32_t walTakeSnapshot(SWal *pWal, int64_t ver) {
-  pWal->snapshotVersion = ver;
-  int ts = taosGetTimestampSec();
-
-  int deleteCnt = 0;
-  int64_t newTotSize = pWal->totSize;
-  WalFileInfo tmp;
-  tmp.firstVer = ver;
-  //mark files safe to delete
-  WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
-  //iterate files, until the searched result
-  for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
-    if(pWal->totSize > pWal->retentionSize ||
-        iter->closeTs + pWal->retentionPeriod > ts) {
-      //delete according to file size or close time
-      deleteCnt++;
-      newTotSize -= iter->fileSize;
-    }
-  }
-  char fnameStr[WAL_FILE_LEN];
-  //remove file
-  for(int i = 0; i < deleteCnt; i++) {
-    WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i);
-    walBuildLogName(pWal, pInfo->firstVer, fnameStr); 
-    remove(fnameStr);
-    walBuildIdxName(pWal, pInfo->firstVer, fnameStr); 
-    remove(fnameStr);
-  }
-
-  //save snapshot ver, commit ver
-
-
-  //make new array, remove files
-  taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); 
-  pWal->totSize = newTotSize;
-
-  return 0;
-}
 
 #if 0
 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId);
@@ -172,6 +113,188 @@ void walRemoveAllOldFiles(void *handle) {
 }
 #endif
 
+int32_t walCommit(SWal *pWal, int64_t ver) {
+  ASSERT(pWal->commitVersion >= pWal->snapshotVersion);
+  ASSERT(pWal->commitVersion <= pWal->lastVersion);
+  if(ver < pWal->commitVersion || ver > pWal->lastVersion) {
+    return -1;
+  }
+  pWal->commitVersion = ver;
+  return 0;
+}
+
+int32_t walRollback(SWal *pWal, int64_t ver) {
+  int code;
+  char fnameStr[WAL_FILE_LEN];
+  if(ver == pWal->lastVersion) {
+    return 0;
+  }
+  if(ver > pWal->lastVersion || ver < pWal->commitVersion) {
+    return -1;
+  }
+  pthread_mutex_lock(&pWal->mutex);
+
+  //find correct file
+  if(ver < walGetLastFileFirstVer(pWal)) {
+    //close current files
+    tfClose(pWal->writeIdxTfd);
+    tfClose(pWal->writeLogTfd);
+    //open old files
+    code = walChangeFile(pWal, ver);
+    if(code != 0) {
+      return -1;
+    }
+
+    //delete files
+    int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
+    for(int i = pWal->writeCur; i < fileSetSize; i++) {
+      walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+      remove(fnameStr);
+      walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+      remove(fnameStr);
+    }
+    //pop from fileInfoSet
+    taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1);
+  }
+
+  walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
+  int64_t idxTfd = tfOpenReadWrite(fnameStr);
+
+  //change to deserialize function
+
+  if(idxTfd < 0) {
+    pthread_mutex_unlock(&pWal->mutex);
+    return -1;
+  }
+  int idxOff = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE;
+  code = tfLseek(idxTfd, idxOff, SEEK_SET);
+  if(code < 0) {
+    pthread_mutex_unlock(&pWal->mutex);
+    return -1;
+  }
+  //read idx file and get log file pos
+  //TODO:change to deserialize function
+  WalIdxEntry entry;
+  if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
+    pthread_mutex_unlock(&pWal->mutex);
+    return -1;
+  }
+  ASSERT(entry.ver == ver);
+
+  walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
+  int64_t logTfd = tfOpenReadWrite(fnameStr);
+  if(logTfd < 0) {
+    //TODO
+    pthread_mutex_unlock(&pWal->mutex);
+    return -1;
+  }
+  code = tfLseek(logTfd, entry.offset, SEEK_SET);
+  if(code < 0) {
+    //TODO
+    pthread_mutex_unlock(&pWal->mutex);
+    return -1;
+  }
+  //validate offset
+  SWalHead head;
+  ASSERT(tfValid(logTfd));
+  int size = tfRead(logTfd, &head, sizeof(SWalHead));
+  if(size != sizeof(SWalHead)) {
+    return -1;
+  }
+  code = walValidHeadCksum(&head);
+
+  ASSERT(code == 0);
+  if(code != 0) {
+    return -1;
+  }
+  if(head.head.version != ver) {
+    //TODO
+    return -1;
+  }
+  //truncate old files
+  code = tfFtruncate(logTfd, entry.offset);
+  if(code < 0) {
+    return -1;
+  }
+  code = tfFtruncate(idxTfd, idxOff);
+  if(code < 0) {
+    return -1;
+  }
+  pWal->lastVersion = ver - 1;
+  ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
+  ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
+
+  //unlock
+  pthread_mutex_unlock(&pWal->mutex);
+  return 0;
+}
+
+int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) {
+  pWal->snapshottingVer = ver;
+  //check file rolling
+  if(pWal->retentionPeriod == 0) {
+    walRoll(pWal);
+  }
+
+  return 0;
+}
+
+int32_t walEndTakeSnapshot(SWal *pWal) {
+  int64_t ver = pWal->snapshottingVer;
+  if(ver == -1) return -1;
+
+  pWal->snapshotVersion = ver;
+  int ts = taosGetTimestampSec();
+
+  int deleteCnt = 0;
+  int64_t newTotSize = pWal->totSize;
+  WalFileInfo tmp;
+  tmp.firstVer = ver;
+  //find files safe to delete
+  WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
+  if(ver >= pInfo->lastVer) {
+    pInfo++;
+  }
+  //iterate files, until the searched result
+  for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
+    if(pWal->totSize > pWal->retentionSize ||
+        iter->closeTs + pWal->retentionPeriod > ts) {
+      //delete according to file size or close time
+      deleteCnt++;
+      newTotSize -= iter->fileSize;
+    }
+  }
+  char fnameStr[WAL_FILE_LEN];
+  //remove file
+  for(int i = 0; i < deleteCnt; i++) {
+    WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i);
+    walBuildLogName(pWal, pInfo->firstVer, fnameStr); 
+    remove(fnameStr);
+    walBuildIdxName(pWal, pInfo->firstVer, fnameStr); 
+    remove(fnameStr);
+  }
+
+  //make new array, remove files
+  taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); 
+  if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
+    pWal->writeCur = -1;
+    pWal->firstVersion = -1;
+  } else {
+    pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
+  }
+  pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;;
+  pWal->totSize = newTotSize;
+  pWal->snapshottingVer = -1;
+
+  //save snapshot ver, commit ver
+  int code = walWriteMeta(pWal);
+  if(code != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
 int walRoll(SWal *pWal) {
   int code = 0;
   if(pWal->writeIdxTfd != -1) {
@@ -211,6 +334,7 @@ int walRoll(SWal *pWal) {
   //switch file
   pWal->writeIdxTfd = idxTfd;
   pWal->writeLogTfd = logTfd;
+  pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
   //change status
   pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE;
 
@@ -218,44 +342,11 @@ int walRoll(SWal *pWal) {
   return 0;
 }
 
-int walChangeFileToLast(SWal *pWal) {
-  int64_t idxTfd, logTfd;
-  WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
-  ASSERT(pRet != NULL);
-  int64_t fileFirstVer = pRet->firstVer;
-
-  char fnameStr[WAL_FILE_LEN];
-  walBuildIdxName(pWal, fileFirstVer, fnameStr);
-  idxTfd = tfOpenReadWrite(fnameStr);
-  if(idxTfd < 0) {
-    return -1;
-  }
-  walBuildLogName(pWal, fileFirstVer, fnameStr);
-  logTfd = tfOpenReadWrite(fnameStr);
-  if(logTfd < 0) {
-    return -1;
-  }
-  //switch file
-  pWal->writeIdxTfd = idxTfd;
-  pWal->writeLogTfd = logTfd;
-  //change status
-  pWal->curVersion = fileFirstVer;
-  pWal->curStatus = WAL_CUR_FILE_WRITABLE;
-  return 0;
-}
-
 static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
-  int code = 0;
+  WalIdxEntry entry = { .ver = ver, .offset = offset };
-  //get index file
+  int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry));
-  if(!tfValid(pWal->writeIdxTfd)) {
+  if(size != sizeof(WalIdxEntry)) {
-    code = TAOS_SYSTEM_ERROR(errno);
+    //TODO truncate
-
-    wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno));
-    return code;
-  }
-  int64_t writeBuf[2] = { ver, offset };
-  int size = tfWrite(pWal->writeIdxTfd, writeBuf, sizeof(writeBuf));
-  if(size != sizeof(writeBuf)) {
     return -1;
   }
   return 0;
@@ -270,13 +361,14 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
 
   if (index == pWal->lastVersion + 1) {
     if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
+      pWal->firstVersion = index;
       code = walRoll(pWal);
       ASSERT(code == 0);
     } else {
       int64_t passed = walGetSeq() - pWal->lastRollSeq;
-      if(pWal->rollPeriod != -1 && passed > pWal->rollPeriod) {
+      if(pWal->rollPeriod != -1 && pWal->rollPeriod != 0 && passed > pWal->rollPeriod) {
         walRoll(pWal);
-      } else if(pWal->segSize != -1 && walGetLastFileSize(pWal) > pWal->segSize) {
+      } else if(pWal->segSize != -1 && pWal->segSize != 0 && walGetLastFileSize(pWal) > pWal->segSize) {
         walRoll(pWal);
       }
     }
@@ -287,16 +379,13 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
   }
   /*if (!tfValid(pWal->curLogTfd)) return 0;*/
 
-  pWal->head.version = index;
-
-  pWal->head.signature = WAL_SIGNATURE;
-  pWal->head.len = bodyLen;
-  pWal->head.msgType = msgType;
-
-  pWal->head.cksumHead = taosCalcChecksum(0, (const uint8_t*)&pWal->head, sizeof(SWalHead)- sizeof(uint32_t)*2);
-  pWal->head.cksumBody = taosCalcChecksum(0, (const uint8_t*)&body, bodyLen);
-
   pthread_mutex_lock(&pWal->mutex);
+  pWal->head.head.version = index;
+
+  pWal->head.head.len = bodyLen;
+  pWal->head.head.msgType = msgType;
+  pWal->head.cksumHead = walCalcHeadCksum(&pWal->head);
+  pWal->head.cksumBody = walCalcBodyCksum(body, bodyLen);
 
   if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) {
     //ftruncate
@@ -312,6 +401,7 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
   code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal));
   if(code != 0) {
     //TODO
+    return -1;
   }
 
   //set status
@@ -326,8 +416,6 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
 }
 
 void walFsync(SWal *pWal, bool forceFsync) {
-  if (pWal == NULL || !tfValid(pWal->writeLogTfd)) return;
-
   if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) {
     wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal));
     if (tfFsync(pWal->writeLogTfd) < 0) {
@@ -408,7 +496,7 @@ static int walValidateOffset(SWal* pWal, int64_t ver) {
   int code = 0;
   SWalHead *pHead = NULL;
   code = (int)walRead(pWal, &pHead, ver);
-  if(pHead->version != ver) {
+  if(pHead->head.version != ver) {
     return -1;
   }
   return 0;
@@ -428,15 +516,6 @@ static int64_t walGetOffset(SWal* pWal, int64_t ver) {
   return 0;
 }
 
-static void walFtruncate(SWal *pWal, int64_t ver) {
-  int64_t tfd = pWal->writeLogTfd;
-  tfFtruncate(tfd, ver);
-  tfFsync(tfd);
-  tfd = pWal->writeIdxTfd;
-  tfFtruncate(tfd, ver * WAL_IDX_ENTRY_SIZE);
-  tfFsync(tfd);
-}
-
 #if 0
 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) {
   int64_t pos = *offset;

source/libs/wal/test/walMetaTest.cpp

@@ -18,12 +18,47 @@ class WalCleanEnv : public ::testing::Test {
 
     void SetUp() override {
       taosRemoveDir(pathName);
-      SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal));
+      SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
       memset(pCfg, 0, sizeof(SWalCfg));
       pCfg->rollPeriod = -1;
       pCfg->segSize = -1;
+      pCfg->retentionPeriod = 0;
+      pCfg->retentionSize = 0;
       pCfg->walLevel = TAOS_WAL_FSYNC;
       pWal = walOpen(pathName, pCfg);
+      free(pCfg);
+      ASSERT(pWal != NULL);
+    }
+
+    void TearDown() override {
+      walClose(pWal);
+      pWal = NULL;
+    }
+
+    SWal* pWal = NULL;
+    const char* pathName = "/tmp/wal_test";
+};
+
+class WalCleanDeleteEnv : public ::testing::Test {
+  protected:
+    static void SetUpTestCase() {
+      int code = walInit();
+      ASSERT(code == 0);
+    }
+
+    static void TearDownTestCase() {
+      walCleanUp();
+    }
+
+    void SetUp() override {
+      taosRemoveDir(pathName);
+      SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
+      memset(pCfg, 0, sizeof(SWalCfg));
+      pCfg->retentionPeriod = 0;
+      pCfg->retentionSize = 0;
+      pCfg->walLevel = TAOS_WAL_FSYNC;
+      pWal = walOpen(pathName, pCfg);
+      free(pCfg);
       ASSERT(pWal != NULL);
     }
 
@@ -47,13 +82,22 @@ class WalKeepEnv : public ::testing::Test {
       walCleanUp();
     }
 
+    void walResetEnv() {
+      TearDown();
+      taosRemoveDir(pathName);
+      SetUp();
+    }
+
     void SetUp() override {
-      SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); 
+      SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); 
       memset(pCfg, 0, sizeof(SWalCfg));
       pCfg->rollPeriod = -1;
       pCfg->segSize = -1;
+      pCfg->retentionPeriod = 0;
+      pCfg->retentionSize = 0;
       pCfg->walLevel = TAOS_WAL_FSYNC;
       pWal = walOpen(pathName, pCfg);
+      free(pCfg);
       ASSERT(pWal != NULL);
     }
 
@@ -94,6 +138,7 @@ TEST_F(WalCleanEnv, serialize) {
   ASSERT(code == 0);
   char*ss = walMetaSerialize(pWal);
   printf("%s\n", ss);
+  free(ss);
   code = walWriteMeta(pWal);
   ASSERT(code == 0);
 }
@@ -111,39 +156,105 @@ TEST_F(WalCleanEnv, removeOldMeta) {
 }
 
 TEST_F(WalKeepEnv, readOldMeta) {
-  int code = walRollFileInfo(pWal);
+  walResetEnv();
-  ASSERT(code == 0);
+  const char* ranStr = "tvapq02tcp";
-  code = walWriteMeta(pWal);
+  int len = strlen(ranStr);
-  ASSERT(code == 0);
+  int code;
-  code = walRollFileInfo(pWal);
+
-  ASSERT(code == 0);
+  for(int i = 0; i < 10; i++) {
-  code = walWriteMeta(pWal);
+    code = walWrite(pWal, i, i+1, (void*)ranStr, len); 
-  ASSERT(code == 0);
+    ASSERT_EQ(code, 0);
-  char*oldss = walMetaSerialize(pWal);
+    ASSERT_EQ(pWal->lastVersion, i);
+    code = walWrite(pWal, i+2, i, (void*)ranStr, len);
+    ASSERT_EQ(code, -1);
+    ASSERT_EQ(pWal->lastVersion, i);
+  }
+  char* oldss = walMetaSerialize(pWal);
 
   TearDown();
   SetUp();
-  code = walReadMeta(pWal);
+
-  ASSERT(code == 0);
+  ASSERT_EQ(pWal->firstVersion, 0);
+  ASSERT_EQ(pWal->lastVersion, 9);
+
   char* newss = walMetaSerialize(pWal);
 
-  int len = strlen(oldss);
+  len = strlen(oldss);
   ASSERT_EQ(len, strlen(newss));
   for(int i = 0; i < len; i++) {
     EXPECT_EQ(oldss[i], newss[i]);
   }
+  free(oldss);
+  free(newss);
 }
 
-TEST_F(WalKeepEnv, write) {
+TEST_F(WalCleanEnv, write) {
   const char* ranStr = "tvapq02tcp";
   const int len = strlen(ranStr);
   int code;
   for(int i = 0; i < 10; i++) {
     code = walWrite(pWal, i, i+1, (void*)ranStr, len); 
     ASSERT_EQ(code, 0);
+    ASSERT_EQ(pWal->lastVersion, i);
     code = walWrite(pWal, i+2, i, (void*)ranStr, len);
     ASSERT_EQ(code, -1);
+    ASSERT_EQ(pWal->lastVersion, i);
   }
   code = walWriteMeta(pWal);
   ASSERT_EQ(code, 0);
 }
+
+TEST_F(WalCleanEnv, rollback) {
+  const char* ranStr = "tvapq02tcp";
+  const int len = strlen(ranStr);
+  int code;
+  for(int i = 0; i < 10; i++) {
+    code = walWrite(pWal, i, i+1, (void*)ranStr, len); 
+    ASSERT_EQ(code, 0);
+    ASSERT_EQ(pWal->lastVersion, i);
+  }
+  code = walRollback(pWal, 5);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(pWal->lastVersion, 4);
+  code = walRollback(pWal, 3);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(pWal->lastVersion, 2);
+  code = walWriteMeta(pWal);
+  ASSERT_EQ(code, 0);
+}
+
+TEST_F(WalCleanDeleteEnv, roll) {
+  const char* ranStr = "tvapq02tcp";
+  const int len = strlen(ranStr);
+  int code;
+  int i;
+  for(i = 0; i < 100; i++) {
+    code = walWrite(pWal, i, 0, (void*)ranStr, len);
+    ASSERT_EQ(code, 0);
+    ASSERT_EQ(pWal->lastVersion, i);
+    code = walCommit(pWal, i);
+    ASSERT_EQ(pWal->commitVersion, i);
+  }
+
+  walBeginTakeSnapshot(pWal, i-1);
+  ASSERT_EQ(pWal->snapshottingVer, i-1); 
+  walEndTakeSnapshot(pWal);
+  ASSERT_EQ(pWal->snapshotVersion, i-1); 
+  ASSERT_EQ(pWal->snapshottingVer, -1); 
+
+  code = walWrite(pWal, 5, 0, (void*)ranStr, len);
+  ASSERT_NE(code, 0);
+
+  for(; i < 200; i++) {
+    code = walWrite(pWal, i, 0, (void*)ranStr, len);
+    ASSERT_EQ(code, 0);
+    code = walCommit(pWal, i);
+    ASSERT_EQ(pWal->commitVersion, i);
+  }
+
+  //code = walWriteMeta(pWal);
+  code = walBeginTakeSnapshot(pWal, i - 1);
+  ASSERT_EQ(code, 0);
+  code = walEndTakeSnapshot(pWal);
+  ASSERT_EQ(code, 0);
+}

source/os/src/osString.c

@@ -37,7 +37,7 @@ char *taosCharsetReplace(char *charsetstr) {
   return strdup(charsetstr);
 }
 
-int64_t taosStr2int64(char *str) {
+int64_t taosStr2int64(const char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
@@ -107,7 +107,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
   return len;
 }
 
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
+bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
   memset(ucs4, 0, ucs4_max_len);
   mbstate_t state = {0};
   int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);

source/util/src/tarray.c

@@ -58,24 +58,31 @@ static int32_t taosArrayResize(SArray* pArray) {
   return 0;
 }
 
-void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) {
+int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
-  if (pArray == NULL || pData == NULL) {
+  if (newCap > pArray->capacity) {
-    return NULL;
-  }
-
-  if (pArray->size + nEles > pArray->capacity) {
     size_t tsize = (pArray->capacity << 1u);
-    while (pArray->size + nEles > tsize) {
+    while (newCap > tsize) {
       tsize = (tsize << 1u);
     }
 
     pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize);
     if (pArray->pData == NULL) {
-      return NULL;
+      return -1;
     }
 
     pArray->capacity = tsize;
   }
+  return 0;
+}
+
+void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) {
+  if (pArray == NULL || pData == NULL) {
+    return NULL;
+  }
+
+  if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){
+    return NULL;
+  }
 
   void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
   memcpy(dst, pData, pArray->elemSize * nEles);
@@ -241,12 +248,16 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) {
   assert(cnt <= pArray->size);
   pArray->size = pArray->size - cnt;
   if(pArray->size == 0) {
-    pArray->size = 0;
     return;
   }
   memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size);
 }
 
+void taosArrayPopTailBatch(SArray* pArray, size_t cnt) {
+  assert(cnt <= pArray->size);
+  pArray->size = pArray->size - cnt;
+}
+
 void taosArrayRemove(SArray* pArray, size_t index) {
   assert(index < pArray->size);
   
@@ -329,6 +340,11 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa
   return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
 }
 
+int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) {
+  void* item = taosArraySearch(pArray, key, comparFn, flags);
+  return (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize;
+}
+
 void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
   assert(pArray != NULL);
   qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);

source/util/src/terror.c

@@ -162,22 +162,23 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_VER,         "Invalid raw data vers
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN,         "Invalid raw data len")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT,     "Invalid raw data content")
 
+// mnode-dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST,      "DNode already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST,          "DNode does not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         "VGroup does not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER,         "Master DNode cannot be removed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES,         "Out of DNodes")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, "Cluster cfg inconsistent")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG,      "Cluster cfg inconsistent")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, "Invalid dnode cfg option")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       "Cluster id not match")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED,          "Balance already enabled")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG,        "Invalid dnode cfg")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP,         "Invalid dnode end point")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID,         "Invalid dnode id")
+
+// mnode-vgroup
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         "VGroup does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE,      "Vgroup not in dnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  "Vgroup already in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE,           "Dnode not avaliable")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID,       "Cluster id not match")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY,                "Cluster not ready")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED,  "Dnode Id not configured")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED,  "Dnode Ep not configured")
 
+// mnode-acct
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST,       "Account already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_NOT_EXIST,           "Invalid account")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION,      "Invalid account options")
@@ -193,8 +194,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS,           "Too many users")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST,       "Mnode already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST,           "Mnode not there")
 
-
+// mnode-stable
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST,      "Table already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST,        "Stable already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID,         "Table name too long")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME,       "Table does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE,       "Invalid table type in tsdb")

source/util/src/tutil.c

@@ -166,7 +166,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) {
   return split;
 }
 
-char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
+char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) {
   for (int32_t i = 0; i < len; ++i) {
 
     // skip the needle in quote, jump to the end of quoted string
@@ -179,7 +179,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
     }
 
     if (haystack[i] == needle) {
-      return &haystack[i];
+      return (char *)&haystack[i];
     }
   }
 
@@ -417,16 +417,3 @@ void taosIp2String(uint32_t ip, char *str) {
 void taosIpPort2String(uint32_t ip, uint16_t port, char *str) {
   sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port);
 }
-
-int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
-  *port = 0;
-  strcpy(fqdn, ep);
-
-  char *temp = strchr(fqdn, ':');
-  if (temp) {
-    *temp = 0;
-    *port = atoi(temp + 1);
-  }
-
-  return 0;
-}

src/client/inc/tscParseLine.h

@@ -28,7 +28,7 @@ typedef struct {
 } TAOS_SML_KV;
 
 typedef struct {
-  char* stableName;
+  char* stbName;
 
   char* childTableName;
   TAOS_SML_KV* tags;