Merge remote-tracking branch 'origin/3.0' into feature/shm

include/client/taos.h

@@ -54,6 +54,7 @@ typedef void   TAOS_SUB;
 #define TSDB_DATA_TYPE_BLOB       18  // binary
 #define TSDB_DATA_TYPE_MEDIUMBLOB 19
 #define TSDB_DATA_TYPE_BINARY     TSDB_DATA_TYPE_VARCHAR  // string
+#define TSDB_DATA_TYPE_MAX        20
 
 typedef enum {
   TSDB_OPTION_LOCALE,

include/common/taosdef.h

@@ -76,6 +76,13 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
+typedef enum {
+  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
+  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
+} ETsdbBSmaType;
+
+#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tdataformat.h

@@ -70,11 +70,13 @@ typedef struct {
 #pragma pack(pop)
 
 #define colType(col)   ((col)->type)
+#define colSma(col)    ((col)->sma)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
+#define colSetSma(col, s)    (colSma(col) = (s))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
@@ -139,7 +141,7 @@ typedef struct {
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition

include/common/tmsg.h

@@ -265,6 +265,20 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+typedef struct {
+  int8_t   type;
+  int8_t   sma;  // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I
+  col_id_t colId;
+  int32_t  bytes;
+  char     name[TSDB_COL_NAME_LEN];
+} SSchemaEx;
+
+#define SSCHMEA_TYPE(s) ((s)->type)
+#define SSCHMEA_SMA(s) ((s)->sma)
+#define SSCHMEA_COLID(s) ((s)->colId)
+#define SSCHMEA_BYTES(s) ((s)->bytes)
+#define SSCHMEA_NAME(s) ((s)->name)
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
@@ -524,6 +538,7 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
+  int8_t  replications;
 } SAlterDbReq;
 
 int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);
@@ -1402,13 +1417,12 @@ typedef struct SVCreateTbReq {
   };
   union {
     struct {
-      tb_uid_t    suid;
+      tb_uid_t   suid;
-      uint32_t    nCols;
+      col_id_t   nCols;
-      SSchema*    pSchema;
+      col_id_t   nBSmaCols;
-      uint32_t    nTagCols;
+      SSchemaEx* pSchema;
-      SSchema*    pTagSchema;
+      col_id_t   nTagCols;
-      col_id_t    nBSmaCols;
+      SSchema*   pTagSchema;
-      col_id_t*   pBSmaCols;
       SRSmaParam* pRSmaParam;
     } stbCfg;
     struct {
@@ -1416,10 +1430,9 @@ typedef struct SVCreateTbReq {
       SKVRow   pTag;
     } ctbCfg;
     struct {
-      uint32_t    nCols;
+      col_id_t   nCols;
-      SSchema*    pSchema;
+      col_id_t   nBSmaCols;
-      col_id_t    nBSmaCols;
+      SSchemaEx* pSchema;
-      col_id_t*   pBSmaCols;
       SRSmaParam* pRSmaParam;
     } ntbCfg;
   };
@@ -1898,7 +1911,10 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq)
 
 typedef struct {
   uint32_t nCols;
-  SSchema* pSchema;
+  union {
+    SSchema*   pSchema;
+    SSchemaEx* pSchemaEx;
+  };
 } SSchemaWrapper;
 
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {

include/common/ttime.h

@@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
 
 int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
+char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/common/ttokendef.h

@@ -82,21 +82,21 @@
 #define TK_SINGLE_STABLE                   64
 #define TK_STREAM_MODE                     65
 #define TK_RETENTIONS                      66
-#define TK_TABLE                           67
+#define TK_NK_COMMA                        67
-#define TK_NK_LP                           68
+#define TK_TABLE                           68
-#define TK_NK_RP                           69
+#define TK_NK_LP                           69
-#define TK_STABLE                          70
+#define TK_NK_RP                           70
-#define TK_ADD                             71
+#define TK_STABLE                          71
-#define TK_COLUMN                          72
+#define TK_ADD                             72
-#define TK_MODIFY                          73
+#define TK_COLUMN                          73
-#define TK_RENAME                          74
+#define TK_MODIFY                          74
-#define TK_TAG                             75
+#define TK_RENAME                          75
-#define TK_SET                             76
+#define TK_TAG                             76
-#define TK_NK_EQ                           77
+#define TK_SET                             77
-#define TK_USING                           78
+#define TK_NK_EQ                           78
-#define TK_TAGS                            79
+#define TK_USING                           79
-#define TK_NK_DOT                          80
+#define TK_TAGS                            80
-#define TK_NK_COMMA                        81
+#define TK_NK_DOT                          81
 #define TK_COMMENT                         82
 #define TK_BOOL                            83
 #define TK_TINYINT                         84
@@ -131,69 +131,91 @@
 #define TK_FUNCTIONS                      113
 #define TK_INDEXES                        114
 #define TK_FROM                           115
-#define TK_LIKE                           116
+#define TK_ACCOUNTS                       116
-#define TK_INDEX                          117
+#define TK_APPS                           117
-#define TK_FULLTEXT                       118
+#define TK_CONNECTIONS                    118
-#define TK_FUNCTION                       119
+#define TK_LICENCE                        119
-#define TK_INTERVAL                       120
+#define TK_QUERIES                        120
-#define TK_TOPIC                          121
+#define TK_SCORES                         121
-#define TK_AS                             122
+#define TK_TOPICS                         122
-#define TK_DESC                           123
+#define TK_VARIABLES                      123
-#define TK_DESCRIBE                       124
+#define TK_LIKE                           124
-#define TK_RESET                          125
+#define TK_INDEX                          125
-#define TK_QUERY                          126
+#define TK_FULLTEXT                       126
-#define TK_EXPLAIN                        127
+#define TK_FUNCTION                       127
-#define TK_ANALYZE                        128
+#define TK_INTERVAL                       128
-#define TK_VERBOSE                        129
+#define TK_TOPIC                          129
-#define TK_NK_BOOL                        130
+#define TK_AS                             130
-#define TK_RATIO                          131
+#define TK_DESC                           131
-#define TK_NULL                           132
+#define TK_DESCRIBE                       132
-#define TK_NK_VARIABLE                    133
+#define TK_RESET                          133
-#define TK_NK_UNDERLINE                   134
+#define TK_QUERY                          134
-#define TK_ROWTS                          135
+#define TK_EXPLAIN                        135
-#define TK_TBNAME                         136
+#define TK_ANALYZE                        136
-#define TK_QSTARTTS                       137
+#define TK_VERBOSE                        137
-#define TK_QENDTS                         138
+#define TK_NK_BOOL                        138
-#define TK_WSTARTTS                       139
+#define TK_RATIO                          139
-#define TK_WENDTS                         140
+#define TK_COMPACT                        140
-#define TK_WDURATION                      141
+#define TK_VNODES                         141
-#define TK_BETWEEN                        142
+#define TK_IN                             142
-#define TK_IS                             143
+#define TK_OUTPUTTYPE                     143
-#define TK_NK_LT                          144
+#define TK_AGGREGATE                      144
-#define TK_NK_GT                          145
+#define TK_BUFSIZE                        145
-#define TK_NK_LE                          146
+#define TK_STREAM                         146
-#define TK_NK_GE                          147
+#define TK_INTO                           147
-#define TK_NK_NE                          148
+#define TK_KILL                           148
-#define TK_MATCH                          149
+#define TK_CONNECTION                     149
-#define TK_NMATCH                         150
+#define TK_MERGE                          150
-#define TK_IN                             151
+#define TK_VGROUP                         151
-#define TK_JOIN                           152
+#define TK_REDISTRIBUTE                   152
-#define TK_INNER                          153
+#define TK_SPLIT                          153
-#define TK_SELECT                         154
+#define TK_SYNCDB                         154
-#define TK_DISTINCT                       155
+#define TK_NULL                           155
-#define TK_WHERE                          156
+#define TK_NK_VARIABLE                    156
-#define TK_PARTITION                      157
+#define TK_NK_UNDERLINE                   157
-#define TK_BY                             158
+#define TK_ROWTS                          158
-#define TK_SESSION                        159
+#define TK_TBNAME                         159
-#define TK_STATE_WINDOW                   160
+#define TK_QSTARTTS                       160
-#define TK_SLIDING                        161
+#define TK_QENDTS                         161
-#define TK_FILL                           162
+#define TK_WSTARTTS                       162
-#define TK_VALUE                          163
+#define TK_WENDTS                         163
-#define TK_NONE                           164
+#define TK_WDURATION                      164
-#define TK_PREV                           165
+#define TK_BETWEEN                        165
-#define TK_LINEAR                         166
+#define TK_IS                             166
-#define TK_NEXT                           167
+#define TK_NK_LT                          167
-#define TK_GROUP                          168
+#define TK_NK_GT                          168
-#define TK_HAVING                         169
+#define TK_NK_LE                          169
-#define TK_ORDER                          170
+#define TK_NK_GE                          170
-#define TK_SLIMIT                         171
+#define TK_NK_NE                          171
-#define TK_SOFFSET                        172
+#define TK_MATCH                          172
-#define TK_LIMIT                          173
+#define TK_NMATCH                         173
-#define TK_OFFSET                         174
+#define TK_JOIN                           174
-#define TK_ASC                            175
+#define TK_INNER                          175
-#define TK_NULLS                          176
+#define TK_SELECT                         176
-#define TK_FIRST                          177
+#define TK_DISTINCT                       177
-#define TK_LAST                           178
+#define TK_WHERE                          178
+#define TK_PARTITION                      179
+#define TK_BY                             180
+#define TK_SESSION                        181
+#define TK_STATE_WINDOW                   182
+#define TK_SLIDING                        183
+#define TK_FILL                           184
+#define TK_VALUE                          185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_NEXT                           189
+#define TK_GROUP                          190
+#define TK_HAVING                         191
+#define TK_ORDER                          192
+#define TK_SLIMIT                         193
+#define TK_SOFFSET                        194
+#define TK_LIMIT                          195
+#define TK_OFFSET                         196
+#define TK_ASC                            197
+#define TK_NULLS                          198
+#define TK_FIRST                          199
+#define TK_LAST                           200
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -207,10 +229,9 @@
 #define TK_NK_COLON                       500
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502
-#define TK_INTO                           503
 #define TK_NOW                            504
 #define TK_VALUES                         507
-#define TK_IMPORT                         507
+#define TK_IMPORT                         509
 #define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535

include/libs/command/command.h

@@ -15,5 +15,10 @@
 
 #include "cmdnodes.h"
 #include "tmsg.h"
+#include "plannodes.h"
 
 int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
+
+

include/libs/nodes/cmdnodes.h

@@ -38,7 +38,9 @@ typedef struct SDatabaseOptions {
   int32_t fsyncPeriod;
   int32_t maxRowsPerBlock;
   int32_t minRowsPerBlock;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t precision;
   int32_t quorum;
   int32_t replica;
@@ -76,7 +78,9 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType type;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t ttl;
   char comments[TSDB_STB_COMMENT_LEN];
   SNodeList* pSma;
@@ -193,6 +197,12 @@ typedef struct SShowStmt {
   SNode* pTbNamePattern; // SValueNode
 } SShowStmt;
 
+typedef struct SShowCreatStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SShowCreatStmt;
+
 typedef enum EIndexType {
   INDEX_TYPE_SMA = 1,
   INDEX_TYPE_FULLTEXT

include/libs/nodes/nodes.h

@@ -103,6 +103,15 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_STMT,
   QUERY_NODE_DESCRIBE_STMT,
   QUERY_NODE_RESET_QUERY_CACHE_STMT,
+  QUERY_NODE_COMPACT_STMT,
+  QUERY_NODE_CREATE_FUNCTION_STMT,
+  QUERY_NODE_DROP_FUNCTION_STMT,
+  QUERY_NODE_CREATE_STREAM_STMT,
+  QUERY_NODE_DROP_STREAM_STMT,
+  QUERY_NODE_MERGE_VGROUP_STMT,
+  QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
+  QUERY_NODE_SPLIT_VGROUP_STMT,
+  QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -115,6 +124,18 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
   QUERY_NODE_SHOW_STREAMS_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_CONNECTIONS_STMT,
+  QUERY_NODE_SHOW_LICENCE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
+  QUERY_NODE_SHOW_QUERIES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TOPICS_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -215,6 +236,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/plannodes.h

@@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode {
   int64_t    sliding;
   int8_t     intervalUnit;
   int8_t     slidingUnit;
-  uint8_t precision;
+  uint8_t    precision;
   SFillNode* pFill;
 } SIntervalPhysiNode;
 

include/libs/nodes/querynodes.h

@@ -314,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
 void* nodesGetValueFromNode(SValueNode *pNode);
+char* nodesGetStrValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus
 }

include/libs/qcom/query.h

@@ -53,6 +53,7 @@ typedef struct SIndexMeta {
 
 } SIndexMeta;
 
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
     }                                                                    \
   } while (0)
 
+#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+
 #ifdef __cplusplus
 }
 #endif

include/os/osMemory.h

@@ -32,13 +32,16 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 
 #define taosMemoryFreeClear(ptr) \
   do {                           \
-    taosMemoryFree(ptr);         \
+    if (ptr) {                   \
-    (ptr)=NULL;                  \
+      taosMemoryFree(ptr);       \
+      (ptr) = NULL;              \
+    }                            \
   } while (0)
 
 #ifdef __cplusplus

include/os/osTime.h

@@ -46,6 +46,14 @@ extern "C" {
 #define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
 #define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
 
+#define NANOSECOND_PER_USEC   (1000L)
+#define NANOSECOND_PER_MSEC   (1000000L)
+#define NANOSECOND_PER_SEC    (1000000000L)
+#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60)
+#define NANOSECOND_PER_HOUR   (NANOSECOND_PER_MINUTE * 60)
+#define NANOSECOND_PER_DAY    (NANOSECOND_PER_HOUR * 24)
+#define NANOSECOND_PER_WEEK   (NANOSECOND_PER_DAY * 7)
+
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
 //@return timestamp in second

include/util/taoserror.h

@@ -479,6 +479,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 #define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
+#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
+#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 
 #ifdef __cplusplus
 }

include/util/tcoding.h

@@ -57,6 +57,8 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
   return POINTER_SHIFT(buf, sizeof(*value));
 }
 
+static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {

include/util/tdef.h

@@ -224,6 +224,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_APP_NAME_LEN    TSDB_UNI_LEN
 #define TSDB_STB_COMMENT_LEN 1024
+
 /**
  *  In some scenarios uint16_t (0~65535) is used to store the row len.
  *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
@@ -303,13 +304,13 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TOTAL_BLOCKS     10000
 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
 
-#define TSDB_MIN_DAYS_PER_FILE     1
+#define TSDB_MIN_DAYS_PER_FILE     (1 * 1440)    // unit minute
-#define TSDB_MAX_DAYS_PER_FILE     3650
+#define TSDB_MAX_DAYS_PER_FILE     (3650 * 1440)
-#define TSDB_DEFAULT_DAYS_PER_FILE 10
+#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
 
-#define TSDB_MIN_KEEP     1       // data in db to be reserved.
+#define TSDB_MIN_KEEP     (1 * 1440)       // data in db to be reserved. unit minute
-#define TSDB_MAX_KEEP     365000  // data in db to be reserved.
+#define TSDB_MAX_KEEP     (365000 * 1440) // data in db to be reserved.
-#define TSDB_DEFAULT_KEEP 3650    // ten years
+#define TSDB_DEFAULT_KEEP (3650 * 1440)    // ten years
 
 #define TSDB_MIN_MIN_ROW_FBLOCK     10
 #define TSDB_MAX_MIN_ROW_FBLOCK     1000
@@ -327,7 +328,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_FSYNC_PERIOD     180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD 3000    // three second
 
-#define TSDB_MIN_WAL_LEVEL     0
+#define TSDB_MIN_WAL_LEVEL     1
 #define TSDB_MAX_WAL_LEVEL     2
 #define TSDB_DEFAULT_WAL_LEVEL 1
 
@@ -388,6 +389,7 @@ typedef enum ELogicConditionType {
 #define TSDB_DEFAULT_EXPLAIN_RATIO   0.001
 
 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
 
 #define TSDB_MAX_JOIN_TABLE_NUM 10
 #define TSDB_MAX_UNION_CLAUSE   5
@@ -479,6 +481,9 @@ enum {
 #define QND_VGID 1
 #define VND_VGID 0
 
+#define MAX_NUM_STR_SIZE 40
+
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/tmq.c

@@ -78,6 +78,7 @@ struct tmq_t {
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
+  int8_t         epStatus;
   int32_t        waitingRequest;
   int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
@@ -311,6 +312,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -833,7 +835,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
     tscWarn("msg discard, code:%x", code);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
 
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
@@ -873,7 +875,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
   tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
   if (pRsp == NULL) {
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
   tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
@@ -882,11 +884,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
+#if 0
   if (pRsp->msg.numOfTopics == 0) {
     /*printf("no data\n");*/
     taosFreeQitem(pRsp);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
+#endif
 
   tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
            pRsp->msg.rspOffset);
@@ -897,7 +901,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tsem_post(&tmq->rspSem);
   return 0;
 
-WRITE_QUEUE_FAIL:
+CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
@@ -938,7 +942,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
         for (int32_t k = 0; k < vgNumCur; k++) {
           SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
           sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
-          /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/
+          tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);
           taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
         }
         break;
@@ -952,12 +956,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
       int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
       int64_t  offset = pVgEp->offset;
-      /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);
       if (pOffset != NULL) {
         offset = *pOffset;
-        /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/
+        tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);
       }
-      /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);
       SMqClientVg clientVg = {
           .pollCnt = 0,
           .currentOffset = offset,
@@ -1018,6 +1022,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 
 END:
+  atomic_store_8(&tmq->epStatus, 0);
   if (pParam->sync) {
     tsem_post(&pParam->rspSem);
   }
@@ -1025,6 +1030,10 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  if (epStatus == 1) {
+    return 0;
+  }
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
   SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
@@ -1205,7 +1214,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
-        /*printf("skip vg %d\n", pVg->vgId);*/
+        tscDebug("skip vg %d", pVg->vgId);
         continue;
       }
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
@@ -1249,7 +1258,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/
+      tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1260,13 +1269,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 }
 
 // return
-int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
+int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
   if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
       SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
       tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
-      tmqClearUnhandleMsg(tmq);
+      /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
     } else {
       *pReset = false;
@@ -1297,6 +1306,11 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = rspMsg->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        if (rspMsg->msg.numOfTopics == 0) {
+          taosFreeQitem(rspMsg);
+          rspHead = NULL;
+          continue;
+        }
         return rspMsg;
       } else {
         /*printf("epoch mismatch\n");*/
@@ -1305,10 +1319,10 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
-      tmqHandleRes(tmq, rspHead, &reset);
+      tmqHandleNoPollRsp(tmq, rspHead, &reset);
       taosFreeQitem(rspHead);
       if (pollIfReset && reset) {
-        printf("reset and repoll\n");
+        tscDebug("reset and repoll\n");
         tmqPollImpl(tmq, blockingTime);
       }
     }

source/common/src/tdataformat.c

@@ -85,6 +85,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) {
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     tlen += taosEncodeFixedI8(buf, colType(pCol));
+    tlen += taosEncodeFixedI8(buf, colSma(pCol));
     tlen += taosEncodeFixedI16(buf, colColId(pCol));
     tlen += taosEncodeFixedI16(buf, colBytes(pCol));
   }
@@ -107,12 +108,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
+    int8_t      sma = TSDB_BSMA_TYPE_NONE;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);
+    buf = taosDecodeFixedI8(buf, &sma);
     buf = taosDecodeFixedI16(buf, &colId);
     buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
+    if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;
     }
@@ -148,7 +151,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->version = version;
 }
 
-int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) {
+int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) {
   if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
@@ -161,6 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
   colSetType(pCol, type);
   colSetColId(pCol, colId);
+  colSetSma(pCol, sma);
   if (pBuilder->nCols == 0) {
     colSetOffset(pCol, 0);
   } else {
@@ -168,9 +172,6 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
     colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
   }
 
-  // TODO: set sma value by user input
-  pCol->sma = 1;
-
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);

source/common/src/tmsg.c

@@ -296,24 +296,22 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
@@ -330,17 +328,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
       tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
         tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
@@ -370,31 +366,24 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
-      buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
-      pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
+        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
         buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
         buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols);
+      buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      if (pReq->stbCfg.nBSmaCols > 0) {
-        pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->stbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
@@ -418,23 +407,16 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols);
+      buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
-      pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
+      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
+        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
         buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      if (pReq->ntbCfg.nBSmaCols > 0) {
-        pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->ntbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
@@ -1629,6 +1611,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1650,6 +1633,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);

source/common/src/ttime.c

@@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+char getPrecisionUnit(int32_t precision) {
+  static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
+  switch (precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+    case TSDB_TIME_PRECISION_MICRO:
+    case TSDB_TIME_PRECISION_NANO:
+      return units[precision];
+    default:
+      return 0;
+  }
+}
+
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
@@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec
   return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
 }
 
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
+  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+         fromPrecision == TSDB_TIME_PRECISION_NANO);
+  static double factors[3] = {1000000., 1000., 1.};
+  switch (toUnit) {
+    case 's':
+      return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
+    case 'm':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
+    case 'h':
+      return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
+    case 'd':
+      return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
+    case 'w':
+      return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
+    case 'a':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
+    case 'u':
+      return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
+    case 'b':
+      return time * factors[fromPrecision];
+    default: {
+      return -1;
+    }
+  }  
+}
+
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
   switch (unit) {
     case 's':

source/common/src/ttypes.c

@@ -431,7 +431,7 @@ FORCE_INLINE void *getDataMax(int32_t type) {
   }
 }
 
-bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; }
+bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; }
 
 void setVardataNull(void *val, int32_t type) {
   if (type == TSDB_DATA_TYPE_BINARY) {

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -124,37 +124,37 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "tag3");
@@ -164,7 +164,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);
@@ -196,37 +196,37 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "_tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "_tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "_tag3");
@@ -236,7 +236,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);

source/dnode/mgmt/vm/src/vmWorker.c

@@ -187,7 +187,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
-    return -1;
+    return terrno;
   }
 
   int32_t code = 0;

source/dnode/mnode/impl/inc/mndDef.h

@@ -620,13 +620,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
 
 static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
   if (pSub->consumers) {
-    taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
+    //taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
     // taosArrayDestroy(pSub->consumers);
     pSub->consumers = NULL;
   }
 
   if (pSub->unassignedVg) {
-    taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
+    //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
     // taosArrayDestroy(pSub->unassignedVg);
     pSub->unassignedVg = NULL;
   }

source/dnode/mnode/impl/src/mndConsumer.c

@@ -160,6 +160,7 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
+  pOldConsumer->epoch++;
 
   // TODO handle update
   /*taosWLockLatch(&pOldConsumer->lock);*/

source/dnode/mnode/impl/src/mndDb.c

@@ -1309,7 +1309,7 @@ static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMet
   cols++;
 
   pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "days");
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
@@ -1444,7 +1444,7 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
-  *(int16_t *)pWrite = pDb->cfg.daysPerFile;
+  *(int32_t *)pWrite = pDb->cfg.daysPerFile;
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -53,7 +53,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},

source/dnode/mnode/impl/src/mndStb.c

@@ -333,6 +333,15 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
   return mndAcquireDb(pMnode, db);
 }
 
+static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) {
+  if (*(col_id_t *)colId < ((SSchemaEx *)pSchema)->colId) {
+    return -1;
+  } else if (*(col_id_t *)colId > ((SSchemaEx *)pSchema)->colId) {
+    return 1;
+  }
+  return 0;
+}
+
 static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
   SName name = {0};
   tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -348,13 +357,58 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.type = TD_SUPER_TABLE;
   req.stbCfg.suid = pStb->uid;
   req.stbCfg.nCols = pStb->numOfColumns;
-  req.stbCfg.pSchema = pStb->pColumns;
   req.stbCfg.nTagCols = pStb->numOfTags;
   req.stbCfg.pTagSchema = pStb->pTags;
+  req.stbCfg.nBSmaCols = pStb->numOfSmas;
+  req.stbCfg.pSchema = (SSchemaEx *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaEx));
+  if (req.stbCfg.pSchema == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   
-  int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
+  int bSmaStat = 0;                             // no column has bsma
+  if (pStb->numOfSmas == pStb->numOfColumns) {  // assume pColumns > 0
+    bSmaStat = 1;                               // all columns have bsma
+  } else if (pStb->numOfSmas != 0) {
+    bSmaStat = 2;                  // partial columns have bsma
+    TASSERT(pStb->pSmas != NULL);  // TODO: remove the assert
+  }
+
+  for (int32_t i = 0; i < req.stbCfg.nCols; ++i) {
+    SSchemaEx *pSchemaEx = req.stbCfg.pSchema + i;
+    SSchema   *pSchema = pStb->pColumns + i;
+    pSchemaEx->type = pSchema->type;
+    pSchemaEx->sma = (bSmaStat == 1) ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE;
+    pSchemaEx->colId = pSchema->colId;
+    pSchemaEx->bytes = pSchema->bytes;
+    memcpy(pSchemaEx->name, pSchema->name, TSDB_COL_NAME_LEN);
+  }
+
+  if (bSmaStat == 2) {
+    if (pStb->pSmas == NULL) {
+      mError("stb:%s, sma options is empty", pStb->name);
+      taosMemoryFreeClear(req.stbCfg.pSchema);
+      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
+      return NULL;
+    }
+    for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
+      SSchema   *pSmaSchema = pStb->pSmas + i;
+      SSchemaEx *pColSchema = taosbsearch(&pSmaSchema->colId, req.stbCfg.pSchema, req.stbCfg.nCols, sizeof(SSchemaEx),
+                                          schemaExColIdCompare, TD_EQ);
+      if (pColSchema == NULL) {
+        terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
+        taosMemoryFreeClear(req.stbCfg.pSchema);
+        mError("stb:%s, sma col:%s not found in columns", pStb->name, pSmaSchema->name);
+        return NULL;
+      }
+      pColSchema->sma = TSDB_BSMA_TYPE_LATEST;
+    }
+  }
+
+  int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
+    taosMemoryFreeClear(req.stbCfg.pSchema);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
@@ -366,6 +420,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   tSerializeSVCreateTbReq(&pBuf, &req);
 
   *pContLen = contLen;
+  taosMemoryFreeClear(req.stbCfg.pSchema);
   return pHead;
 }
 
@@ -498,7 +553,6 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
 
@@ -559,9 +613,9 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 }
 
 static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) {
-  for (int32_t col = 0; col < pStb->numOfColumns; col++) {
+  for (int32_t col = 0; col < pStb->numOfColumns; ++col) {
     SSchema *pSchema = &pStb->pColumns[col];
-    if (strcasecmp(pStb->pColumns[col].name, colName) == 0) {
+    if (strncasecmp(pSchema->name, colName, TSDB_COL_NAME_LEN) == 0) {
       return pSchema;
     }
   }
@@ -625,7 +679,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     SSchema *pSchema = &stbObj.pSmas[i];
     SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
     if (pColSchema == NULL) {
-      mError("stb:%s, sma:%s not found in columns", stbObj.name, pSchema->name);
+      mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
       terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
       return -1;
     }
@@ -1061,7 +1115,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
 

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -237,7 +237,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
       for (int32_t j = 0; j < csz; j++) {
         SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
         if (consumerId == pSubConsumer->consumerId) {
-          int32_t       vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
+          mInfo("topic %s has %d vg", topicName, pConsumer->epoch);
           SMqSubTopicEp topicEp;
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
@@ -419,7 +420,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
       int32_t vgNum = pSub->vgNum;
       int32_t vgEachConsumer = vgNum / consumerNum;
       int32_t imbalanceVg = vgNum % consumerNum;
-      int32_t imbalanceSolved = 0;
 
       // iterate all consumers, set unassignedVgStash
       for (int32_t i = 0; i < consumerNum; i++) {
@@ -446,9 +446,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
         if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
             (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
             (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) {
-          if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {
+          /*if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {*/
-            pRebConsumer->epoch++;
+            /*pRebConsumer->epoch++;*/
-          }
+          /*}*/
           if (vgThisConsumerAfterRb != 0) {
             atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
           } else {
@@ -460,7 +460,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
 
           SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
           sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-          mndTransAppendRedolog(pTrans, pConsumerRaw);
+          mndTransAppendCommitlog(pTrans, pConsumerRaw);
         }
         mndReleaseConsumer(pMnode, pRebConsumer);
       }
@@ -469,7 +469,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
       if (taosArrayGetSize(pSub->unassignedVg) != 0) {
         for (int32_t i = 0; i < consumerNum; i++) {
           SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i);
-          int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
           int32_t         vgThisConsumerAfterRb;
           if (i < imbalanceVg)
             vgThisConsumerAfterRb = vgEachConsumer + 1;
@@ -508,7 +507,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
 
       // TODO: log rebalance statistics
       SSdbRaw *pSubRaw = mndSubActionEncode(pSub);
-      sdbSetRawStatus(pSubRaw, SDB_STATUS_READY);
+      sdbSetRawStatus(pSubRaw, SDB_STATUS_UPDATING);
       mndTransAppendRedolog(pTrans, pSubRaw);
     }
     mndReleaseSubscribe(pMnode, pSub);

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -42,10 +42,10 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
+  createReq.daysPerFile = 10 * 1440;
-  createReq.daysToKeep0 = 3650;
+  createReq.daysToKeep0 = 3650 * 1440;
-  createReq.daysToKeep1 = 3650;
+  createReq.daysToKeep1 = 3650 * 1440;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -35,10 +35,10 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
+  createReq.daysPerFile = 10 * 1440;
-  createReq.daysToKeep0 = 3650;
+  createReq.daysToKeep0 = 3650 * 1440;
-  createReq.daysToKeep1 = 3650;
+  createReq.daysToKeep1 = 3650 * 1440;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/user/user.cpp

@@ -324,10 +324,10 @@ TEST_F(MndTestUser, 03_Alter_User) {
     createReq.numOfVgroups = 2;
     createReq.cacheBlockSize = 16;
     createReq.totalBlocks = 10;
-    createReq.daysPerFile = 10;
+    createReq.daysPerFile = 10 * 1440;
-    createReq.daysToKeep0 = 3650;
+    createReq.daysToKeep0 = 3650 * 1440;
-    createReq.daysToKeep1 = 3650;
+    createReq.daysToKeep1 = 3650 * 1440;
-    createReq.daysToKeep2 = 3650;
+    createReq.daysToKeep2 = 3650 * 1440;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
     createReq.commitTime = 3600;

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -70,9 +70,12 @@ static void    *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static void     metaClearTbCfg(STbCfg *pTbCfg);
 static int      metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
 static void    *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
+static int      metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW);
+static void    *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx);
 static void     metaDBWLock(SMetaDB *pDB);
 static void     metaDBRLock(SMetaDB *pDB);
 static void     metaDBULock(SMetaDB *pDB);
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx);
 
 #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code))
 
@@ -155,13 +158,13 @@ void metaCloseDB(SMeta *pMeta) {
 }
 
 int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
-  tb_uid_t uid;
+  tb_uid_t   uid;
-  char     buf[512];
+  char       buf[512];
-  char     buf1[512];
+  char       buf1[512];
-  void    *pBuf;
+  void      *pBuf;
-  DBT      key1, value1;
+  DBT        key1, value1;
-  DBT      key2, value2;
+  DBT        key2, value2;
-  SSchema *pSchema = NULL;
+  SSchemaEx *pSchema = NULL;
 
   if (pTbCfg->type == META_SUPER_TABLE) {
     uid = pTbCfg->stbCfg.suid;
@@ -204,8 +207,8 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     key2.data = &schemaKey;
     key2.size = sizeof(schemaKey);
 
-    SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema};
+    SSchemaWrapper sw = {.nCols = ncols, .pSchemaEx = pSchema};
-    metaEncodeSchema(&pBuf, &sw);
+    metaEncodeSchemaEx(&pBuf, &sw);
 
     value2.data = buf1;
     value2.size = POINTER_DISTANCE(pBuf, buf1);
@@ -298,6 +301,8 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
 
   buf = taosDecodeFixedU32(buf, &pSW->nCols);
   pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+
+  int8_t dummy;
   for (int i = 0; i < pSW->nCols; i++) {
     pSchema = pSW->pSchema + i;
     buf = taosDecodeFixedI8(buf, &pSchema->type);
@@ -309,6 +314,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
   return buf;
 }
 
+static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) {
+  int        tlen = 0;
+  SSchemaEx *pSchema;
+
+  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  for (int i = 0; i < pSW->nCols; i++) {
+    pSchema = pSW->pSchemaEx + i;
+    tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI8(buf, pSchema->sma);
+    tlen += taosEncodeFixedI16(buf, pSchema->colId);
+    tlen += taosEncodeFixedI32(buf, pSchema->bytes);
+    tlen += taosEncodeString(buf, pSchema->name);
+  }
+
+  return tlen;
+}
+
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) {
+  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  if (isGetEx) {
+    pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchemaEx *pSchema = pSW->pSchemaEx + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosDecodeFixedI8(buf, &pSchema->sma);
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  } else {
+    pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchema *pSchema = pSW->pSchema + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosSkipFixedLen(buf, sizeof(int8_t));
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  }
+
+  return buf;
+}
+
 static SMetaDB *metaNewDB() {
   SMetaDB *pDB = NULL;
   pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB));
@@ -652,12 +701,16 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
 }
 
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
+}
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
   uint32_t        nCols;
   SSchemaWrapper *pSW = NULL;
   SMetaDB        *pDB = pMeta->pDB;
   int             ret;
   void           *pBuf;
-  SSchema        *pSchema;
+  // SSchema        *pSchema;
   SSchemaKey      schemaKey = {uid, sver, 0};
   DBT             key = {0};
   DBT             value = {0};
@@ -678,7 +731,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   // Decode the schema
   pBuf = value.data;
   pSW = taosMemoryMalloc(sizeof(*pSW));
-  metaDecodeSchema(pBuf, pSW);
+  metaDecodeSchemaEx(pBuf, pSW, isGetEx);
 
   return pSW;
 }
@@ -755,7 +808,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
 STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   STSchemaBuilder sb;
   STSchema       *pTSchema = NULL;
-  SSchema        *pSchema;
+  SSchemaEx      *pSchema;
   SSchemaWrapper *pSW;
   STbCfg         *pTbCfg;
   tb_uid_t        quid;
@@ -767,16 +820,16 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
     quid = uid;
   }
 
-  pSW = metaGetTableSchema(pMeta, quid, sver, true);
+  pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
   if (pSW == NULL) {
     return NULL;
   }
 
   // Rebuild a schema
   tdInitTSchemaBuilder(&sb, 0);
-  for (int32_t i = 0; i < pSW->nCols; i++) {
+  for (int32_t i = 0; i < pSW->nCols; ++i) {
-    pSchema = pSW->pSchema + i;
+    pSchema = pSW->pSchemaEx + i;
-    tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes);
+    tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes);
   }
   pTSchema = tdGetSchemaFromBuilder(&sb);
   tdDestroyTSchemaBuilder(&sb);

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -46,6 +46,10 @@ static int   metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
 static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
 static int   metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
 static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
+static int   metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW);
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx);
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx);
 
 static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) {
   tb_uid_t uid1, uid2;
@@ -228,7 +232,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
       schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema;
     }
     pVal = pBuf = buf;
-    metaEncodeSchema(&pBuf, &schemaWrapper);
+    metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
     ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen);
     if (ret < 0) {
@@ -345,6 +349,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
 }
 
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  return *metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
+}
+
+static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
   void           *pKey;
   void           *pVal;
   int             kLen;
@@ -368,7 +376,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   // decode
   pBuf = pVal;
   pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper));
-  metaDecodeSchema(pBuf, pSchemaWrapper);
+  metaDecodeSchemaEx(pBuf, pSchemaWrapper, isGetEx);
 
   TDB_FREE(pVal);
 
@@ -379,7 +387,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   tb_uid_t        quid;
   SSchemaWrapper *pSW;
   STSchemaBuilder sb;
-  SSchema        *pSchema;
+  SSchemaEx      *pSchema;
   STSchema       *pTSchema;
   STbCfg         *pTbCfg;
 
@@ -390,15 +398,15 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
     quid = uid;
   }
 
-  pSW = metaGetTableSchema(pMeta, quid, sver, true);
+  pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
   if (pSW == NULL) {
     return NULL;
   }
 
   tdInitTSchemaBuilder(&sb, 0);
   for (int i = 0; i < pSW->nCols; i++) {
-    pSchema = pSW->pSchema + i;
+    pSchema = pSW->pSchemaEx + i;
-    tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes);
+    tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes);
   }
   pTSchema = tdGetSchemaFromBuilder(&sb);
   tdDestroyTSchemaBuilder(&sb);
@@ -605,6 +613,50 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) {
   return buf;
 }
 
+static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) {
+  int        tlen = 0;
+  SSchemaEx *pSchema;
+
+  tlen += taosEncodeFixedU32(buf, pSW->nCols);
+  for (int i = 0; i < pSW->nCols; ++i) {
+    pSchema = pSW->pSchemaEx + i;
+    tlen += taosEncodeFixedI8(buf, pSchema->type);
+    tlen += taosEncodeFixedI8(buf, pSchema->sma);
+    tlen += taosEncodeFixedI16(buf, pSchema->colId);
+    tlen += taosEncodeFixedI32(buf, pSchema->bytes);
+    tlen += taosEncodeString(buf, pSchema->name);
+  }
+
+  return tlen;
+}
+
+static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) {
+  buf = taosDecodeFixedU32(buf, &pSW->nCols);
+  if (isGetEx) {
+    pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchemaEx *pSchema = pSW->pSchemaEx + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosDecodeFixedI8(buf, &pSchema->sma);
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  } else {
+    pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
+    for (int i = 0; i < pSW->nCols; i++) {
+      SSchema *pSchema = pSW->pSchema + i;
+      buf = taosDecodeFixedI8(buf, &pSchema->type);
+      buf = taosSkipFixedLen(buf, sizeof(int8_t));
+      buf = taosDecodeFixedI16(buf, &pSchema->colId);
+      buf = taosDecodeFixedI32(buf, &pSchema->bytes);
+      buf = taosDecodeStringTo(buf, pSchema->name);
+    }
+  }
+
+  return buf;
+}
+
 static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
   int tsize = 0;
 

source/dnode/vnode/src/tq/tq.c

@@ -299,8 +299,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // response to user
       break;
     }
-    /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType,
+    /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/
-     * pReq->epoch);*/
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -353,9 +352,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = tlen;
       pMsg->code = 0;
-      /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset,
+      /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, pReq->epoch);*/
-       * pHead->msgType,*/
-      /*pReq->epoch);*/
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
       return 0;
@@ -377,6 +374,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
   ((SMqRspHead*)buf)->epoch = pReq->epoch;
+  rsp.rspOffset = fetchOffset - 1;
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
   tEncodeSMqPollRsp(&abuf, &rsp);
@@ -452,7 +450,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
     pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
     ASSERT(pTopic->buffer.output[i].task);
   }
-  printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId);
+  /*printf("set topic %s to consumer %ld on vg %d\n", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);*/
   taosArrayPush(pConsumer->topics, pTopic);
   tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
   tqHandleCommit(pTq->tqMeta, req.consumerId);

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -81,7 +81,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       // TODO: maybe need to clear the request struct
       taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
       taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
-      taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols);
       taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
       taosMemoryFree(vCreateTbReq.dbFName);
       taosMemoryFree(vCreateTbReq.name);
@@ -116,13 +115,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
         if (pCreateTbReq->type == TD_SUPER_TABLE) {
           taosMemoryFree(pCreateTbReq->stbCfg.pSchema);
           taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema);
-          taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols);
           taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
         } else if (pCreateTbReq->type == TD_CHILD_TABLE) {
           taosMemoryFree(pCreateTbReq->ctbCfg.pTag);
         } else {
           taosMemoryFree(pCreateTbReq->ntbCfg.pSchema);
-          taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols);
           taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
         }
       }
@@ -150,7 +147,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq);
       taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
       taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema);
-      taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols);
       taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
       taosMemoryFree(vAlterTbReq.dbFName);
       taosMemoryFree(vAlterTbReq.name);

source/libs/catalog/inc/catalogInt.h

@@ -112,7 +112,14 @@ typedef struct SCtgRuntimeStat {
 } SCtgRuntimeStat;
 
 typedef struct SCtgCacheStat {
-
+  uint64_t clusterNum;
+  uint64_t dbNum;
+  uint64_t tblNum;
+  uint64_t stblNum;
+  uint64_t vgHitNum;
+  uint64_t vgMissNum;
+  uint64_t tblHitNum;
+  uint64_t tblMissNum;
 } SCtgCacheStat;
 
 typedef struct SCatalogStat {
@@ -186,7 +193,7 @@ typedef struct SCatalogMgmt {
   bool                  exit;
   SRWLatch              lock;
   SCtgQueue             queue;
-  TdThread             updateThread;  
+  TdThread              updateThread;  
   SHashObj             *pCluster;     //key: clusterId, value: SCatalog*
   SCatalogStat          stat;
   SCatalogCfg           cfg;
@@ -204,8 +211,13 @@ typedef struct SCtgAction {
 #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 
-#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1)
+#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n)
-#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1)
+#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n)
+#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
+
+#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n))
+#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n))
+#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n))
 
 #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
 #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
@@ -291,6 +303,9 @@ typedef struct SCtgAction {
 #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); }  } while (0)
 
 
+extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
+extern void ctgdShowClusterCache(SCatalog* pCtg);
+extern int32_t ctgdShowCacheInfo(void);
 
 #ifdef __cplusplus
 }

source/libs/catalog/src/catalog.c

@@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action);
 int32_t ctgActRemoveStb(SCtgMetaAction *action);
 int32_t ctgActRemoveTbl(SCtgMetaAction *action);
 
+extern SCtgDebug gCTGDebug;
 SCatalogMgmt gCtgMgmt = {0};
-SCtgDebug gCTGDebug = {0};
 SCtgAction gCtgAction[CTG_ACT_MAX] = {{
                             CTG_ACT_UPDATE_VG,
                             "update vgInfo",
@@ -53,182 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{
                           }
 };
 
-int32_t ctgDbgEnableDebug(char *option) {
-  if (0 == strcasecmp(option, "lock")) {
-    gCTGDebug.lockEnable = true;
-    qDebug("lock debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "cache")) {
-    gCTGDebug.cacheEnable = true;
-    qDebug("cache debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "api")) {
-    gCTGDebug.apiEnable = true;
-    qDebug("api debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "meta")) {
-    gCTGDebug.metaEnable = true;
-    qDebug("api debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  qError("invalid debug option:%s", option);
-  
-  return TSDB_CODE_CTG_INTERNAL_ERROR;
-}
-
-int32_t ctgDbgGetStatNum(char *option, void *res) {
-  if (0 == strcasecmp(option, "runtime.qDoneNum")) {
-    *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  qError("invalid stat option:%s", option);
-  
-  return TSDB_CODE_CTG_INTERNAL_ERROR;
-}
-
-int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) {
-  return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
-}
-
-int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) {
-  return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
-}
-
-int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) {
-  int32_t num = 0;
-  for (uint16_t i = 0; i < rent->slotNum; ++i) {
-    SCtgRentSlot *slot = &rent->slots[i];
-    if (NULL == slot->meta) {
-      continue;
-    }
-
-    num += taosArrayGetSize(slot->meta);
-  }
-
-  return num;
-}
-
-int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
-  if (NULL == pCtg || NULL == pCtg->dbCache) {
-    return 0;
-  }
-
-  switch (type) {
-    case CTG_DBG_DB_NUM:
-      return (int32_t)taosHashGetSize(pCtg->dbCache);
-    case CTG_DBG_DB_RENT_NUM:
-      return ctgDbgGetRentNum(&pCtg->dbRent);
-    case CTG_DBG_STB_RENT_NUM:
-      return ctgDbgGetRentNum(&pCtg->stbRent);
-    default:
-      break;
-  }
-
-  SCtgDBCache *dbCache = NULL;
-  int32_t num = 0;
-  void *pIter = taosHashIterate(pCtg->dbCache, NULL);
-  while (pIter) {
-    dbCache = (SCtgDBCache *)pIter;
-    switch (type) {
-      case CTG_DBG_META_NUM:
-        num += ctgDbgGetTbMetaNum(dbCache);
-        break;
-      case CTG_DBG_STB_NUM:
-        num += ctgDbgGetStbNum(dbCache);
-        break;
-      default:
-        ctgError("invalid type:%d", type);
-        break;
-    }
-    pIter = taosHashIterate(pCtg->dbCache, pIter);
-  }
-
-  return num;
-}
-
-void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
-  if (!gCTGDebug.metaEnable) {
-    return;
-  }
-
-  STableComInfo *c = &p->tableInfo;
-
-  if (TSDB_CHILD_TABLE == p->tableType) {
-    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
-    return;
-  } else {
-    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
-     tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
-  }
-
-  int32_t colNum = c->numOfColumns + c->numOfTags;
-  for (int32_t i = 0; i < colNum; ++i) {
-    SSchema *s = &p->schema[i];
-    ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes);
-  }
-}
-
-void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
-  if (NULL == dbHash || !gCTGDebug.cacheEnable) {
-    return;
-  }
-
-  int32_t i = 0;
-  SCtgDBCache *dbCache = NULL;
-  void *pIter = taosHashIterate(dbHash, NULL);
-  while (pIter) {
-    char *dbFName = NULL;
-    size_t len = 0;
-    
-    dbCache = (SCtgDBCache *)pIter;
-
-    dbFName = taosHashGetKey(pIter, &len);
-
-    int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
-    int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
-    int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
-    int32_t hashMethod = -1;
-    int32_t vgNum = 0;
-
-    if (dbCache->vgInfo) {
-      vgVersion = dbCache->vgInfo->vgVersion;
-      hashMethod = dbCache->vgInfo->hashMethod;
-      if (dbCache->vgInfo->vgHash) {
-        vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
-      }
-    }
-    
-    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
-      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
-
-    pIter = taosHashIterate(dbHash, pIter);
-  }
-}
-
-
-
-
-void ctgDbgShowClusterCache(SCatalog* pCtg) {
-  if (!gCTGDebug.cacheEnable || NULL == pCtg) {
-    return;
-  }
-
-  ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
-  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
-    ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
-  
-  ctgDbgShowDBCache(pCtg, pCtg->dbCache);
-}
-
-
 void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
   if (NULL == mgmt->slots) {
     return;
@@ -249,15 +73,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
 void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) {
   CTG_LOCK(CTG_WRITE, &cache->stbLock);
   if (cache->stbCache) {
+    int32_t stblNum = taosHashGetSize(cache->stbCache);  
     taosHashCleanup(cache->stbCache);
     cache->stbCache = NULL;
+    CTG_CACHE_STAT_SUB(stblNum, stblNum);
   }
   CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
 
   CTG_LOCK(CTG_WRITE, &cache->metaLock);
   if (cache->metaCache) {
+    int32_t tblNum = taosHashGetSize(cache->metaCache);
     taosHashCleanup(cache->metaCache);
     cache->metaCache = NULL;
+    CTG_CACHE_STAT_SUB(tblNum, tblNum);
   }
   CTG_UNLOCK(CTG_WRITE, &cache->metaLock);
 }
@@ -293,6 +121,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
   ctgFreeMetaRent(&pCtg->stbRent);
   
   if (pCtg->dbCache) {
+    int32_t dbNum = taosHashGetSize(pCtg->dbCache);
+    
     void *pIter = taosHashIterate(pCtg->dbCache, NULL);
     while (pIter) {
       SCtgDBCache *dbCache = pIter;
@@ -305,6 +135,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
     }  
 
     taosHashCleanup(pCtg->dbCache);
+    
+    CTG_CACHE_STAT_SUB(dbNum, dbNum);
   }
   
   taosMemoryFree(pCtg);
@@ -361,7 +193,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) {
   CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
 
   CTG_QUEUE_ADD();
-  CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum);
+  CTG_RUNTIME_STAT_ADD(qNum, 1);
 
   tsem_post(&gCtgMgmt.queue.reqSem);
 
@@ -620,34 +452,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache)
 
 
 int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) {
+  SCtgDBCache *dbCache = NULL;
+
   if (NULL == pCtg->dbCache) {
-    *pCache = NULL;
+    ctgDebug("empty db cache, dbFName:%s", dbFName);
-    *inCache = false;
+    goto _return;
-    ctgWarn("empty db cache, dbFName:%s", dbFName);
-    return TSDB_CODE_SUCCESS;
   }
 
-  SCtgDBCache *dbCache = NULL;
   ctgAcquireDBCache(pCtg, dbFName, &dbCache);
   if (NULL == dbCache) {  
-    *pCache = NULL;
+    ctgDebug("db %s not in cache", dbFName);
-    *inCache = false;
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
   
   ctgAcquireVgInfo(pCtg, dbCache, inCache);
   if (!(*inCache)) {
-    ctgReleaseDBCache(pCtg, dbCache);
+    ctgDebug("vgInfo of db %s not in cache", dbFName);
-  
+    goto _return;
-    *pCache = NULL;
-    return TSDB_CODE_SUCCESS;
   }
 
   *pCache = dbCache;
   *inCache = true;
 
+  CTG_CACHE_STAT_ADD(vgHitNum, 1);
+
   ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
   
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  if (dbCache) {
+    ctgReleaseDBCache(pCtg, dbCache);
+  }
+
+  *pCache = NULL;
+  *inCache = false;
+
+  CTG_CACHE_STAT_ADD(vgMissNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -763,11 +606,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName,
 }
 
 
-int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) {
+int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) {
   if (NULL == pCtg->dbCache) {
-    *exist = 0;
+    ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
-    ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname);
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
 
   char dbFName[TSDB_DB_FNAME_LEN] = {0};
@@ -782,8 +624,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   SCtgDBCache *dbCache = NULL;
   ctgAcquireDBCache(pCtg, dbFName, &dbCache);
   if (NULL == dbCache) {
-    *exist = 0;
+    ctgDebug("db %s not in cache", pTableName->tname);
-    return TSDB_CODE_SUCCESS;
+    goto _return;
   }
   
   int32_t sz = 0;
@@ -792,13 +634,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
 
   if (NULL == *pTableMeta) {
-    *exist = 0;
     ctgReleaseDBCache(pCtg, dbCache);
     ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
-    return TSDB_CODE_SUCCESS;
+    goto _return;
   }
 
-  *exist = 1;
   if (dbId) {
     *dbId = dbCache->dbId;
   }
@@ -808,6 +648,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   if (tbMeta->tableType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
     ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
+
+    *inCache = true;
+    CTG_CACHE_STAT_ADD(tblHitNum, 1);
+    
     return TSDB_CODE_SUCCESS;
   }
   
@@ -819,8 +663,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
     ctgReleaseDBCache(pCtg, dbCache);
     ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid);
     taosMemoryFreeClear(*pTableMeta);
-    *exist = 0;
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
 
   if ((*stbMeta)->suid != tbMeta->suid) {    
@@ -846,8 +689,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
 
   ctgReleaseDBCache(pCtg, dbCache);
 
+  *inCache = true;
+  CTG_CACHE_STAT_ADD(tblHitNum, 1);
+
   ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
   
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  *inCache = false;
+  CTG_CACHE_STAT_ADD(tblMissNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1378,6 +1231,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
     CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  CTG_CACHE_STAT_ADD(dbNum, 1);
+ 
   SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
   strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
 
@@ -1436,6 +1291,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) {
     CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
   }
 
+  CTG_CACHE_STAT_SUB(dbNum, 1);
+
   ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
   
   return TSDB_CODE_SUCCESS;
@@ -1568,6 +1425,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
         CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
         if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
           ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
+        } else {
+          CTG_CACHE_STAT_SUB(stblNum, 1);
         }
         CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
 
@@ -1594,8 +1453,12 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  if (NULL == orig) {
+    CTG_CACHE_STAT_ADD(tblNum, 1);
+  }
+
   ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
-  ctgDbgShowTableMeta(pCtg, tbName, meta);
+  ctgdShowTableMeta(pCtg, tbName, meta);
 
   if (!isStb) {
     CTG_UNLOCK(CTG_READ, &tbCache->metaLock);  
@@ -1616,6 +1479,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  CTG_CACHE_STAT_ADD(stblNum, 1);
+  
   CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
 
   CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
@@ -1874,7 +1739,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
   }
   
-  int32_t exist = 0;
+  bool inCache = false;
   int32_t code = 0;
   uint64_t dbId = 0;
   uint64_t suid = 0;
@@ -1884,11 +1749,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     CTG_FLAG_SET_INF_DB(flag);
   }
 
-  CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId));
+  CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId));
 
   int32_t tbType = 0;
 
-  if (exist) {
+  if (inCache) {
     if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) {
       goto _return;
     }
@@ -1930,8 +1795,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     SName stbName = *pTableName;
     strcpy(stbName.tname, output->tbName);
     
-    CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL));
+    CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL));
-    if (0 == exist) {
+    if (!inCache) {
       ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
       continue;
     }
@@ -1943,7 +1808,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
 _return:
 
-  if (CTG_TABLE_NOT_EXIST(code) && exist) {
+  if (CTG_TABLE_NOT_EXIST(code) && inCache) {
     char dbFName[TSDB_DB_FNAME_LEN] = {0};
     if (CTG_FLAG_IS_INF_DB(flag)) {
       strcpy(dbFName, pTableName->dbname);
@@ -1962,7 +1827,7 @@ _return:
 
   if (*pTableMeta) {
     ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType);
-    ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
+    ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
   }
 
   CTG_RET(code);
@@ -2075,11 +1940,15 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) {
   CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
   if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) {
     ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
+  } else {
+    CTG_CACHE_STAT_SUB(stblNum, 1);
   }
 
   CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
   if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) {  
     ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
+  } else {
+    CTG_CACHE_STAT_SUB(tblNum, 1);
   }
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
   
@@ -2119,6 +1988,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
     CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
     ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
+  } else {
+    CTG_CACHE_STAT_SUB(tblNum, 1);
   }
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
 
@@ -2140,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) {
   CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
   
   while (true) {
-    tsem_wait(&gCtgMgmt.queue.reqSem);
+    if (tsem_wait(&gCtgMgmt.queue.reqSem)) {
+      qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    }
     
     if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
       tsem_post(&gCtgMgmt.queue.rspSem);
@@ -2161,9 +2034,9 @@ void* ctgUpdateThreadFunc(void* param) {
       tsem_post(&gCtgMgmt.queue.rspSem);
     }
 
-    CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); 
+    CTG_RUNTIME_STAT_ADD(qDoneNum, 1); 
 
-    ctgDbgShowClusterCache(pCtg);
+    ctgdShowClusterCache(pCtg);
   }
 
   CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
@@ -2304,8 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) {
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
-  tsem_init(&gCtgMgmt.queue.reqSem, 0, 0);
+  if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) {
-  tsem_init(&gCtgMgmt.queue.rspSem, 0, 0);
+    qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
+  }
+  
+  if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) {
+    qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
+  }
 
   gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode));
   if (NULL == gCtgMgmt.queue.head) {
@@ -2384,6 +2264,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
 
   *catalogHandle = clusterCtg;
 
+  CTG_CACHE_STAT_ADD(clusterNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 
 _return:
@@ -2403,6 +2285,8 @@ void catalogFreeHandle(SCatalog* pCtg) {
     return;
   }
 
+  CTG_CACHE_STAT_SUB(clusterNum, 1);
+
   uint64_t clusterId = pCtg->clusterId;
   
   ctgFreeHandle(pCtg);
@@ -2417,24 +2301,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
     CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
-  if (NULL == pCtg->dbCache) {
-    *version = CTG_DEFAULT_INVALID_VERSION;
-    ctgInfo("empty db cache, dbFName:%s", dbFName);
-    CTG_API_LEAVE(TSDB_CODE_SUCCESS);
-  }
-
   SCtgDBCache *dbCache = NULL;
-  ctgAcquireDBCache(pCtg, dbFName, &dbCache);
-  if (NULL == dbCache) {
-    *version = CTG_DEFAULT_INVALID_VERSION;
-    CTG_API_LEAVE(TSDB_CODE_SUCCESS);
-  }
-
   bool inCache = false;
-  ctgAcquireVgInfo(pCtg, dbCache, &inCache);
+  int32_t code = 0;
-  if (!inCache) {
-    ctgReleaseDBCache(pCtg, dbCache);
 
+  CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache));
+  if (!inCache) {
     *version = CTG_DEFAULT_INVALID_VERSION;
     CTG_API_LEAVE(TSDB_CODE_SUCCESS);
   }
@@ -2449,6 +2321,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
   ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version);
 
   CTG_API_LEAVE(TSDB_CODE_SUCCESS);
+
+_return:
+
+  CTG_API_LEAVE(code);
 }
 
 int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) {
@@ -2549,11 +2425,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) {
   }
 
   STableMeta *tblMeta = NULL;
-  int32_t exist = 0;
+  bool inCache = false;
   uint64_t dbId = 0;
-  CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId));
+  CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId));
 
-  if (0 == exist) {
+  if (!inCache) {
     ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
     goto _return;
   }
@@ -2851,8 +2727,13 @@ void catalogDestroy(void) {
 
   atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
 
-  tsem_post(&gCtgMgmt.queue.reqSem);
+  if (tsem_post(&gCtgMgmt.queue.reqSem)) {
-  tsem_post(&gCtgMgmt.queue.rspSem);
+    qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+  }
+  
+  if (tsem_post(&gCtgMgmt.queue.rspSem)) {
+    qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+  }
 
   while (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
     taosUsleep(1);

source/libs/catalog/src/catalogDbg.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "trpc.h"
+#include "query.h"
+#include "tname.h"
+#include "catalogInt.h"
+
+extern SCatalogMgmt gCtgMgmt;
+SCtgDebug gCTGDebug = {0};
+
+int32_t ctgdEnableDebug(char *option) {
+  if (0 == strcasecmp(option, "lock")) {
+    gCTGDebug.lockEnable = true;
+    qDebug("lock debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "cache")) {
+    gCTGDebug.cacheEnable = true;
+    qDebug("cache debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "api")) {
+    gCTGDebug.apiEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "meta")) {
+    gCTGDebug.metaEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid debug option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetStatNum(char *option, void *res) {
+  if (0 == strcasecmp(option, "runtime.qDoneNum")) {
+    *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid stat option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+}
+
+int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+}
+
+int32_t ctgdGetRentNum(SCtgRentMgmt *rent) {
+  int32_t num = 0;
+  for (uint16_t i = 0; i < rent->slotNum; ++i) {
+    SCtgRentSlot *slot = &rent->slots[i];
+    if (NULL == slot->meta) {
+      continue;
+    }
+
+    num += taosArrayGetSize(slot->meta);
+  }
+
+  return num;
+}
+
+int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
+  if (NULL == pCtg || NULL == pCtg->dbCache) {
+    return 0;
+  }
+
+  switch (type) {
+    case CTG_DBG_DB_NUM:
+      return (int32_t)taosHashGetSize(pCtg->dbCache);
+    case CTG_DBG_DB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->dbRent);
+    case CTG_DBG_STB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->stbRent);
+    default:
+      break;
+  }
+
+  SCtgDBCache *dbCache = NULL;
+  int32_t num = 0;
+  void *pIter = taosHashIterate(pCtg->dbCache, NULL);
+  while (pIter) {
+    dbCache = (SCtgDBCache *)pIter;
+    switch (type) {
+      case CTG_DBG_META_NUM:
+        num += ctgdGetTbMetaNum(dbCache);
+        break;
+      case CTG_DBG_STB_NUM:
+        num += ctgdGetStbNum(dbCache);
+        break;
+      default:
+        ctgError("invalid type:%d", type);
+        break;
+    }
+    pIter = taosHashIterate(pCtg->dbCache, pIter);
+  }
+
+  return num;
+}
+
+void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
+  if (!gCTGDebug.metaEnable) {
+    return;
+  }
+
+  STableComInfo *c = &p->tableInfo;
+
+  if (TSDB_CHILD_TABLE == p->tableType) {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
+    return;
+  } else {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
+     tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
+  }
+
+  int32_t colNum = c->numOfColumns + c->numOfTags;
+  for (int32_t i = 0; i < colNum; ++i) {
+    SSchema *s = &p->schema[i];
+    ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes);
+  }
+}
+
+void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
+  if (NULL == dbHash || !gCTGDebug.cacheEnable) {
+    return;
+  }
+
+  int32_t i = 0;
+  SCtgDBCache *dbCache = NULL;
+  void *pIter = taosHashIterate(dbHash, NULL);
+  while (pIter) {
+    char *dbFName = NULL;
+    size_t len = 0;
+    
+    dbCache = (SCtgDBCache *)pIter;
+
+    dbFName = taosHashGetKey(pIter, &len);
+
+    int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+    int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+    int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
+    int32_t hashMethod = -1;
+    int32_t vgNum = 0;
+
+    if (dbCache->vgInfo) {
+      vgVersion = dbCache->vgInfo->vgVersion;
+      hashMethod = dbCache->vgInfo->hashMethod;
+      if (dbCache->vgInfo->vgHash) {
+        vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
+      }
+    }
+    
+    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
+      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
+
+    pIter = taosHashIterate(dbHash, pIter);
+  }
+}
+
+
+
+
+void ctgdShowClusterCache(SCatalog* pCtg) {
+  if (!gCTGDebug.cacheEnable || NULL == pCtg) {
+    return;
+  }
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
+  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
+    ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
+  
+  ctgdShowDBCache(pCtg, pCtg->dbCache);
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg);
+}
+
+int32_t ctgdShowCacheInfo(void) {
+  if (!gCTGDebug.cacheEnable) {
+    return TSDB_CODE_CTG_OUT_OF_SERVICE;
+  }
+
+  CTG_API_ENTER();
+  
+  SCatalog *pCtg = NULL;
+  void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+  while (pIter) {
+    pCtg = *(SCatalog **)pIter;
+
+    if (pCtg) {
+      ctgdShowClusterCache(pCtg);
+    }
+    
+    pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
+  }
+
+  CTG_API_LEAVE(TSDB_CODE_SUCCESS);
+}
+

source/libs/catalog/test/catalogTests.cpp

@@ -38,11 +38,11 @@
 namespace {
 
 extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
-                                            int32_t *exist, int32_t flag, uint64_t *dbId);
+                                            bool *inCache, int32_t flag, uint64_t *dbId);
-extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
+extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
 extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action);
-extern "C" int32_t ctgDbgEnableDebug(char *option);
+extern "C" int32_t ctgdEnableDebug(char *option);
-extern "C" int32_t ctgDbgGetStatNum(char *option, void *res);
+extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 
 void ctgTestSetRspTableMeta();
 void ctgTestSetRspCTableMeta();
@@ -140,9 +140,9 @@ void ctgTestInitLogFile() {
   qDebugFlag = 159;
   strcpy(tsLogDir, "/var/log/taos");
 
-  ctgDbgEnableDebug("api");
+  ctgdEnableDebug("api");
-  ctgDbgEnableDebug("meta");
+  ctgdEnableDebug("meta");
-  ctgDbgEnableDebug("cache");
+  ctgdEnableDebug("cache");
   
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) {
   int32_t          code = 0;
   int32_t          n = 0;
   STableMeta      *tbMeta = NULL;
-  int32_t          exist = 0;
+  bool             inCache = false;
 
   SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(cn.dbname, "db1");
   strcpy(cn.tname, ctgTestCTablename);
 
   while (!ctgTestStop) {
-    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL);
+    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL);
-    if (code || 0 == exist) {
+    if (code || !inCache) {
       assert(0);
     }
 
@@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
     taosMsleep(50);
   }
   
@@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (2 != n) {
       taosMsleep(50);
     } else {
@@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(code, 0);
 
   while (true) {
-    int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
-    int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
+    int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
     if (n || m) {
       taosMsleep(50);
     } else {
@@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) {
   }
 
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) {
     }
   }
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
 
   code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
   ASSERT_EQ(code, 0);
@@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) {
     ASSERT_EQ(tableMeta->tableInfo.precision, 1);
     ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
-    while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
+    while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
       taosMsleep(50);
     }
 

source/libs/command/inc/commandInt.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_QUERY_INT_H_
+#define _TD_QUERY_INT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "nodes.h"
+#include "plannodes.h"
+#include "ttime.h"
+
+#define EXPLAIN_MAX_GROUP_NUM 100
+
+//newline area
+#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d"
+#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d"
+#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d"
+#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d"
+#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d"
+#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d"
+#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d"
+#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d"
+#define EXPLAIN_ORDER_FORMAT "Order: %s"
+#define EXPLAIN_FILTER_FORMAT "Filter: "
+#define EXPLAIN_FILL_FORMAT "Fill: %s"
+#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
+#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
+
+//append area
+#define EXPLAIN_GROUPS_FORMAT " groups=%d"
+#define EXPLAIN_WIDTH_FORMAT " width=%d"
+#define EXPLAIN_LOOPS_FORMAT " loops=%d"
+#define EXPLAIN_REVERSE_FORMAT " reverse=%d"
+
+typedef struct SExplainGroup {
+  int32_t   nodeNum;
+  SSubplan *plan;
+  void     *execInfo;  //TODO
+} SExplainGroup;
+
+typedef struct SExplainResNode {
+  SNodeList*  pChildren;
+  SPhysiNode* pNode;
+  void*       pExecInfo;
+} SExplainResNode;
+
+typedef struct SQueryExplainRowInfo {
+  int32_t level;
+  int32_t len;
+  char   *buf;
+} SQueryExplainRowInfo;
+
+typedef struct SExplainCtx {
+  int32_t   totalSize;
+  bool      verbose;
+  char     *tbuf;
+  SArray   *rows;
+  SHashObj *groupHash;
+} SExplainCtx;
+
+#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending")
+#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
+
+#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u)))
+
+#define EXPLAIN_ROW_NEW(level, ...)                                                                               \
+  do {                                                                                                            \
+    if (isVerboseLine) {                                                                                          \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, "");       \
+    } else {                                                                                                      \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+    }                                                                                                             \
+    tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__);         \
+  } while (0)
+  
+#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)
+#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_QUERY_INT_H_*/

source/libs/command/src/explain.c

@@ -0,0 +1,687 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "query.h"
+#include "plannodes.h"
+#include "commandInt.h"
+
+int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes);
+int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level);
+
+
+void qFreeExplainResTree(SExplainResNode *res) {
+  if (NULL == res) {
+    return;
+  }
+
+  taosMemoryFreeClear(res->pExecInfo);
+
+  SNode* node = NULL;
+  FOREACH(node, res->pChildren) {
+    qFreeExplainResTree((SExplainResNode *)node);
+  }  
+  nodesClearList(res->pChildren);
+  
+  taosMemoryFreeClear(res);
+}
+
+void qFreeExplainCtx(void *ctx) {
+  if (NULL == ctx) {
+    return;
+  }
+
+  SExplainCtx *pCtx = (SExplainCtx *)ctx;
+  int32_t rowSize = taosArrayGetSize(pCtx->rows);
+  for (int32_t i = 0; i < rowSize; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    taosMemoryFreeClear(row->buf);
+  }
+
+  taosHashCleanup(pCtx->groupHash);
+  taosArrayDestroy(pCtx->rows);
+  taosMemoryFree(pCtx);
+}
+
+int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) {
+  int32_t code = 0;
+  SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx));
+  if (NULL == ctx) {
+    qError("calloc SExplainCtx failed");
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+  
+  SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo));
+  if (NULL == rows) {
+    qError("taosArrayInit SQueryExplainRowInfo failed");
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE);
+  if (NULL == tbuf) {
+    qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE);
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  ctx->verbose = verbose;
+  ctx->tbuf = tbuf;
+  ctx->rows = rows;
+  ctx->groupHash = groupHash;
+  
+  *pCtx = ctx;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  taosArrayDestroy(rows);
+  taosHashCleanup(groupHash);
+  taosMemoryFree(ctx);
+
+  QRY_RET(code);
+}
+
+
+char *qFillModeString(EFillMode mode) {
+  switch (mode) {
+    case FILL_MODE_NONE:
+      return "none";
+    case FILL_MODE_VALUE:
+      return "value";
+    case FILL_MODE_PREV:
+      return "prev";
+    case FILL_MODE_NULL:
+      return "null";
+    case FILL_MODE_LINEAR:
+      return "linear";
+    case FILL_MODE_NEXT:
+      return "next";
+    default:
+      return "unknown";
+  }
+}
+
+char *qGetNameFromColumnNode(SNode *pNode) {
+  if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) {
+    return "NULL";
+  }
+  
+  return ((SColumnNode *)pNode)->colName;
+}
+
+int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) {
+  int32_t tlen = 0;
+  SNodeList *pPhysiChildren = NULL;
+  
+  switch (pNode->type) {
+    case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
+      STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
+      pPhysiChildren = pTagScanNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
+      STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
+      pPhysiChildren = pTblScanNode->scan.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
+      SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
+      pPhysiChildren = pSTblScanNode->scan.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
+      SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
+      pPhysiChildren = pPrjNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
+      SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
+      pPhysiChildren = pJoinNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_AGG:{
+      SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
+      pPhysiChildren = pAggNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
+      SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
+      pPhysiChildren = pExchNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:{
+      SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
+      pPhysiChildren = pSortNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
+      SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
+      pPhysiChildren = pIntNode->window.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
+      SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode;
+      pPhysiChildren = pSessNode->window.node.pChildren;
+      break;
+    }
+    default:
+      qError("not supported physical node type %d", pNode->type);
+      QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (pPhysiChildren) {
+    *pChildren = nodesMakeList();
+    if (NULL == *pChildren) {
+      qError("nodesMakeList failed");
+      QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+  }
+
+  SNode* node = NULL;
+  SExplainResNode *pResNode = NULL;
+  FOREACH(node, pPhysiChildren) {
+    QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode));
+    QRY_ERR_RET(nodesListAppend(*pChildren, pResNode));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) {
+  if (NULL == pNode) {
+    *pRes = NULL;
+    qError("physical node is NULL");
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode));
+  if (NULL == res) {
+    qError("calloc SPhysiNodeExplainRes failed");
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+
+  int32_t code = 0;
+  res->pNode = pNode;
+  res->pExecInfo = pExecInfo;
+  QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren));
+  
+  *pRes = res;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  qFreeExplainResTree(res);
+  
+  QRY_RET(code);
+}
+
+int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) {
+  int32_t tlen = *len;
+  
+  EXPLAIN_ROW_APPEND("(exec info here)");
+  
+  *len = tlen;
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) {
+  SQueryExplainRowInfo row = {0};
+  row.buf = taosMemoryMalloc(len);
+  if (NULL == row.buf) {
+    qError("taosMemoryMalloc %d failed", len);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  memcpy(row.buf, tbuf, len);
+  row.level = level;
+  row.len = len;
+  ctx->totalSize += len;
+
+  if (NULL == taosArrayPush(ctx->rows, &row)) {
+    qError("taosArrayPush row to explain res rows failed");
+    taosMemoryFree(row.buf);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
+  int32_t tlen = 0;
+  bool isVerboseLine = false;
+  char *tbuf = ctx->tbuf;
+  bool verbose = ctx->verbose;
+  SPhysiNode* pNode = pResNode->pNode;
+  if (NULL == pNode) {
+    qError("pyhsical node in explain res node is NULL");
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+  
+  switch (pNode->type) {
+    case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
+      STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count);
+      if (pTagScanNode->reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
+      STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count);
+      if (pTblScanNode->scan.reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {      
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+        if (pTblScanNode->scan.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
+      SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count);
+      if (pSTblScanNode->scan.reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {            
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));   
+
+        if (pSTblScanNode->scan.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+        
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
+      SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pPrjNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
+      SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pJoinNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+        
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);      
+        QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_AGG:{
+      SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length);
+      if (pAggNode->pGroupKeys) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length);
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize);
+      
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pAggNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
+      SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
+      SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId));
+      if (NULL == group) {
+        qError("exchange src group %d not in groupHash", pExchNode->srcGroupId);
+        QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pExchNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+
+      QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1));
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:{
+      SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pSortNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
+      SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, 
+        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, 
+        pIntNode->offset, getPrecisionUnit(pIntNode->precision), 
+        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, 
+        pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pIntNode->pFill) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode));      
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+
+        if (pIntNode->window.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
+      SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pIntNode->window.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    default:
+      qError("not supported physical node type %d", pNode->type);
+      return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
+  if (NULL == pResNode) {
+    qError("explain res node is NULL");
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  int32_t code = 0;
+  QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level));
+
+  SNode* pNode = NULL;
+  FOREACH(pNode, pResNode->pChildren) {
+    QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) {
+  SExplainResNode *node = NULL;
+  int32_t code = 0;
+  SExplainCtx *ctx = (SExplainCtx *)pCtx;
+
+  SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId));
+  if (NULL == group) {
+    qError("group %d not in groupHash", groupId);
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node));
+
+  QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level));
+
+_return:
+
+  qFreeExplainResTree(node);
+  
+  QRY_RET(code);
+}
+
+
+int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
+  SExplainCtx *pCtx = (SExplainCtx *)ctx;
+  int32_t rowNum = taosArrayGetSize(pCtx->rows);
+  if (rowNum <= 0) {
+    qError("empty explain res rows");
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  int32_t colNum = 1;
+  int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize;
+  SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
+  if (NULL == rsp) {
+    qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  rsp->completed = 1;
+  rsp->numOfRows = htonl(rowNum);
+
+  *(int32_t *)rsp->data = htonl(pCtx->totalSize);
+
+  int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t));
+  char *data = (char *)(offset + rowNum);
+  int32_t tOffset = 0;
+  
+  for (int32_t i = 0; i < rowNum; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    *offset = tOffset;
+    tOffset += row->len;
+
+    memcpy(data, row->buf, row->len);
+    
+    ++offset;
+    data += row->len;
+  }
+
+  *pRsp = rsp;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
+  int32_t code = 0;
+  SNodeListNode *plans = NULL;
+  int32_t        taskNum = 0;
+  SExplainGroup *pGroup = NULL;
+  void *pCtx = NULL;
+  int32_t rootGroupId = 0;
+
+  if (pDag->numOfSubplans <= 0) {
+    qError("invalid subplan num:%d", pDag->numOfSubplans);
+    QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans);
+  if (levelNum <= 0) {
+    qError("invalid level num:%d", levelNum);
+    QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  if (NULL == groupHash) {
+    qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose));
+
+  for (int32_t i = 0; i < levelNum; ++i) {
+    plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
+    if (NULL == plans) {
+      qError("empty level plan, level:%d", i);
+      QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+
+    taskNum = (int32_t)LIST_LENGTH(plans->pNodeList);
+    if (taskNum <= 0) {
+      qError("invalid level plan number:%d, level:%d", taskNum, i);
+      QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+
+    SSubplan *plan = NULL;
+    for (int32_t n = 0; n < taskNum; ++n) {
+      plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
+      pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId));
+      if (pGroup) {
+        ++pGroup->nodeNum;
+        continue;
+      }
+
+      SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL};
+      if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) {
+        qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n);
+        QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+      }
+    }
+
+    if (0 == i) {
+      if (taskNum > 1) {
+        qError("invalid taskNum %d for level 0", taskNum);
+        QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+      }
+
+      rootGroupId = plan->id.groupId;
+    }
+
+    qDebug("level %d group handled, taskNum:%d", i, taskNum);
+  }
+
+  QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0));
+  
+  QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp));
+  
+_return:
+
+  qFreeExplainCtx(pCtx);
+
+  QRY_RET(code);
+}
+
+
+

source/libs/executor/inc/executorimpl.h

@@ -362,6 +362,7 @@ typedef struct SSourceDataInfo {
   int32_t               index;
   SRetrieveTableRsp    *pRsp;
   uint64_t              totalRows;
+  int32_t               code;
   EX_SOURCE_STATUS      status;
 } SSourceDataInfo;
 

source/libs/executor/src/executorimpl.c

@@ -844,8 +844,7 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowI
   return TSDB_CODE_SUCCESS;
 }
 
-static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, SqlFunctionCtx* pCtx,
+static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
-                                            int32_t numOfOutput, int32_t* rowCellInfoOffset);
 
 static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan,
                                           SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
@@ -863,7 +862,7 @@ static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_
   // set time window for current result
   pResultRow->win = (*win);
   *pResult = pResultRow;
-  setResultRowOutputBufInitCtx_rv(pAggSup->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1046,7 +1045,9 @@ static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin) {
 static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol,
                              int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
   SScalarParam intervalParam = {.numOfRows = 5, .columnData = pTimeWindowData};  //TODO move out of this function
-  updateTimeWindowInfo(pTimeWindowData, pWin);
+  if (pTimeWindowData != NULL) {
+    updateTimeWindowInfo(pTimeWindowData, pWin);
+  }
 
   for (int32_t k = 0; k < numOfOutput; ++k) {
     pCtx[k].startTs = pWin->skey;
@@ -1894,9 +1895,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals);
-    int32_t ret =
+    int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
-        setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len,
-                                   0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
     if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
     }
@@ -2018,12 +2017,11 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo
   SqlFunctionCtx* pCtx = binfo->pCtx;
 
   SResultRow* pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId,
-                                                     pTaskInfo, true, pAggSup);
+                                                     pTaskInfo, false, pAggSup);
   assert(pResultRow != NULL);
 
   setResultRowKey(pResultRow, pData, type);
-
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset);
-  setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2178,8 +2176,8 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num
         }
       }
       pCtx->resDataInfo.interBufSize = env.calcMemSize;
-    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN) {
+    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) {
-    } else if (pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) {
+      pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element.
     }
 
     pCtx->input.numOfInputCols = pFunct->numOfParams;
@@ -3775,8 +3773,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pRes
   }
 }
 
-void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, SqlFunctionCtx* pCtx,
+void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) {
-                                     int32_t numOfOutput, int32_t* rowCellInfoOffset) {
   for (int32_t i = 0; i < numOfOutput; ++i) {
     pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
 
@@ -3789,18 +3786,13 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, S
       continue;
     }
 
-    //    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
-    //      if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput;
-    //    }
-
     if (!pResInfo->initialized && pCtx[i].functionId != -1) {
       pCtx[i].fpSet.init(&pCtx[i], pResInfo);
     }
   }
 }
 
-void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId,
+void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) {
-                              SExecTaskInfo* pTaskInfo) {
   // for simple group by query without interval, all the tables belong to one group result.
   int64_t uid = 0;
   int64_t tid = 0;
@@ -3819,14 +3811,13 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i
    * all group belong to one result set, and each group result has different group id so set the id to be one
    */
   if (pResultRow->pageId == -1) {
-    int32_t ret =
+    int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize);
-        addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize);
     if (ret != TSDB_CODE_SUCCESS) {
       return;
     }
   }
 
-  setResultRowOutputBufInitCtx_rv(pAggInfo->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
 }
 
 void setExecutionContext(int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey, SExecTaskInfo* pTaskInfo,
@@ -5013,12 +5004,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
 
 int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*)param;
-  pSourceDataInfo->pRsp = pMsg->pData;
+  if (code == TSDB_CODE_SUCCESS) {
+    pSourceDataInfo->pRsp = pMsg->pData;
 
-  SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
+    SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
-  pRsp->numOfRows = htonl(pRsp->numOfRows);
+    pRsp->numOfRows = htonl(pRsp->numOfRows);
-  pRsp->useconds = htobe64(pRsp->useconds);
+    pRsp->compLen   = htonl(pRsp->compLen);
-  pRsp->compLen = htonl(pRsp->compLen);
+    pRsp->useconds  = htobe64(pRsp->useconds);
+  } else {
+    pSourceDataInfo->code = code;
+  }
 
   pSourceDataInfo->status = EX_SOURCE_DATA_READY;
   tsem_post(&pSourceDataInfo->pEx->ready);
@@ -5274,7 +5269,6 @@ static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo* pOperator) {
          totalSources, endTs - startTs);
 
   tsem_wait(&pExchangeInfo->ready);
-
   pOperator->status = OP_RES_TO_RETURN;
   return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
 }
@@ -5318,18 +5312,22 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     }
 
     doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
-
     tsem_wait(&pExchangeInfo->ready);
 
-    SSourceDataInfo*       pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current);
+    SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current);
     SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
 
+    if (pDataInfo->code != TSDB_CODE_SUCCESS) {
+      qError("%s vgId:%d, taskID:0x%" PRIx64 " error happens, code:%s",
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, tstrerror(pDataInfo->code));
+      pOperator->pTaskInfo->code = pDataInfo->code;
+      return NULL;
+    }
+
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
-
     if (pRsp->numOfRows == 0) {
-      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
+      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next",
-             " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 

source/libs/nodes/inc/nodesUtil.h

@@ -20,12 +20,16 @@
 extern "C" {
 #endif
 
-#define nodesFatal(param, ...)  qFatal("NODES: " param, __VA_ARGS__)
+#define nodesFatal(...)  qFatal("NODES: " __VA_ARGS__)
-#define nodesError(param, ...)  qError("NODES: " param, __VA_ARGS__)
+#define nodesError(...)  qError("NODES: " __VA_ARGS__)
-#define nodesWarn(param, ...)   qWarn("NODES: " param, __VA_ARGS__)
+#define nodesWarn(...)   qWarn("NODES: " __VA_ARGS__)
-#define nodesInfo(param, ...)   qInfo("NODES: " param, __VA_ARGS__)
+#define nodesInfo(...)   qInfo("NODES: " __VA_ARGS__)
-#define nodesDebug(param, ...)  qDebug("NODES: " param, __VA_ARGS__)
+#define nodesDebug(...)  qDebug("NODES: " __VA_ARGS__)
-#define nodesTrace(param, ...)  qTrace("NODES: " param, __VA_ARGS__)
+#define nodesTrace(...)  qTrace("NODES: " __VA_ARGS__)
+
+#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
 #ifdef __cplusplus
 }

source/libs/nodes/src/nodesCloneFuncs.c

@@ -206,6 +206,12 @@ static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode
   return (SNode*)pDst;
 }
 
+static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
+  CLONE_NODE_LIST_FIELD(pNodeList);
+  return (SNode*)pDst;
+}
+
 static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
   COPY_SCALAR_FIELD(mode);
   CLONE_NODE_FIELD(pValues);
@@ -360,6 +366,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return orderByExprNodeCopy((const SOrderByExprNode*)pNode, (SOrderByExprNode*)pDst);
     case QUERY_NODE_LIMIT:
       break;
+    case QUERY_NODE_NODE_LIST:
+      return nodeListNodeCopy((const SNodeListNode*)pNode, (SNodeListNode*)pDst);
     case QUERY_NODE_FILL:
       return fillNodeCopy((const SFillNode*)pNode, (SFillNode*)pDst);
     case QUERY_NODE_DATABLOCK_DESC:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -2015,6 +2015,31 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkFillMode = "Mode";
+static const char* jkFillValues = "Values";
+
+static int32_t fillNodeToJson(const void* pObj, SJson* pJson) {
+  const SFillNode* pNode = (const SFillNode*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkFillMode, pNode->mode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues);
+  }
+
+  return code;
+}
+
+static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
+  SFillNode* pNode = (SFillNode*)pObj;
+
+  int32_t code = tjsonGetNumberValue(pJson, jkFillMode, pNode->mode);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
+  }
+
+  return code;
+}
+
 static const char* jkTargetDataBlockId = "DataBlockId";
 static const char* jkTargetSlotId = "SlotId";
 static const char* jkTargetExpr = "Expr";
@@ -2328,6 +2353,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_NODE_LIST:
       return nodeListNodeToJson(pObj, pJson);
     case QUERY_NODE_FILL:
+      return fillNodeToJson(pObj, pJson);
     case QUERY_NODE_RAW_EXPR:
       break;
     case QUERY_NODE_TARGET:
@@ -2431,7 +2457,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToIntervalWindowNode(pJson, pObj);
     case QUERY_NODE_NODE_LIST:
       return jsonToNodeListNode(pJson, pObj);
-    // case QUERY_NODE_FILL:
+    case QUERY_NODE_FILL:
+      return jsonToFillNode(pJson, pObj);
     case QUERY_NODE_TARGET:
       return jsonToTargetNode(pJson, pObj);
     // case QUERY_NODE_RAW_EXPR:

source/libs/nodes/src/nodesToSQLFuncs.c

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdnodes.h"
+#include "nodesUtil.h"
+#include "plannodes.h"
+#include "querynodes.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "thash.h"
+
+char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>", 
+                        "IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL",
+                        "IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"};
+char *gLogicConditionStr[] = {"AND", "OR", "NOT"};
+
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
+  switch (pNode->type) {
+    case QUERY_NODE_COLUMN: {
+      SColumnNode *colNode = (SColumnNode *)pNode;
+      if (colNode->dbName[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName);
+      }
+      
+      if (colNode->tableAlias[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias);
+      } else if (colNode->tableName[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName);
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName);
+      
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_VALUE:{
+      SValueNode *colNode = (SValueNode *)pNode;
+      char *t = nodesGetStrValueFromNode(colNode);
+      if (NULL == t) {
+        nodesError("fail to get str value from valueNode");
+        NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, "%s", t);
+      taosMemoryFree(t);
+      
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_OPERATOR: {
+      SOperatorNode* pOpNode = (SOperatorNode*)pNode;
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      if (pOpNode->pLeft) {
+        NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len));
+      }
+
+      if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) {
+        nodesError("unknown operation type:%d", pOpNode->opType);
+        NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]);
+
+      if (pOpNode->pRight) {
+        NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len));
+      }    
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_LOGIC_CONDITION:{
+      SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      
+      FOREACH(node, pLogicNode->pParameterList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]);
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_FUNCTION:{
+      SFunctionNode* pFuncNode = (SFunctionNode*)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName);
+      
+      FOREACH(node, pFuncNode->pParameterList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, ", ");
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_NODE_LIST:{
+      SNodeListNode* pListNode = (SNodeListNode *)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      
+      FOREACH(node, pListNode->pNodeList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, ", ");
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    default:
+      break;
+  }
+  
+  nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type));
+  NODES_RET(TSDB_CODE_QRY_APP_ERROR);
+}

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -17,7 +17,8 @@
 
 typedef enum ETraversalOrder {
   TRAVERSAL_PREORDER = 1,
-  TRAVERSAL_POSTORDER
+  TRAVERSAL_INORDER,
+  TRAVERSAL_POSTORDER,
 } ETraversalOrder;
 
 static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -790,6 +790,71 @@ void* nodesGetValueFromNode(SValueNode *pNode) {
   return NULL;
 }
 
+char* nodesGetStrValueFromNode(SValueNode *pNode) {
+  switch (pNode->node.resType.type) {
+    case TSDB_DATA_TYPE_BOOL: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%s", pNode->datum.b ? "true" : "false");
+      return buf;
+    }
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%" PRId64, pNode->datum.i);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%" PRIu64, pNode->datum.u);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%e", pNode->datum.d);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_VARCHAR:
+    case TSDB_DATA_TYPE_VARBINARY: {
+      int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1;
+      void *buf = taosMemoryMalloc(bufSize);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p));
+      return buf;
+    }
+    default:
+      break;
+  }
+
+  return NULL;
+}
+
 bool nodesIsExprNode(const SNode* pNode) {
   ENodeType type = nodeType(pNode);
   return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);

source/libs/parser/inc/parAst.h

@@ -71,6 +71,7 @@ typedef enum ETableOptionType {
 typedef struct SAlterOption {
   int32_t type;
   SToken val;
+  SNodeList* pKeep;
 } SAlterOption;
 
 extern SToken nil_token;
@@ -121,6 +122,8 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
 SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
 SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
 SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
+SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
+SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
 SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
 SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
 SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
@@ -129,6 +132,8 @@ SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
 SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
 SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
 SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
+SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
+SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
 SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
 SDataType createDataType(uint8_t type);
 SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
@@ -145,6 +150,8 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int
 SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
 SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern);
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword);
 SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal);
 SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
@@ -165,6 +172,16 @@ SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* p
 SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery);
 SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
 SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups);
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize);
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName);
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery);
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName);
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName);
 
 #ifdef __cplusplus
 }

source/libs/parser/inc/parInsertData.h

@@ -78,8 +78,6 @@ typedef struct STableDataBlocks {
   char       *pData;
   bool        cloned;
   STagData    tagData; 
-  char        tableName[TSDB_TABLE_NAME_LEN];
-  char        dbFName[TSDB_DB_FNAME_LEN];
   
   SParsedDataColInfo boundColumnInfo;
   SRowBuilder        rowBuilder;

source/libs/parser/inc/sql.y

@@ -137,7 +137,7 @@ db_options(A) ::= db_options(B) DAYS NK_INTEGER(C).
 db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C).                              { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); }
 db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); }
 db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); }
-db_options(A) ::= db_options(B) KEEP NK_INTEGER(C).                               { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); }
+db_options(A) ::= db_options(B) KEEP integer_list(C).                             { A = setDatabaseKeepOption(pCxt, B, C); }
 db_options(A) ::= db_options(B) PRECISION NK_STRING(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); }
 db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C).                             { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); }
 db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); }
@@ -148,17 +148,23 @@ db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C).
 db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C).                        { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
 db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C).                          { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
 
-alter_db_options(A) ::= alter_db_option(B).                                       { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
+alter_db_options(A) ::= alter_db_option(B).                                       { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); }
-alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
+alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setDatabaseAlterOption(pCxt, B, &C); }
 
 %type alter_db_option                                                             { SAlterOption }
 %destructor alter_db_option                                                       { }
 alter_db_option(A) ::= BLOCKS NK_INTEGER(B).                                      { A.type = DB_OPTION_BLOCKS; A.val = B; }
 alter_db_option(A) ::= FSYNC NK_INTEGER(B).                                       { A.type = DB_OPTION_FSYNC; A.val = B; }
-alter_db_option(A) ::= KEEP NK_INTEGER(B).                                        { A.type = DB_OPTION_KEEP; A.val = B; }
+alter_db_option(A) ::= KEEP integer_list(B).                                      { A.type = DB_OPTION_KEEP; A.pKeep = B; }
 alter_db_option(A) ::= WAL NK_INTEGER(B).                                         { A.type = DB_OPTION_WAL; A.val = B; }
 alter_db_option(A) ::= QUORUM NK_INTEGER(B).                                      { A.type = DB_OPTION_QUORUM; A.val = B; }
 alter_db_option(A) ::= CACHELAST NK_INTEGER(B).                                   { A.type = DB_OPTION_CACHELAST; A.val = B; }
+alter_db_option(A) ::= REPLICA NK_INTEGER(B).                                     { A.type = DB_OPTION_REPLICA; A.val = B; }
+
+%type integer_list                                                                { SNodeList* }
+%destructor integer_list                                                          { nodesDestroyList($$); }
+integer_list(A) ::= NK_INTEGER(B).                                                { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C).                       { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
 
 /************************************************ create/drop table/stable ********************************************/
 cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
@@ -259,20 +265,20 @@ tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP.
 
 table_options(A) ::= .                                                            { A = createDefaultTableOptions(pCxt); }
 table_options(A) ::= table_options(B) COMMENT NK_STRING(C).                       { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); }
-table_options(A) ::= table_options(B) KEEP NK_INTEGER(C).                         { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
+table_options(A) ::= table_options(B) KEEP integer_list(C).                       { A = setTableKeepOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) TTL NK_INTEGER(C).                          { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
 table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP.           { A = setTableSmaOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP.       { A = setTableRollupOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C).                    { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); }
 table_options(A) ::= table_options(B) DELAY NK_INTEGER(C).                        { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); }
 
-alter_table_options(A) ::= alter_table_option(B).                                 { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
+alter_table_options(A) ::= alter_table_option(B).                                 { A = createDefaultAlterTableOptions(pCxt); A = setTableAlterOption(pCxt, A, &B); }
-alter_table_options(A) ::= alter_table_options(B) alter_table_option(C).          { A = setTableOption(pCxt, B, C.type, &C.val); }
+alter_table_options(A) ::= alter_table_options(B) alter_table_option(C).          { A = setTableAlterOption(pCxt, B, &C); }
 
 %type alter_table_option                                                          { SAlterOption }
 %destructor alter_table_option                                                    { }
 alter_table_option(A) ::= COMMENT NK_STRING(B).                                   { A.type = TABLE_OPTION_COMMENT; A.val = B; }
-alter_table_option(A) ::= KEEP NK_INTEGER(B).                                     { A.type = TABLE_OPTION_KEEP; A.val = B; }
+alter_table_option(A) ::= KEEP integer_list(B).                                   { A.type = TABLE_OPTION_KEEP; A.pKeep = B; }
 alter_table_option(A) ::= TTL NK_INTEGER(B).                                      { A.type = TABLE_OPTION_TTL; A.val = B; }
 
 %type col_name_list                                                               { SNodeList* }
@@ -295,6 +301,17 @@ cmd ::= SHOW QNODES.
 cmd ::= SHOW FUNCTIONS.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); }
 cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B).                      { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); }
 cmd ::= SHOW STREAMS.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); }
+cmd ::= SHOW ACCOUNTS.                                                            { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+cmd ::= SHOW APPS.                                                                { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); }
+cmd ::= SHOW CONNECTIONS.                                                         { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); }
+cmd ::= SHOW LICENCE.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); }
+cmd ::= SHOW CREATE DATABASE db_name(A).                                          { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
+cmd ::= SHOW CREATE TABLE full_table_name(A).                                     { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
+cmd ::= SHOW CREATE STABLE full_table_name(A).                                    { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }
+cmd ::= SHOW QUERIES.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); }
+cmd ::= SHOW SCORES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT, NULL, NULL); }
+cmd ::= SHOW TOPICS.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT, NULL, NULL); }
+cmd ::= SHOW VARIABLES.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); }
 
 db_name_cond_opt(A) ::= .                                                         { A = createDefaultDatabaseCondValue(pCxt); }
 db_name_cond_opt(A) ::= db_name(B) NK_DOT.                                        { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
@@ -358,6 +375,45 @@ explain_options(A) ::= .
 explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C).                     { A = setExplainVerbose(pCxt, B, &C); }
 explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C).                      { A = setExplainRatio(pCxt, B, &C); }
 
+/************************************************ compact *************************************************************/
+cmd ::= COMPACT VNODES IN NK_LP integer_list(A) NK_RP.                            { pCxt->pRootNode = createCompactStmt(pCxt, A); }
+
+/************************************************ create/drop function ************************************************/
+cmd ::= CREATE agg_func_opt(A) FUNCTION function_name(B) 
+  AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E).                         { pCxt->pRootNode = createCreateFunctionStmt(pCxt, A, &B, &C, D, E); }
+cmd ::= DROP FUNCTION function_name(A).                                           { pCxt->pRootNode = createDropFunctionStmt(pCxt, &A); }
+
+%type agg_func_opt                                                                { bool }
+%destructor agg_func_opt                                                          { }
+agg_func_opt(A) ::= .                                                             { A = false; }
+agg_func_opt(A) ::= AGGREGATE.                                                    { A = true; }
+
+%type bufsize_opt                                                                 { int32_t }
+%destructor bufsize_opt                                                           { }
+bufsize_opt(A) ::= .                                                              { A = 0; }
+bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B).                                         { A = strtol(B.z, NULL, 10); }
+
+/************************************************ create/drop stream **************************************************/
+cmd ::= CREATE STREAM stream_name(A) INTO table_name(B) AS query_expression(C).   { pCxt->pRootNode = createCreateStreamStmt(pCxt, &A, &B, C); }
+cmd ::= DROP STREAM stream_name(A).                                               { pCxt->pRootNode = createDropStreamStmt(pCxt, &A); }
+
+/************************************************ kill connection/query ***********************************************/
+cmd ::= KILL CONNECTION NK_INTEGER(A).                                            { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); }
+cmd ::= KILL QUERY NK_INTEGER(A).                                                 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &A); }
+
+/************************************************ merge/redistribute/ vgroup ******************************************/
+cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B).                                 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
+cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B).                          { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
+cmd ::= SPLIT VGROUP NK_INTEGER(A).                                               { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
+
+%type dnode_list                                                                  { SNodeList* }
+%destructor dnode_list                                                            { nodesDestroyList($$); }
+dnode_list(A) ::= DNODE NK_INTEGER(B).                                            { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+dnode_list(A) ::= dnode_list(B) DNODE NK_INTEGER(C).                              { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
+
+/************************************************ syncdb **************************************************************/
+cmd ::= SYNCDB db_name(A) REPLICA.                                                { pCxt->pRootNode = createSyncdbStmt(pCxt, &A); }
+
 /************************************************ select **************************************************************/
 cmd ::= query_expression(A).                                                      { pCxt->pRootNode = A; }
 
@@ -436,6 +492,10 @@ index_name(A) ::= NK_ID(B).
 %destructor topic_name                                                            { }
 topic_name(A) ::= NK_ID(B).                                                       { A = B; }
 
+%type stream_name                                                                 { SToken }
+%destructor stream_name                                                           { }
+stream_name(A) ::= NK_ID(B).                                                      { A = B; }
+
 /************************************************ expression **********************************************************/
 expression(A) ::= literal(B).                                                     { A = B; }
 //expression(A) ::= NK_QUESTION(B).                                                 { A = B; }

source/libs/parser/src/parAstCreater.c

@@ -138,18 +138,6 @@ static SDatabaseOptions* setDbMinRows(SAstCreateContext* pCxt, SDatabaseOptions*
   return pOptions;
 }
 
-static SDatabaseOptions* setDbKeep(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
-  int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
-    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid db option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-    pCxt->valid = false;
-    return pOptions;
-  }
-  pOptions->keep = val;
-  return pOptions;
-}
-
 static SDatabaseOptions* setDbPrecision(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
   char val[10] = {0};
   trimString(pVal->z, pVal->n, val, sizeof(val));
@@ -180,9 +168,9 @@ static SDatabaseOptions* setDbQuorum(SAstCreateContext* pCxt, SDatabaseOptions*
 
 static SDatabaseOptions* setDbReplica(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
   int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_DB_REPLICA_OPTION || val > TSDB_MAX_DB_REPLICA_OPTION) {
+  if (!(val == TSDB_MIN_DB_REPLICA_OPTION || val == TSDB_MAX_DB_REPLICA_OPTION)) {
     snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid db option replications: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+        "invalid db option replications: %"PRId64", only 1, 3 allowed", val);
     pCxt->valid = false;
     return pOptions;
   }
@@ -291,7 +279,6 @@ static void initSetDatabaseOptionFp() {
   setDbOptionFuncs[DB_OPTION_FSYNC] = setDbFsync;
   setDbOptionFuncs[DB_OPTION_MAXROWS] = setDbMaxRows;
   setDbOptionFuncs[DB_OPTION_MINROWS] = setDbMinRows;
-  setDbOptionFuncs[DB_OPTION_KEEP] = setDbKeep;
   setDbOptionFuncs[DB_OPTION_PRECISION] = setDbPrecision;
   setDbOptionFuncs[DB_OPTION_QUORUM] = setDbQuorum;
   setDbOptionFuncs[DB_OPTION_REPLICA] = setDbReplica;
@@ -303,18 +290,6 @@ static void initSetDatabaseOptionFp() {
   setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
 }
 
-static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
-  int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
-    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid table option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-    pCxt->valid = false;
-    return pOptions;
-  }
-  pOptions->keep = val;
-  return pOptions;
-}
-
 static STableOptions* setTableTtl(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
   int64_t val = strtol(pVal->z, NULL, 10);
   if (val < TSDB_MIN_DB_TTL_OPTION) {
@@ -363,7 +338,6 @@ static STableOptions* setTableDelay(SAstCreateContext* pCxt, STableOptions* pOpt
 }
 
 static void initSetTableOptionFp() {
-  setTableOptionFuncs[TABLE_OPTION_KEEP] = setTableKeep;
   setTableOptionFuncs[TABLE_OPTION_TTL] = setTableTtl;
   setTableOptionFuncs[TABLE_OPTION_COMMENT] = setTableComment;
   setTableOptionFuncs[TABLE_OPTION_FILE_FACTOR] = setTableFileFactor;
@@ -397,7 +371,9 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) {
       pCxt->valid = false;
     }
   }
-  trimEscape(pUserName);
+  if (pCxt->valid) {
+    trimEscape(pUserName);
+  }
   return pCxt->valid;
 }
 
@@ -472,45 +448,50 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t
 
 static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) {
   if (NULL == pDbName) {
-    pCxt->valid = (query ? NULL != pCxt->pQueryCxt->db : true);
+    if (query && NULL == pCxt->pQueryCxt->db) {
-    if (!pCxt->valid) {
+      generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED);
-      snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified");
+      pCxt->valid = false;
     }
   } else {
-    pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false;
+    if (pDbName->n >= TSDB_DB_NAME_LEN) {
+      generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z);
+      pCxt->valid = false;
+    }
+  }
+  if (pCxt->valid) {
+    trimEscape(pDbName);
   }
-  trimEscape(pDbName);
   return pCxt->valid;
 }
 
 static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
-  if (NULL == pTableName) {
+  if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) {
-    pCxt->valid = true;
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z);
-  } else {
+    pCxt->valid = false;
-    pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false;
+    return false;
   }
   trimEscape(pTableName);
-  return pCxt->valid;
+  return true;
 }
 
 static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) {
-  if (NULL == pColumnName) {
+  if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) {
-    pCxt->valid = true;
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z);
-  } else {
+    pCxt->valid = false;
-    pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false;
+    return false;
   }
   trimEscape(pColumnName);
-  return pCxt->valid;
+  return true;
 }
 
 static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
-  if (NULL == pIndexName) {
+  if (NULL != pIndexName && pIndexName->n >= TSDB_INDEX_NAME_LEN) {
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z);
     pCxt->valid = false;
-  } else {
+    return false;
-    pCxt->valid = pIndexName->n < TSDB_INDEX_NAME_LEN ? true : false;
   }
   trimEscape(pIndexName);
-  return pCxt->valid;
+  return true;
 }
 
 SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) {
@@ -885,7 +866,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
   pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
   pOptions->maxRowsPerBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
   pOptions->minRowsPerBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
-  pOptions->keep = TSDB_DEFAULT_KEEP;
+  pOptions->keep0 = TSDB_DEFAULT_KEEP;
+  pOptions->keep1 = TSDB_DEFAULT_KEEP;
+  pOptions->keep2 = TSDB_DEFAULT_KEEP;
   pOptions->precision = TSDB_TIME_PRECISION_MILLI;
   pOptions->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
   pOptions->replica = TSDB_DEFAULT_DB_REPLICA_OPTION;
@@ -908,7 +891,9 @@ SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
   pOptions->fsyncPeriod = -1;
   pOptions->maxRowsPerBlock = -1;
   pOptions->minRowsPerBlock = -1;
-  pOptions->keep = -1;
+  pOptions->keep0 = -1;
+  pOptions->keep1 = -1;
+  pOptions->keep2= -1;
   pOptions->precision = -1;
   pOptions->quorum = -1;
   pOptions->replica = -1;
@@ -924,6 +909,48 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti
   return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
 }
 
+static bool checkAndSetKeepOption(SAstCreateContext* pCxt, SNodeList* pKeep, int32_t* pKeep0, int32_t* pKeep1, int32_t* pKeep2) {
+  int32_t numOfKeep = LIST_LENGTH(pKeep);
+  if (numOfKeep > 3 || numOfKeep < 1) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid number of keep options");
+    return false;
+  }
+
+  int32_t daysToKeep0 = strtol(((SValueNode*)nodesListGetNode(pKeep, 0))->literal, NULL, 10);
+  int32_t daysToKeep1 = numOfKeep > 1 ? strtol(((SValueNode*)nodesListGetNode(pKeep, 1))->literal, NULL, 10) : daysToKeep0;
+  int32_t daysToKeep2 = numOfKeep > 2 ? strtol(((SValueNode*)nodesListGetNode(pKeep, 2))->literal, NULL, 10) : daysToKeep1;
+  if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP ||
+      daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
+        "invalid option keep: %d, %d, %d valid range: [%d, %d]", daysToKeep0, daysToKeep1, daysToKeep2, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
+    return false;
+  }
+
+  if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid keep value, should be keep0 <= keep1 <= keep2");
+    return false;
+  }
+
+  *pKeep0 = daysToKeep0;
+  *pKeep1 = daysToKeep1;
+  *pKeep2 = daysToKeep2;
+  return true;
+}
+
+SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
+  SDatabaseOptions* pOp = (SDatabaseOptions*)pOptions;
+  pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
+  return pOptions;
+}
+
+SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
+  if (DB_OPTION_KEEP == pAlterOption->type) {
+    return setDatabaseKeepOption(pCxt, pOptions, pAlterOption->pKeep);
+  } else {
+    return setDatabaseOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val);
+  }
+}
+
 SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) {
   if (!checkDbName(pCxt, pDbName, false)) {
     return NULL;
@@ -961,7 +988,9 @@ SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode*
 SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
   STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
   CHECK_OUT_OF_MEM(pOptions);
-  pOptions->keep = TSDB_DEFAULT_KEEP;
+  pOptions->keep0 = TSDB_DEFAULT_KEEP;
+  pOptions->keep1 = TSDB_DEFAULT_KEEP;
+  pOptions->keep2 = TSDB_DEFAULT_KEEP;
   pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
   pOptions->filesFactor = TSDB_DEFAULT_DB_FILE_FACTOR;
   pOptions->delay = TSDB_DEFAULT_DB_DELAY;
@@ -971,7 +1000,9 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
 SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
   STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
   CHECK_OUT_OF_MEM(pOptions);
-  pOptions->keep = -1;
+  pOptions->keep0 = -1;
+  pOptions->keep1 = -1;
+  pOptions->keep2 = -1;
   pOptions->ttl = -1;
   pOptions->filesFactor = -1;
   pOptions->delay = -1;
@@ -997,6 +1028,20 @@ SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList*
   return pOptions;
 }
 
+SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
+  STableOptions* pOp = (STableOptions*)pOptions;
+  pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
+  return pOptions;
+}
+
+SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
+  if (TABLE_OPTION_KEEP == pAlterOption->type) {
+    return setTableKeepOption(pCxt, pOptions, pAlterOption->pKeep);
+  } else {
+    return setTableOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val);
+  }
+}
+
 SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
   SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF);
   CHECK_OUT_OF_MEM(pCol);
@@ -1176,6 +1221,18 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, S
   return (SNode*)pStmt;
 }
 
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SHOW_CREATE_DATABASE_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) {
   char password[TSDB_USET_PASSWORD_LEN] = {0};
   if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) {
@@ -1388,3 +1445,63 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) {
   CHECK_OUT_OF_MEM(pStmt);
   return pStmt;
 }
+
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_COMPACT_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SPLIT_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SYNCDB_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}

source/libs/parser/src/parInsert.c

@@ -52,8 +52,6 @@ typedef struct SInsertParseContext {
   SParseContext* pComCxt;       // input
   char          *pSql;          // input
   SMsgBuf        msg;           // input
-  char           dbFName[TSDB_DB_FNAME_LEN];
-  char           tableName[TSDB_TABLE_NAME_LEN];
   STableMeta* pTableMeta;       // each table
   SParsedDataColInfo tags;      // each table
   SKVRowBuilder tagsBuilder;    // each table
@@ -231,9 +229,6 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
   SVgroupInfo vg;
   CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
   CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-  pCxt->pTableMeta->vgId = vg.vgId; // todo remove
-  strcpy(pCxt->tableName, name.tname);
-  tNameGetFullDbName(&name, pCxt->dbFName);
   
   return TSDB_CODE_SUCCESS;
 }
@@ -977,8 +972,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     STableDataBlocks *dataBuf = NULL;
     CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
         sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL));
-    strcpy(dataBuf->tableName, pCxt->tableName);
-    strcpy(dataBuf->dbFName, pCxt->dbFName);
     
     if (TK_NK_LP == sToken.type) {
       // pSql -> field1_name, ...)

source/libs/parser/src/parTokenizer.c

@@ -29,10 +29,14 @@ typedef struct SKeyword {
 // keywords in sql string
 static SKeyword keywordTable[] = {
     {"ACCOUNT",       TK_ACCOUNT},
+    {"ACCOUNTS",      TK_ACCOUNTS},
+    {"ADD",           TK_ADD},
+    {"AGGREGATE",     TK_AGGREGATE},
     {"ALL",           TK_ALL},
     {"ALTER",         TK_ALTER},
     {"ANALYZE",       TK_ANALYZE},
     {"AND",           TK_AND},
+    {"APPS",          TK_APPS},
     {"AS",            TK_AS},
     {"ASC",           TK_ASC},
     {"BETWEEN",       TK_BETWEEN},
@@ -40,15 +44,22 @@ static SKeyword keywordTable[] = {
     {"BIGINT",        TK_BIGINT},
     {"BLOCKS",        TK_BLOCKS},
     {"BOOL",          TK_BOOL},
+    {"BUFSIZE",       TK_BUFSIZE},
     {"BY",            TK_BY},
     {"CACHE",         TK_CACHE},
     {"CACHELAST",     TK_CACHELAST},
+    {"COLUMN",        TK_COLUMN},
     {"COMMENT",       TK_COMMENT},
     {"COMP",          TK_COMP},
+    {"COMPACT",       TK_COMPACT},
+    {"CONNS",         TK_CONNS},
+    {"CONNECTION",    TK_CONNECTION},
+    {"CONNECTIONS",   TK_CONNECTIONS},
     {"CREATE",        TK_CREATE},
     {"DATABASE",      TK_DATABASE},
     {"DATABASES",     TK_DATABASES},
     {"DAYS",          TK_DAYS},
+    {"DBS",           TK_DBS},
     {"DELAY",         TK_DELAY},
     {"DESC",          TK_DESC},
     {"DESCRIBE",      TK_DESCRIBE},
@@ -83,14 +94,18 @@ static SKeyword keywordTable[] = {
     {"JOIN",          TK_JOIN},
     {"JSON",          TK_JSON},
     {"KEEP",          TK_KEEP},
+    {"KILL",          TK_KILL},
+    {"LICENCE",       TK_LICENCE},
     {"LIKE",          TK_LIKE},
     {"LIMIT",         TK_LIMIT},
     {"LINEAR",        TK_LINEAR},
+    {"LOCAL",         TK_LOCAL},
     {"MATCH",         TK_MATCH},
     {"MAXROWS",       TK_MAXROWS},
     {"MINROWS",       TK_MINROWS},
     {"MINUS",         TK_MINUS},
     {"MNODES",        TK_MNODES},
+    {"MODIFY",        TK_MODIFY},
     {"MODULES",       TK_MODULES},
     {"NCHAR",         TK_NCHAR},
     {"NMATCH",        TK_NMATCH},
@@ -102,9 +117,11 @@ static SKeyword keywordTable[] = {
     {"ON",            TK_ON},
     {"OR",            TK_OR},
     {"ORDER",         TK_ORDER},
+    {"OUTPUTTYPE",    TK_OUTPUTTYPE},
     {"PARTITION",     TK_PARTITION},
     {"PASS",          TK_PASS},
     {"PORT",          TK_PORT},
+    {"PPS",           TK_PPS},
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
@@ -112,6 +129,8 @@ static SKeyword keywordTable[] = {
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
     {"QSTARTTS",      TK_QSTARTTS},
+    {"QTIME",         TK_QTIME},
+    {"QUERIES",       TK_QUERIES},
     {"QUERY",         TK_QUERY},
     {"QUORUM",        TK_QUORUM},
     {"RATIO",         TK_RATIO},
@@ -120,8 +139,10 @@ static SKeyword keywordTable[] = {
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
     {"ROWTS",         TK_ROWTS},
+    {"SCORES",        TK_SCORES},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
+    {"SET",           TK_SET},
     {"SHOW",          TK_SHOW},
     {"SINGLE_STABLE", TK_SINGLE_STABLE},
     {"SLIDING",       TK_SLIDING},
@@ -131,16 +152,23 @@ static SKeyword keywordTable[] = {
     {"SOFFSET",       TK_SOFFSET},
     {"STABLE",        TK_STABLE},
     {"STABLES",       TK_STABLES},
+    {"STATE",         TK_STATE},
     {"STATE_WINDOW",  TK_STATE_WINDOW},
+    {"STORAGE",       TK_STORAGE},
+    {"STREAM",       TK_STREAM},
     {"STREAMS",       TK_STREAMS},
     {"STREAM_MODE",   TK_STREAM_MODE},
+    {"SYNCDB",        TK_SYNCDB},
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
+    {"TAG",           TK_TAG},
     {"TAGS",          TK_TAGS},
     {"TBNAME",        TK_TBNAME},
     {"TIMESTAMP",     TK_TIMESTAMP},
     {"TINYINT",       TK_TINYINT},
     {"TOPIC",         TK_TOPIC},
+    {"TOPICS",        TK_TOPICS},
+    {"TSERIES",       TK_TSERIES},
     {"TTL",           TK_TTL},
     {"UNION",         TK_UNION},
     {"UNSIGNED",      TK_UNSIGNED},
@@ -148,10 +176,13 @@ static SKeyword keywordTable[] = {
     {"USER",          TK_USER},
     {"USERS",         TK_USERS},
     {"USING",         TK_USING},
+    {"VALUE",         TK_VALUE},
     {"VALUES",        TK_VALUES},
     {"VARCHAR",       TK_VARCHAR},
+    {"VARIABLES",     TK_VARIABLES},
     {"VERBOSE",       TK_VERBOSE},
     {"VGROUPS",       TK_VGROUPS},
+    {"VNODES",        TK_VNODES},
     {"WAL",           TK_WAL},
     {"WDURATION",     TK_WDURATION},
     {"WENDTS",        TK_WENDTS},
@@ -182,22 +213,8 @@ static SKeyword keywordTable[] = {
     // {"UMINUS",       TK_UMINUS},
     // {"UPLUS",        TK_UPLUS},
     // {"BITNOT",       TK_BITNOT},
-    // {"ACCOUNTS",     TK_ACCOUNTS},
-    // {"QUERIES",      TK_QUERIES},
-    // {"CONNECTIONS",  TK_CONNECTIONS},
-    // {"VARIABLES",    TK_VARIABLES},
-    // {"SCORES",       TK_SCORES},
     // {"GRANTS",       TK_GRANTS},
     // {"DOT",          TK_DOT},
-    // {"SYNCDB",       TK_SYNCDB},
-    // {"LOCAL",        TK_LOCAL},
-    // {"PPS",          TK_PPS},
-    // {"TSERIES",      TK_TSERIES},
-    // {"DBS",          TK_DBS},
-    // {"STORAGE",      TK_STORAGE},
-    // {"QTIME",        TK_QTIME},
-    // {"CONNS",        TK_CONNS},
-    // {"STATE",        TK_STATE},
     // {"CTIME",        TK_CTIME},
     // {"LP",           TK_LP},
     // {"RP",           TK_RP},
@@ -205,15 +222,8 @@ static SKeyword keywordTable[] = {
     // {"EVERY",        TK_EVERY},
     // {"VARIABLE",     TK_VARIABLE},
     // {"UPDATE",       TK_UPDATE},
-    // {"ADD",          TK_ADD},
-    // {"COLUMN",       TK_COLUMN},
-    // {"TAG",          TK_TAG},
     // {"CHANGE",       TK_CHANGE},
-    // {"SET",          TK_SET},
-    // {"KILL",         TK_KILL},
-    // {"CONNECTION",   TK_CONNECTION},
     // {"COLON",        TK_COLON},
-    // {"STREAM",       TK_STREAM},
     // {"ABORT",        TK_ABORT},
     // {"AFTER",        TK_AFTER},
     // {"ATTACH",       TK_ATTACH},
@@ -244,14 +254,7 @@ static SKeyword keywordTable[] = {
     // {"TRIGGER",      TK_TRIGGER},
     // {"VIEW",         TK_VIEW},
     // {"SEMI",         TK_SEMI},
-    // {"VNODES",       TK_VNODES},
+    // {"PARTITIONS",   TK_PARTITIONS},
-//    {"PARTITIONS",   TK_PARTITIONS},
-    // {"TOPICS",       TK_TOPICS},
-    // {"COMPACT",      TK_COMPACT},
-    // {"MODIFY",       TK_MODIFY},
-    // {"OUTPUTTYPE",   TK_OUTPUTTYPE},
-    // {"AGGREGATE",    TK_AGGREGATE},
-    // {"BUFSIZE",      TK_BUFSIZE},
     // {"MODE",         TK_MODE},
 };
 

source/libs/parser/src/parTranslater.c

@@ -240,7 +240,11 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   if (NULL != pTable) {
     strcpy(pCol->tableAlias, pTable->tableAlias);
   } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
-    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
+    SColumnNode* pProjCol = (SColumnNode*)pExpr;
+    strcpy(pCol->tableAlias, pProjCol->tableAlias);
+    pCol->tableId = pProjCol->tableId;
+    pCol->colId = pProjCol->colId;
+    pCol->colType = pProjCol->colType;
   }
   strcpy(pCol->colName, pExpr->aliasName);
   pCol->node.resType = pExpr->resType;
@@ -956,9 +960,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
   pReq->cacheBlockSize = pStmt->pOptions->cacheBlockSize;
   pReq->totalBlocks = pStmt->pOptions->numOfBlocks;
   pReq->daysPerFile = pStmt->pOptions->daysPerFile;
-  pReq->daysToKeep0 = pStmt->pOptions->keep;
+  pReq->daysToKeep0 = pStmt->pOptions->keep0;
-  pReq->daysToKeep1 = -1;
+  pReq->daysToKeep1 = pStmt->pOptions->keep1;
-  pReq->daysToKeep2 = -1;
+  pReq->daysToKeep2 = pStmt->pOptions->keep2;
   pReq->minRows = pStmt->pOptions->minRowsPerBlock;
   pReq->maxRows = pStmt->pOptions->maxRowsPerBlock;
   pReq->commitTime = -1;
@@ -1041,13 +1045,14 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   tNameGetFullDbName(&name, pReq->db);
   pReq->totalBlocks = pStmt->pOptions->numOfBlocks;
-  pReq->daysToKeep0 = pStmt->pOptions->keep;
+  pReq->daysToKeep0 = pStmt->pOptions->keep0;
-  pReq->daysToKeep1 = -1;
+  pReq->daysToKeep1 = pStmt->pOptions->keep1;
-  pReq->daysToKeep2 = -1;
+  pReq->daysToKeep2 = pStmt->pOptions->keep2;
   pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
   pReq->walLevel = pStmt->pOptions->walLevel;
   pReq->quorum = pStmt->pOptions->quorum;
   pReq->cacheLastRow = pStmt->pOptions->cachelast;
+  pReq->replications = pStmt->pOptions->replica;
   return;
 }
 
@@ -1094,10 +1099,6 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
 }
 
 static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
-  if (NULL == pList) {
-    return TSDB_CODE_SUCCESS;
-  }
-
   *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
   SNode* pNode;
   FOREACH(pNode, pList) {
@@ -1119,7 +1120,7 @@ static const SColumnDefNode* findColDef(const SNodeList* pCols, const SColumnNod
   return NULL;
 }
 
-static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+static int32_t checkCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
   if (NULL != pStmt->pOptions->pSma) {
     SNode* pNode = NULL;
     FOREACH(pNode, pStmt->pOptions->pSma) {
@@ -1148,7 +1149,7 @@ static int32_t getAggregationMethod(SNodeList* pFuncs) {
 }
 
 static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
-  int32_t code = checkCreateTable(pCxt, pStmt);
+  int32_t code = checkCreateSuperTable(pCxt, pStmt);
   if (TSDB_CODE_SUCCESS != code) {
     return code;
   }
@@ -1160,10 +1161,15 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt
   createReq.delay = pStmt->pOptions->delay;
   columnDefNodeToField(pStmt->pCols, &createReq.pColumns);
   columnDefNodeToField(pStmt->pTags, &createReq.pTags);
-  columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
   createReq.numOfColumns = LIST_LENGTH(pStmt->pCols);
   createReq.numOfTags = LIST_LENGTH(pStmt->pTags);
-  createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  if (NULL == pStmt->pOptions->pSma) {
+    columnDefNodeToField(pStmt->pCols, &createReq.pSmas);
+    createReq.numOfSmas = createReq.numOfColumns;
+  } else {
+    columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
+    createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  }
 
   SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
   strcpy(tableName.dbname, pStmt->dbName);
@@ -1469,20 +1475,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
 
 static int32_t nodeTypeToShowType(ENodeType nt) {
   switch (nt) {
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-      return TSDB_MGMT_TABLE_DB;
+      return 0; // todo
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-      return TSDB_MGMT_TABLE_STB;
+      return TSDB_MGMT_TABLE_CONNS;
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-      return TSDB_MGMT_TABLE_USER;
+      return 0; // todo
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-      return TSDB_MGMT_TABLE_DNODE;
+      return TSDB_MGMT_TABLE_QUERIES;
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-      return TSDB_MGMT_TABLE_VGROUP;
+      return 0; // todo
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-      return TSDB_MGMT_TABLE_MNODE;
+      return 0; // todo
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
-      return TSDB_MGMT_TABLE_QNODE;
+      return TSDB_MGMT_TABLE_VARIABLES;
     default:
       break;
   }
@@ -1508,30 +1514,6 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateShowTables(STranslateContext* pCxt) {
-  SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq));
-
-  SArray* array = NULL;
-  int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array);
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
-  }
-  SVgroupInfo* info = taosArrayGet(array, 0);
-  pShowReq->head.vgId = htonl(info->vgId);
-
-  pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
-  if (NULL == pCxt->pCmdMsg) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  pCxt->pCmdMsg->epSet = info->epSet;
-  pCxt->pCmdMsg->msgType = TDMT_VND_SHOW_TABLES;
-  pCxt->pCmdMsg->msgLen = sizeof(SVShowTablesReq);
-  pCxt->pCmdMsg->pMsg = pShowReq;
-  pCxt->pCmdMsg->pExtension = array;
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) {
   SVgroupInfo vg = {0};
   int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg);
@@ -1867,17 +1849,19 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_ALTER_DNODE_STMT:
       code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
       code = translateShow(pCxt, (SShowStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_TABLES_STMT:
+    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
-      code = translateShowTables(pCxt);
+    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
+      // todo
       break;
     case QUERY_NODE_CREATE_INDEX_STMT:
       code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode);
@@ -1952,6 +1936,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema)
   }
   (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
   (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+  strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2167,10 +2152,11 @@ typedef struct SVgroupTablesBatch {
   char               dbName[TSDB_DB_NAME_LEN];
 } SVgroupTablesBatch;
 
-static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) {
+static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) {
   pSchema->colId = colId;
   pSchema->type = pCol->dataType.type;
   pSchema->bytes = calcTypeBytes(pCol->dataType);
+  pSchema->sma = TSDB_BSMA_TYPE_LATEST;  // TODO: use default value currently, and use the real value later.
   strcpy(pSchema->name, pCol->colName);
 }
 
@@ -2192,7 +2178,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con
   req.dbFName = strdup(dbFName);
   req.name = strdup(pTableName);
   req.ntbCfg.nCols = LIST_LENGTH(pColumns);
-  req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema));
+  req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx));
   if (NULL == req.name || NULL == req.ntbCfg.pSchema) {
     destroyCreateTbReq(&req);
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -2203,6 +2189,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con
     toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index);
     ++index;
   }
+  // TODO: use the real sma for normal table.
 
   pBatch->info = *pVgroupInfo;
   strcpy(pBatch->dbName, pDbName);
@@ -2447,9 +2434,19 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) {
+  if (0 != strcmp(pStmt->dbName, pStmt->useDbName)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR);;
+  }
+  return TSDB_CODE_SUCCESS;
+}
 static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt, SHashObj* pVgroupHashmap) {
+  int32_t code = checkCreateSubTable(pCxt, pStmt);
+
   STableMeta* pSuperTableMeta = NULL;
-  int32_t code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
+  }
 
   SKVRowBuilder kvRowBuilder = {0};
   if (TSDB_CODE_SUCCESS == code) {

source/libs/parser/src/parUtil.c

@@ -61,6 +61,12 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "This statement is no longer supported";
     case TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL:
       return "This interval value is too small : %s";
+    case TSDB_CODE_PAR_DB_NOT_SPECIFIED:
+      return "db not specified";
+    case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME:
+      return "Invalid identifier name : %s";
+    case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR:
+      return "corresponding super table not in this db";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/planner/src/planLogicCreater.c

@@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   // set the output
   if (TSDB_CODE_SUCCESS == code) {
     pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
-    if (NULL == pJoin->pOnConditions) {
+    if (NULL == pJoin->node.pTargets) {
       code = TSDB_CODE_OUT_OF_MEMORY;
     }
     if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/test/plannerTest.cpp

@@ -190,10 +190,16 @@ TEST_F(PlannerTest, subquery) {
 TEST_F(PlannerTest, interval) {
   setDatabase("root", "test");
 
-  bind("SELECT count(*) FROM t1 interval(10s)");
+  // bind("SELECT count(*) FROM t1 interval(10s)");
-  ASSERT_TRUE(run());
+  // ASSERT_TRUE(run());
 
-  bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
+  // bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
+  // ASSERT_TRUE(run());
+
+  // bind("SELECT count(*) FROM t1 interval(10s) fill(linear)");
+  // ASSERT_TRUE(run());
+
+  bind("SELECT count(*), sum(c1) FROM t1 interval(10s) fill(value, 10, 20)");
   ASSERT_TRUE(run());
 }
 

source/libs/qcom/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
     qcom
-    PRIVATE os util transport
+    PRIVATE os util transport nodes
 )
 
 if(${BUILD_TEST})

source/libs/qcom/inc/queryInt.h

@@ -21,7 +21,6 @@ extern "C" {
 #endif
 
 
-
 #ifdef __cplusplus
 }
 #endif

source/libs/scalar/inc/filterInt.h

@@ -36,8 +36,6 @@ extern "C" {
 
 #define FILTER_DUMMY_EMPTY_OPTR  127
 
-#define MAX_NUM_STR_SIZE 40
-
 #define FILTER_RM_UNIT_MIN_ROWS 100
 
 enum {

source/libs/scalar/src/sclfunc.c

@@ -23,7 +23,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -36,7 +36,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -49,7 +49,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -62,7 +62,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -75,7 +75,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -88,7 +88,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }

source/libs/scheduler/CMakeLists.txt

@@ -9,7 +9,7 @@ target_include_directories(
 
 target_link_libraries(
         scheduler
-        PUBLIC os util nodes planner qcom common catalog transport
+        PUBLIC os util nodes planner qcom common catalog transport command
 )
 
 if(${BUILD_TEST})

source/libs/scheduler/inc/schedulerInt.h

@@ -142,10 +142,10 @@ typedef struct SSchTask {
 } SSchTask;
 
 typedef struct SSchJobAttr {
-  bool needFetch;
+  EExplainMode explainMode;
-  bool syncSchedule;
+  bool         syncSchedule;
-  bool queryJob;
+  bool         queryJob;
-  bool needFlowCtrl;
+  bool         needFlowCtrl;
 } SSchJobAttr;
 
 typedef struct SSchJob {

source/libs/scheduler/src/scheduler.c

@@ -19,6 +19,7 @@
 #include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
+#include "command.h"
 
 SSchedulerMgmt schMgmt = {0};
 
@@ -2103,6 +2104,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
     SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  pJob->attr.explainMode = pDag->explainInfo.mode;
   pJob->attr.syncSchedule = syncSchedule;
   pJob->transport = transport;
   pJob->sql = sql;
@@ -2136,19 +2138,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
 
   tsem_init(&pJob->rspSem, 0, 0);
 
-  pJob->refId = taosAddRef(schMgmt.jobRef, pJob);
+  int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
-  if (pJob->refId < 0) {
+  if (refId < 0) {
-    SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno));
+    SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
     SCH_ERR_JRET(terrno);
   }
 
+  if (NULL == schAcquireJob(refId)) {
+    SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
+    SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
+  }
+
+  pJob->refId = refId;
+
   SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
 
   pJob->status = JOB_TASK_STATUS_NOT_START;
   SCH_ERR_JRET(schLaunchJob(pJob));
 
-  schAcquireJob(pJob->refId);
-
   *job = pJob->refId;
 
   if (syncSchedule) {
@@ -2168,6 +2175,54 @@ _return:
   SCH_RET(code);
 }
 
+int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql,
+                              bool syncSchedule) {
+  qDebug("QID:0x%" PRIx64 " job started", pDag->queryId);
+
+  int32_t  code = 0;
+  SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
+  if (NULL == pJob) {
+    qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob));
+    SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  pJob->sql = sql;
+  pJob->attr.queryJob = true;
+  pJob->attr.explainMode = pDag->explainInfo.mode;
+  pJob->queryId = pDag->queryId;
+  pJob->subPlans = pDag->pSubplans;
+
+  SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData));
+
+  int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
+  if (refId < 0) {
+    SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
+    SCH_ERR_JRET(terrno);
+  }
+
+  if (NULL == schAcquireJob(refId)) {
+    SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
+    SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
+  }
+
+  pJob->refId = refId;
+
+  SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
+
+  pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
+  *job = pJob->refId;
+  SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
+
+  schReleaseJob(pJob->refId);
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  schFreeJobImpl(pJob);
+  SCH_RET(code);
+}
+
 int32_t schedulerInit(SSchedulerCfg *cfg) {
   if (schMgmt.jobRef) {
     qError("scheduler already initialized");
@@ -2216,7 +2271,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
-  SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
+  if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) {
+    SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true));
+  } else {
+    SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
+  }
 
   SSchJob *job = schAcquireJob(*pJob);
 
@@ -2399,11 +2458,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) {
     SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status));
     goto _return;
   } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
-    SCH_ERR_JRET(schFetchFromRemote(pJob));
+    if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) {
+      SCH_ERR_JRET(schFetchFromRemote(pJob));
+      tsem_wait(&pJob->rspSem);
+    } 
+  } else {
+    SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status));
+    SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
   }
 
-  tsem_wait(&pJob->rspSem);
-
   status = SCH_GET_JOB_STATUS(pJob);
 
   if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {

source/libs/tdb/src/db/tdbBtree.c

@@ -29,15 +29,15 @@ struct SBTree {
   int            minLocal;
   int            maxLeaf;
   int            minLeaf;
-  u8            *pTmp;
+  void          *pBuf;
 };
 
 #define TDB_BTREE_PAGE_COMMON_HDR u8 flags;
 
-#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0]
+#define TDB_BTREE_PAGE_GET_FLAGS(PAGE)        (PAGE)->pData[0]
 #define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags))
-#define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
+#define TDB_BTREE_PAGE_IS_ROOT(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
-#define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
+#define TDB_BTREE_PAGE_IS_LEAF(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
 #define TDB_BTREE_ASSERT_FLAG(flags)                                                 \
   ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
          TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0))
@@ -101,7 +101,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
   // pBt->kcmpr
   pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn;
   // pBt->pageSize
-  pBt->pageSize = tdbPagerGetPageSize(pPager);
+  pBt->pageSize = pPager->pageSize;
   // pBt->maxLocal
   pBt->maxLocal = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr)) / 4;
   // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData]
@@ -127,132 +127,144 @@ int tdbBtreeClose(SBTree *pBt) {
   return 0;
 }
 
-int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen) {
-  int     ret;
+  SBTC   btc;
-  int     idx;
+  SCell *pCell;
-  SPager *pPager;
+  void  *pBuf;
-  SCell  *pCell;
+  int    szCell;
-  int     szCell;
+  int    szBuf;
-  int     cret;
+  int    ret;
-  SBTree *pBt;
+  int    idx;
+  int    c;
 
-  ret = tdbBtcMoveTo(pBtc, pKey, kLen, &cret);
+  tdbBtcOpen(&btc, pBt);
+
+  // move to the position to insert
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
   if (ret < 0) {
-    // TODO: handle error
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  if (pBtc->idx == -1) {
+  if (btc.idx == -1) {
-    ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
     idx = 0;
   } else {
-    if (cret > 0) {
+    if (c > 0) {
-      idx = pBtc->idx + 1;
+      idx = btc.idx + 1;
-    } else if (cret < 0) {
+    } else if (c < 0) {
-      idx = pBtc->idx;
+      idx = btc.idx;
     } else {
-      /* TODO */
+      // TDB does NOT allow same key
+      tdbBtcClose(&btc);
       ASSERT(0);
-    }
-  }
-
-  // TODO: refact code here
-  pBt = pBtc->pBt;
-  if (!pBt->pTmp) {
-    pBt->pTmp = (u8 *)tdbOsMalloc(pBt->pageSize);
-    if (pBt->pTmp == NULL) {
       return -1;
     }
   }
 
-  pCell = pBt->pTmp;
+  // make sure enough space to hold the cell
+  szBuf = kLen + vLen + 14;
+  pBuf = TDB_REALLOC(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize);
+  if (pBuf == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+  pBt->pBuf = pBuf;
+  pCell = (SCell *)pBt->pBuf;
 
-  // Encode the cell
+  // encode cell
-  ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
+  ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // Insert the cell to the index
+  // mark the page dirty
-  ret = tdbPageInsertCell(pBtc->pPage, idx, pCell, szCell, 0);
+  ret = tdbPagerWrite(pBt->pPager, btc.pPage);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // If page is overflow, balance the tree
+  // insert the cell
-  if (pBtc->pPage->nOverflow > 0) {
+  ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0);
-    ret = tdbBtreeBalance(pBtc);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+
+  // check if need balance
+  if (btc.pPage->nOverflow > 0) {
+    ret = tdbBtreeBalance(&btc);
     if (ret < 0) {
+      tdbBtcClose(&btc);
+      ASSERT(0);
       return -1;
     }
   }
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) {
-  SBTC         btc;
+  return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen);
-  SCell       *pCell;
-  int          cret;
-  void        *pVal;
-  SCellDecoder cd;
-
-  tdbBtcOpen(&btc, pBt);
-
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
-
-  if (cret) {
-    return cret;
-  }
-
-  pCell = tdbPageGetCell(btc.pPage, btc.idx);
-  tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
-
-  *vLen = cd.vLen;
-  pVal = TDB_REALLOC(*ppVal, *vLen);
-  if (pVal == NULL) {
-    return -1;
-  }
-
-  *ppVal = pVal;
-  memcpy(*ppVal, cd.pVal, cd.vLen);
-  return 0;
 }
 
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
   SBTC         btc;
   SCell       *pCell;
   int          cret;
-  void        *pTKey;
+  int          ret;
-  void        *pTVal;
+  void        *pTKey = NULL;
+  void        *pTVal = NULL;
   SCellDecoder cd;
 
   tdbBtcOpen(&btc, pBt);
 
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+  }
+
   if (cret) {
-    return cret;
+    tdbBtcClose(&btc);
+    return -1;
   }
 
   pCell = tdbPageGetCell(btc.pPage, btc.idx);
   tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
 
-  pTKey = TDB_REALLOC(*ppKey, cd.kLen);
+  if (ppKey) {
-  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+    pTKey = TDB_REALLOC(*ppKey, cd.kLen);
-
+    if (pTKey == NULL) {
-  if (pTKey == NULL || pTVal == NULL) {
+      tdbBtcClose(&btc);
-    TDB_FREE(pTKey);
+      ASSERT(0);
-    TDB_FREE(pTVal);
+      return -1;
+    }
+    *ppKey = pTKey;
+    *pkLen = cd.kLen;
+    memcpy(*ppKey, cd.pKey, cd.kLen);
   }
 
-  *ppKey = pTKey;
+  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+  if (pTVal == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
   *ppVal = pTVal;
-  *pkLen = cd.kLen;
   *vLen = cd.vLen;
-
-  memcpy(*ppKey, cd.pKey, cd.kLen);
   memcpy(*ppVal, cd.pVal, cd.vLen);
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
@@ -300,7 +312,8 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
     return -1;
   }
 
-  // TODO: Unref the page
+  // TODO: here still has problem
+  tdbPagerReturnPage(pBt->pPager, pPage);
 
   ASSERT(pgno != 0);
   pBt->root = pgno;
@@ -371,17 +384,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
   return 0;
 }
 
-#ifndef TDB_BTREE_BALANCE
+// TDB_BTREE_BALANCE =====================
-typedef struct {
-  SBTree *pBt;
-  SPage  *pParent;
-  int     idx;
-  i8      nOld;
-  SPage  *pOldPages[3];
-  i8      nNewPages;
-  SPage  *pNewPages[5];
-} SBtreeBalanceHelper;
-
 static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
   SPager           *pPager;
   SPage            *pChild;
@@ -408,6 +411,13 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
     ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno;
   }
 
+  ret = tdbPagerWrite(pPager, pChild);
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
+  }
+
   // Copy the root page content to the child page
   tdbPageCopy(pRoot, pChild);
 
@@ -472,6 +482,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         ASSERT(0);
         return -1;
       }
+
+      ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
+      if (ret < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
     }
     // copy the parent key out if child pages are not leaf page
     childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]);
@@ -492,6 +509,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
       }
       rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno;
     }
+
+    ret = tdbPagerWrite(pBt->pPager, pParent);
+    if (ret < 0) {
+      // TODO
+      ASSERT(0);
+      return -1;
+    }
+
     // drop the cells on parent page
     for (int i = 0; i < nOlds; i++) {
       nCells = TDB_PAGE_TOTAL_CELLS(pParent);
@@ -619,6 +644,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         if (ret < 0) {
           ASSERT(0);
         }
+
+        ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
+        if (ret < 0) {
+          // TODO
+          ASSERT(0);
+          return -1;
+        }
       }
     }
 
@@ -732,14 +764,24 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
     }
   }
 
+  // TODO: here is not corrent for drop case
+  for (int i = 0; i < nNews; i++) {
+    if (i < nOlds) {
+      tdbPagerReturnPage(pBt->pPager, pOlds[i]);
+    } else {
+      tdbPagerReturnPage(pBt->pPager, pNews[i]);
+    }
+  }
+
   return 0;
 }
 
 static int tdbBtreeBalance(SBTC *pBtc) {
   int    iPage;
+  int    ret;
+  int    nFree;
   SPage *pParent;
   SPage *pPage;
-  int    ret;
   u8     flags;
   u8     leaf;
   u8     root;
@@ -750,10 +792,11 @@ static int tdbBtreeBalance(SBTC *pBtc) {
     pPage = pBtc->pPage;
     leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
     root = TDB_BTREE_PAGE_IS_ROOT(pPage);
+    nFree = TDB_PAGE_FREE_SIZE(pPage);
 
     // when the page is not overflow and not too empty, the balance work
     // is finished. Just break out the balance loop.
-    if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) {
+    if (pPage->nOverflow == 0 && nFree < TDB_PAGE_USABLE_SIZE(pPage) * 2 / 3) {
       break;
     }
 
@@ -781,6 +824,8 @@ static int tdbBtreeBalance(SBTC *pBtc) {
         return -1;
       }
 
+      tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+
       pBtc->iPage--;
       pBtc->pPage = pBtc->pgStack[pBtc->iPage];
     }
@@ -788,7 +833,7 @@ static int tdbBtreeBalance(SBTC *pBtc) {
 
   return 0;
 }
-#endif
+// TDB_BTREE_BALANCE
 
 // TDB_BTREE_CELL =====================
 static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const void *pKey, int kLen, const void *pVal,
@@ -1028,12 +1073,11 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
     // move upward
     for (;;) {
-      if (pBtc->iPage == 0) {
+      if (pBtc->iPage == iPage) {
         pBtc->idx = 0;
         break;
       }
 
-      if (pBtc->iPage < iPage) break;
       tdbBtcMoveUpward(pBtc);
     }
   }
@@ -1056,6 +1100,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
 int tdbBtcMoveToLast(SBTC *pBtc) {
   int     ret;
+  int     nCells;
   SBTree *pBt;
   SPager *pPager;
   SPgno   pgno;
@@ -1071,27 +1116,56 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
       return -1;
     }
 
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
     pBtc->iPage = 0;
+    if (nCells > 0) {
+      pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells;
+    } else {
+      // no data at all, point to an invalid position
+      ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
+      pBtc->idx = -1;
+      return 0;
+    }
   } else {
-    // move from a position
+    int iPage = 0;
-    ASSERT(0);
+
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage]));
+      nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]);
+      if (pBtc->idxStack[iPage] != nCells) break;
+    }
+
+    // move upward
+    for (;;) {
+      if (pBtc->iPage == iPage) {
+        if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
+        } else {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+        }
+        break;
+      }
+
+      tdbBtcMoveUpward(pBtc);
+    }
   }
 
   // move downward
   for (;;) {
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
-      // TODO: handle empty case
-      ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0);
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
-      break;
-    } else {
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-      ret = tdbBtcMoveDownward(pBtc);
+    ret = tdbBtcMoveDownward(pBtc);
-      if (ret < 0) {
+    if (ret < 0) {
-        ASSERT(0);
+      ASSERT(0);
-        return -1;
+      return -1;
-      }
+    }
+
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+      pBtc->idx = nCells - 1;
+    } else {
+      pBtc->idx = nCells;
     }
   }
 
@@ -1104,6 +1178,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   void        *pKey, *pVal;
   int          ret;
 
+  // current cursor points to an invalid position
   if (pBtc->idx < 0) {
     return -1;
   }
@@ -1134,12 +1209,17 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   memcpy(pVal, cd.pVal, cd.vLen);
 
   ret = tdbBtcMoveToNext(pBtc);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
   return 0;
 }
 
 static int tdbBtcMoveToNext(SBTC *pBtc) {
   int    nCells;
+  int    ret;
   SCell *pCell;
 
   ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
@@ -1151,39 +1231,35 @@ static int tdbBtcMoveToNext(SBTC *pBtc) {
     return 0;
   }
 
-  if (pBtc->iPage == 0) {
+  // move upward
-    pBtc->idx = -1;
-    return 0;
-  }
-
-  // Move upward
   for (;;) {
-    tdbBtcMoveUpward(pBtc);
-    pBtc->idx++;
-
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
-    if (pBtc->idx <= nCells) {
-      break;
-    }
-
     if (pBtc->iPage == 0) {
       pBtc->idx = -1;
       return 0;
     }
-  }
 
-  // Move downward
+    tdbBtcMoveUpward(pBtc);
-  for (;;) {
+    pBtc->idx++;
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-    tdbBtcMoveDownward(pBtc);
+    ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
-    pBtc->idx = 0;
+    if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) {
-
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
       break;
     }
   }
 
+  // move downward
+  for (;;) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
+
+    ret = tdbBtcMoveDownward(pBtc);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    pBtc->idx = 0;
+  }
+
   return 0;
 }
 
@@ -1230,91 +1306,145 @@ static int tdbBtcMoveUpward(SBTC *pBtc) {
 }
 
 static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
-  int     ret;
+  int          ret;
-  SBTree *pBt;
+  int          nCells;
-  SPager *pPager;
+  int          c;
+  SBTree      *pBt;
+  SCell       *pCell;
+  SPager      *pPager;
+  SCellDecoder cd = {0};
 
   pBt = pBtc->pBt;
   pPager = pBt->pPager;
 
   if (pBtc->iPage < 0) {
-    ASSERT(pBtc->iPage == -1);
+    // move from a clear cursor
-    ASSERT(pBtc->idx == -1);
-
-    // Move from the root
     ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
     if (ret < 0) {
+      // TODO
       ASSERT(0);
-      return -1;
-    }
-
-    pBtc->iPage = 0;
-
-    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) {
-      // Current page is empty
-      // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF));
       return 0;
     }
 
-    for (;;) {
+    pBtc->iPage = 0;
-      int          lidx, ridx, midx, c, nCells;
+    pBtc->idx = -1;
-      SCell       *pCell;
+    // for empty tree, just return with an invalid position
-      SPage       *pPage;
+    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0;
-      SCellDecoder cd = {0};
+  } else {
+    SPage *pPage;
+    int    idx;
+    int    iPage = 0;
 
-      pPage = pBtc->pPage;
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      pPage = pBtc->pgStack[iPage];
+      idx = pBtc->idxStack[iPage];
       nCells = TDB_PAGE_TOTAL_CELLS(pPage);
-      lidx = 0;
-      ridx = nCells - 1;
 
-      ASSERT(nCells > 0);
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage));
 
-      for (;;) {
+      // check if key <= current position
-        if (lidx > ridx) break;
+      if (idx < nCells) {
-
+        pCell = tdbPageGetCell(pPage, idx);
-        midx = (lidx + ridx) >> 1;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-
-        pCell = tdbPageGetCell(pPage, midx);
-        ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
-        if (ret < 0) {
-          // TODO: handle error
-          ASSERT(0);
-          return -1;
-        }
-
-        // Compare the key values
         c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        if (c < 0) {
+        if (c > 0) break;
-          /* input-key < cell-key */
-          ridx = midx - 1;
-        } else if (c > 0) {
-          /* input-key > cell-key */
-          lidx = midx + 1;
-        } else {
-          /* input-key == cell-key */
-          break;
-        }
       }
 
-      // Move downward or break
+      // check if key > current - 1 position
-      u8 leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
+      if (idx > 0) {
-      if (leaf) {
+        pCell = tdbPageGetCell(pPage, idx - 1);
-        pBtc->idx = midx;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-        *pCRst = c;
+        c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        break;
+        if (c <= 0) break;
-      } else {
-        if (c <= 0) {
-          pBtc->idx = midx;
-        } else {
-          pBtc->idx = midx + 1;
-        }
-        tdbBtcMoveDownward(pBtc);
       }
     }
 
-  } else {
+    // move upward
-    // TODO: Move the cursor from a some position instead of a clear state
+    for (;;) {
-    ASSERT(0);
+      if (pBtc->iPage == iPage) break;
+      tdbBtcMoveUpward(pBtc);
+    }
+  }
+
+  // search downward to the leaf
+  for (;;) {
+    int    lidx, ridx, midx;
+    SPage *pPage;
+
+    pPage = pBtc->pPage;
+    nCells = TDB_PAGE_TOTAL_CELLS(pPage);
+    lidx = 0;
+    ridx = nCells - 1;
+
+    ASSERT(nCells > 0);
+    ASSERT(pBtc->idx == -1);
+
+    // compare first cell
+    midx = lidx;
+    pCell = tdbPageGetCell(pPage, midx);
+    tdbBtreeDecodeCell(pPage, pCell, &cd);
+    c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+    if (c <= 0) {
+      ridx = lidx - 1;
+    } else {
+      lidx = lidx + 1;
+    }
+
+    // compare last cell
+    if (lidx <= ridx) {
+      midx = ridx;
+      pCell = tdbPageGetCell(pPage, midx);
+      tdbBtreeDecodeCell(pPage, pCell, &cd);
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c >= 0) {
+        lidx = ridx + 1;
+      } else {
+        ridx = ridx - 1;
+      }
+    }
+
+    // binary search
+    for (;;) {
+      if (lidx > ridx) break;
+
+      midx = (lidx + ridx) >> 1;
+
+      pCell = tdbPageGetCell(pPage, midx);
+      ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
+      if (ret < 0) {
+        // TODO: handle error
+        ASSERT(0);
+        return -1;
+      }
+
+      // Compare the key values
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c < 0) {
+        // pKey < cd.pKey
+        ridx = midx - 1;
+      } else if (c > 0) {
+        // pKey > cd.pKey
+        lidx = midx + 1;
+      } else {
+        // pKey == cd.pKey
+        break;
+      }
+    }
+
+    // keep search downward or break
+    if (TDB_BTREE_PAGE_IS_LEAF(pPage)) {
+      pBtc->idx = midx;
+      *pCRst = c;
+      break;
+    } else {
+      if (c <= 0) {
+        pBtc->idx = midx;
+      } else {
+        pBtc->idx = midx + 1;
+      }
+      tdbBtcMoveDownward(pBtc);
+    }
   }
 
   return 0;

source/libs/tdb/src/db/tdbDb.c

@@ -49,6 +49,8 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
     if (ret < 0) {
       return -1;
     }
+
+    tdbEnvAddPager(pEnv, pPager);
   }
 
   ASSERT(pPager != NULL);
@@ -74,22 +76,7 @@ int tdbDbDrop(TDB *pDb) {
 }
 
 int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  SBTC  btc;
+  return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen);
-  SBTC *pCur;
-  int   ret;
-
-  pCur = &btc;
-  ret = tdbBtcOpen(pCur, pDb->pBt);
-  if (ret < 0) {
-    return -1;
-  }
-
-  ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen);
-  if (ret < 0) {
-    return -1;
-  }
-
-  return 0;
 }
 
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {

source/libs/tdb/src/db/tdbEnv.c

@@ -19,6 +19,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
   TENV *pEnv;
   int   dsize;
   int   zsize;
+  int   tsize;
   u8   *pPtr;
   int   ret;
 
@@ -53,6 +54,14 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
     return -1;
   }
 
+  pEnv->nPgrHash = 8;
+  tsize = sizeof(SPager *) * pEnv->nPgrHash;
+  pEnv->pgrHash = TDB_REALLOC(pEnv->pgrHash, tsize);
+  if (pEnv->pgrHash == NULL) {
+    return -1;
+  }
+  memset(pEnv->pgrHash, 0, tsize);
+
   mkdir(rootDir, 0755);
 
   *ppEnv = pEnv;
@@ -64,7 +73,99 @@ int tdbEnvClose(TENV *pEnv) {
   return 0;
 }
 
-SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
+int tdbBegin(TENV *pEnv) {
-  // TODO
+  SPager *pPager;
-  return NULL;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerBegin(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbCommit(TENV *pEnv) {
+  SPager *pPager;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerCommit(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbRollback(TENV *pEnv) {
+  ASSERT(0);
+  return 0;
+}
+
+SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
+  u32      hash;
+  SPager **ppPager;
+
+  hash = tdbCstringHash(fname);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
+  }
+
+  return *ppPager;
+}
+
+void tdbEnvAddPager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // rehash if neccessary
+  if (pEnv->nPager + 1 > pEnv->nPgrHash) {
+    // TODO
+  }
+
+  // add to list
+  pPager->pNext = pEnv->pgrList;
+  pEnv->pgrList = pPager;
+
+  // add to hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  pPager->pHashNext = *ppPager;
+  *ppPager = pPager;
+
+  // increase the counter
+  pEnv->nPager++;
+}
+
+void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // remove from the list
+  for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // remove from hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // decrease the counter
+  pEnv->nPager--;
+
+  // rehash if necessary
+  if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) {
+    // TODO
+  }
 }

source/libs/tdb/src/db/tdbPCache.c

@@ -34,18 +34,6 @@ struct SPCache {
   })
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
-// For page ref
-#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
-#if 0
-#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
-#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
-#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
-#else
-#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
-#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
-#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
-#endif
-
 static int    tdbPCacheOpenImpl(SPCache *pCache);
 static void   tdbPCacheInitLock(SPCache *pCache);
 static void   tdbPCacheClearLock(SPCache *pCache);
@@ -107,12 +95,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
   ASSERT(nRef >= 0);
 
   if (nRef == 0) {
-    if (1 /*TODO: page still clean*/) {
+    tdbPCacheUnpinPage(pCache, pPage);
-      tdbPCacheUnpinPage(pCache, pPage);
-    } else {
-      // TODO
-      ASSERT(0);
-    }
   }
 }
 
@@ -192,6 +175,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
 
   tdbPCacheLock(pCache);
 
+  ASSERT(!pPage->isDirty);
+
   nRef = TDB_GET_PAGE_REF(pPage);
   ASSERT(nRef >= 0);
   if (nRef == 0) {

source/libs/tdb/src/db/tdbPager.c

@@ -15,20 +15,6 @@
 
 #include "tdbInt.h"
 
-struct SPager {
-  char    *dbFileName;
-  char    *jFileName;
-  int      pageSize;
-  uint8_t  fid[TDB_FILE_ID_LEN];
-  tdb_fd_t fd;
-  tdb_fd_t jfd;
-  SPCache *pCache;
-  SPgno    dbFileSize;
-  SPgno    dbOrigSize;
-  SPage   *pDirty;
-  u8       inTran;
-};
-
 typedef struct __attribute__((__packed__)) {
   u8    hdrString[16];
   u16   pageSize;
@@ -41,9 +27,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
 
 #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
 static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
 static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
 static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
 
@@ -131,26 +116,36 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
 }
 
 int tdbPagerWrite(SPager *pPager, SPage *pPage) {
-  int ret;
+  int     ret;
+  SPage **ppPage;
 
+  ASSERT(pPager->inTran);
+#if 0
   if (pPager->inTran == 0) {
     ret = tdbPagerBegin(pPager);
     if (ret < 0) {
       return -1;
     }
   }
+#endif
 
   if (pPage->isDirty) return 0;
 
+  // ref page one more time so the page will not be release
+  TDB_REF_PAGE(pPage);
+
   // Set page as dirty
   pPage->isDirty = 1;
 
-  // Add page to dirty list
+  // Add page to dirty list(TODO: NOT use O(n^2) algorithm)
-  // TODO: sort the list according to the page number
+  for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
-  pPage->pDirtyNext = pPager->pDirty;
+       ppPage = &((*ppPage)->pDirtyNext)) {
-  pPager->pDirty = pPage;
+  }
+  ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
+  pPage->pDirtyNext = *ppPage;
+  *ppPage = pPage;
 
-  // Write page to journal
+  // Write page to journal if neccessary
   if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
     ret = tdbPagerWritePageToJournal(pPager, pPage);
     if (ret < 0) {
@@ -184,54 +179,46 @@ int tdbPagerCommit(SPager *pPager) {
   SPage *pPage;
   int    ret;
 
-  // Begin commit
+  // sync the journal file
-  {
+  ret = tdbOsFSync(pPager->jfd);
-    // TODO: Sync the journal file (Here or when write ?)
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
   }
 
-  for (;;) {
+  // loop to write the dirty pages to file
-    pPage = pPager->pDirty;
+  for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
-
+    // TODO: update the page footer
-    if (pPage == NULL) break;
-
     ret = tdbPagerWritePageToDB(pPager, pPage);
     if (ret < 0) {
       ASSERT(0);
       return -1;
     }
+  }
 
+  // release the page
+  for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
     pPager->pDirty = pPage->pDirtyNext;
     pPage->pDirtyNext = NULL;
 
-    // TODO: release the page
+    pPage->isDirty = 0;
+
+    tdbPCacheRelease(pPager->pCache, pPage);
   }
 
+  // sync the db file
   tdbOsFSync(pPager->fd);
 
+  // remote the journal file
   tdbOsClose(pPager->jfd);
   tdbOsRemove(pPager->jFileName);
-  // pPager->jfd = -1;
+  pPager->dbOrigSize = pPager->dbFileSize;
+  pPager->inTran = 0;
 
   return 0;
 }
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
-  i64 offset;
-  int ret;
-
-  ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
-
-  offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
-  ret = tdbOsPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
-  if (ret < 0) {
-    // TODO: handle error
-    return -1;
-  }
-  return 0;
-}
-
-int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; }
-
 int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
   SPage *pPage;
   SPgid  pgid;
@@ -247,7 +234,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
 
   // Initialize the page if need
   if (!TDB_PAGE_INITIALIZED(pPage)) {
-    ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+    ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 1);
     if (ret < 0) {
       return -1;
     }
@@ -284,7 +271,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   ASSERT(!TDB_PAGE_INITIALIZED(pPage));
 
   // Initialize the page if need
-  ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+  ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
   if (ret < 0) {
     return -1;
   }
@@ -332,10 +319,11 @@ static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
   return 0;
 }
 
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) {
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage) {
   int ret;
   int lcode;
   int nLoops;
+  i64 nRead;
 
   lcode = TDB_TRY_LOCK_PAGE(pPage);
   if (lcode == P_LOCK_SUCC) {
@@ -344,6 +332,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
       return 0;
     }
 
+    if (loadPage) {
+      nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * TDB_PAGE_PGNO(pPage));
+      if (nRead < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      } else if (nRead < pPage->pageSize) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
+    }
+
     ret = (*initPage)(pPage, arg);
     if (ret < 0) {
       TDB_UNLOCK_PAGE(pPage);

source/libs/tdb/src/db/tdbTxn.c

@@ -15,17 +15,29 @@
 
 #include "tdbInt.h"
 
-int tdbTxnBegin(TENV *pEnv) {
+// int tdbTxnBegin(TENV *pEnv) {
-  // TODO
+//   // TODO
-  return 0;
+//   return 0;
-}
+// }
 
-int tdbTxnCommit(TENV *pEnv) {
+// int tdbTxnCommit(TENV *pEnv) {
-  // TODO
+//   SPager *pPager = NULL;
-  return 0;
+//   int     ret;
-}
 
-int tdbTxnRollback(TENV *pEnv) {
+//   for (;;) {
-  // TODO
+//     break;
-  return 0;
+//     ret = tdbPagerCommit(pPager);
-}
+//     if (ret < 0) {
+//       ASSERT(0);
+//       return -1;
+//     }
+//   }
+
+//   // TODO
+//   return 0;
+// }
+
+// int tdbTxnRollback(TENV *pEnv) {
+//   // TODO
+//   return 0;
+// }

source/libs/tdb/src/inc/tdbBtree.h

@@ -40,7 +40,7 @@ struct SBTC {
 // SBTree
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
-int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen);
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 

source/libs/tdb/src/inc/tdbEnv.h

@@ -25,11 +25,20 @@ typedef struct STEnv {
   char    *jfname;
   int      jfd;
   SPCache *pCache;
+  SPager  *pgrList;
+  int      nPager;
+  int      nPgrHash;
+  SPager **pgrHash;
 } TENV;
 
 int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
 int tdbEnvClose(TENV *pEnv);
+int tdbBegin(TENV *pEnv);
+int tdbCommit(TENV *pEnv);
+int tdbRollback(TENV *pEnv);
 
+void    tdbEnvAddPager(TENV *pEnv, SPager *pPager);
+void    tdbEnvRemovePager(TENV *pEnv, SPager *pPager);
 SPager *tdbEnvGetPager(TENV *pEnv, const char *fname);
 
 #ifdef __cplusplus

source/libs/tdb/src/inc/tdbInt.h

@@ -16,8 +16,6 @@
 #ifndef _TD_TDB_INTERNAL_H_
 #define _TD_TDB_INTERNAL_H_
 
-#include "os.h"
-
 #include "tdb.h"
 
 #ifdef __cplusplus
@@ -91,23 +89,6 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
 // dbname
 #define TDB_MAX_DBNAME_LEN 24
 
-// tdb_log
-#define tdbError(var)
-
-#define TERR_A(val, op, flag)  \
-  do {                         \
-    if (((val) = (op)) != 0) { \
-      goto flag;               \
-    }                          \
-  } while (0)
-
-#define TERR_B(val, op, flag)     \
-  do {                            \
-    if (((val) = (op)) == NULL) { \
-      goto flag;                  \
-    }                             \
-  } while (0)
-
 #define TDB_VARIANT_LEN ((int)-1)
 
 typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);

source/libs/tdb/src/inc/tdbOs.h

@@ -24,6 +24,8 @@ extern "C" {
 #define TDB_FOR_TDENGINE
 
 #ifdef TDB_FOR_TDENGINE
+#include "os.h"
+#include "thash.h"
 
 // For memory -----------------
 #define tdbOsMalloc  taosMemoryMalloc
@@ -95,7 +97,11 @@ typedef int tdb_fd_t;
 
 #define tdbOsOpen(PATH, OPTION, MODE) open((PATH), (OPTION), (MODE))
 
-#define tdbOsClose close
+#define tdbOsClose(FD) \
+  do {                 \
+    close(FD);         \
+    (FD) = -1;         \
+  } while (0)
 
 i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes);
 i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset);

source/libs/tdb/src/inc/tdbPCache.h

@@ -33,6 +33,18 @@ extern "C" {
   SPager *pPager;       \
   SPgid   pgid;
 
+// For page ref
+#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
+#if 0
+#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
+#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
+#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
+#else
+#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
+#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
+#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
+#endif
+
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
 SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);

source/libs/tdb/src/inc/tdbPage.h

@@ -100,6 +100,7 @@ struct SPage {
 // APIs
 #define TDB_PAGE_TOTAL_CELLS(pPage)        ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
 #define TDB_PAGE_USABLE_SIZE(pPage)        ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
+#define TDB_PAGE_FREE_SIZE(pPage)          (*(pPage)->pPageMethods->getFreeBytes)(pPage)
 #define TDB_PAGE_PGNO(pPage)               ((pPage)->pgid.pgno)
 #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset)
 #define TDB_PAGE_OFFSET_SIZE(pPage)        ((pPage)->pPageMethods->szOffset)

source/libs/tdb/src/inc/tdbPager.h

@@ -20,13 +20,28 @@
 extern "C" {
 #endif
 
+struct SPager {
+  char    *dbFileName;
+  char    *jFileName;
+  int      pageSize;
+  uint8_t  fid[TDB_FILE_ID_LEN];
+  tdb_fd_t fd;
+  tdb_fd_t jfd;
+  SPCache *pCache;
+  SPgno    dbFileSize;
+  SPgno    dbOrigSize;
+  SPage   *pDirty;
+  u8       inTran;
+  SPager  *pNext;      // used by TENV
+  SPager  *pHashNext;  // used by TENV
+};
+
 int  tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
 int  tdbPagerClose(SPager *pPager);
 int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
 int  tdbPagerWrite(SPager *pPager, SPage *pPage);
 int  tdbPagerBegin(SPager *pPager);
 int  tdbPagerCommit(SPager *pPager);
-int  tdbPagerGetPageSize(SPager *pPager);
 int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage);

source/libs/tdb/src/inc/tdbTxn.h

@@ -28,10 +28,6 @@ struct STxn {
   void *xArg;
 };
 
-int tdbTxnBegin(TENV *pEnv);
-int tdbTxnCommit(TENV *pEnv);
-int tdbTxnRollback(TENV *pEnv);
-
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/src/inc/tdbUtil.h

@@ -101,6 +101,8 @@ static inline int tdbGetVarInt(const u8 *p, int *v) {
   return n;
 }
 
+static inline u32 tdbCstringHash(const char *s) { return MurmurHash3_32(s, strlen(s)); }
+
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/test/tdbTest.cpp

@@ -118,10 +118,10 @@ TEST(tdb_test, simple_test) {
   TENV          *pEnv;
   TDB           *pDb;
   FKeyComparator compFunc;
-  int            nData = 1000000;
+  int            nData = 50000000;
 
   // Open Env
-  ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv);
+  ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
   GTEST_ASSERT_EQ(ret, 0);
 
   // Create a database
@@ -134,36 +134,23 @@ TEST(tdb_test, simple_test) {
     char val[64];
 
     {  // Insert some data
-      int       i = 1;
+      for (int i = 1; i <= nData;) {
-      SPoolMem *pPool;
+        tdbBegin(pEnv);
-      int       memPoolCapacity = 16 * 1024;
 
-      pPool = openPool();
+        for (int k = 0; k < 2000; k++) {
-
+          sprintf(key, "key%d", i);
-      tdbTxnBegin(pEnv);
+          sprintf(val, "value%d", i);
-
+          ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-      for (;;) {
+          GTEST_ASSERT_EQ(ret, 0);
-        if (i > nData) break;
+          i++;
-
-        sprintf(key, "key%d", i);
-        sprintf(val, "value%d", i);
-        ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-        GTEST_ASSERT_EQ(ret, 0);
-
-        if (pPool->size >= memPoolCapacity) {
-          tdbTxnCommit(pEnv);
-
-          clearPool(pPool);
-
-          tdbTxnBegin(pEnv);
         }
 
-        i++;
+        tdbCommit(pEnv);
       }
-
-      closePool(pPool);
     }
 
+    tdbCommit(pEnv);
+
     {  // Query the data
       void *pVal = NULL;
       int   vLen;
@@ -173,6 +160,7 @@ TEST(tdb_test, simple_test) {
         sprintf(val, "value%d", i);
 
         ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen);
+        ASSERT(ret == 0);
         GTEST_ASSERT_EQ(ret, 0);
 
         GTEST_ASSERT_EQ(vLen, strlen(val));

source/os/src/osFile.c

@@ -294,7 +294,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&((*ppFile)->rwlock));
 #endif
-  if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
+  if (ppFile == NULL || *ppFile == NULL) {
     return 0;
   }
   if ((*ppFile)->fp != NULL) {

source/os/src/osMemory.c

@@ -23,15 +23,22 @@
 
 #define TD_MEMORY_STACK_TRACE_DEPTH 10
 
+typedef struct TdMemoryInfo *TdMemoryInfoPtr;
+
 typedef struct TdMemoryInfo {
   int32_t symbol;
   int32_t memorySize;
   void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH];     // gdb: disassemble /m 0xXXX
-} *TdMemoryInfoPtr , TdMemoryInfo;
+  // TdMemoryInfoPtr pNext;
+  // TdMemoryInfoPtr pPrev;
+} TdMemoryInfo;
+
+// static TdMemoryInfoPtr GlobalMemoryPtr = NULL;
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-
+  #define tstrdup(str) _strdup(str)
 #else
+  #define tstrdup(str) strdup(str)
 
 #include<execinfo.h>
 
@@ -129,6 +136,26 @@ void *taosMemoryRealloc(void *ptr, int32_t size) {
 #endif
 }
 
+void *taosMemoryStrDup(void *ptr) {
+#ifdef USE_TD_MEMORY
+  if (ptr == NULL) return NULL;
+  
+  TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
+  assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL);
+
+  void *tmp = tstrdup((const char *)pTdMemoryInfo);
+  if (tmp == NULL) return NULL;
+  
+  memcpy(tmp, pTdMemoryInfo, sizeof(TdMemoryInfo));
+  taosBackTrace(((TdMemoryInfoPtr)tmp)->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH);
+
+  return (char*)tmp  + sizeof(TdMemoryInfo);
+#else
+  return tstrdup((const char *)ptr);
+#endif
+}
+
+
 void taosMemoryFree(const void *ptr) {
   if (ptr == NULL) return;
 

tests/script/jenkins/basic.txt

@@ -23,8 +23,8 @@
 ./test.sh -f tsim/insert/null.sim
 
 # ---- parser
-#./test.sh -f tsim/parser/groupby-basic.sim
+./test.sh -f tsim/parser/groupby-basic.sim
-#./test.sh -f tsim/parser/fourArithmetic-basic.sim
+./test.sh -f tsim/parser/fourArithmetic-basic.sim
 
 # ---- query
 ./test.sh -f tsim/query/interval.sim
@@ -40,6 +40,7 @@
 # ---- tmq
 ./test.sh -f tsim/tmq/basic.sim
 ./test.sh -f tsim/tmq/basic1.sim
+./test.sh -f tsim/tmq/oneTopic.sim
 
 # --- stable
 ./test.sh -f tsim/stable/disk.sim

tests/script/runAllSimCases.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##################################################
+# 
+# Do simulation test 
+#
+##################################################
+
+set -e
+#set -x
+
+while read line
+do
+  firstChar=`echo ${line:0:1}`
+  if [[ -n "$line" ]]  && [[ $firstChar != "#" ]]; then
+    echo "======== $line ========"
+    $line 
+  fi
+done < ./jenkins/basic.txt
+
+

tests/script/tsim/db/alter_option.sim

@@ -66,7 +66,7 @@ print ============= create database
 #  | REPLICA value      [1 | 3]
 #  | WAL value          [1 | 2]
 
-sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1
+sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1
 sql show databases
 print rows: $rows
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
@@ -92,10 +92,10 @@ endi
 if $data5_db != 1 then  # quorum
   return -1
 endi
-if $data6_db != 240 then  # days
+if $data6_db != 345600 then  # days
   return -1
 endi
-if $data7_db != 1000,1000,1000 then # keep
+if $data7_db != 1440000,1440000,1440000 then # keep
   return -1
 endi
 if $data8_db != 3 then  # cache
@@ -138,7 +138,7 @@ sql_error alter database db ntables 0
 sql_error alter database db ntables 1
 sql_error alter database db ntables 10
 
-#print ============== modify replica
+#print ============== modify replica        # TD-14409
 sql_error alter database db replica 2
 sql_error alter database db replica 5
 sql_error alter database db replica -1
@@ -186,7 +186,7 @@ print ============== modify keep
 sql alter database db keep 2000
 sql show databases
 print keep $data7_db
-if $data7_db != 1000,1000,2000 then
+if $data7_db != 2000,2000,2000 then
   return -1
 endi
 
@@ -270,7 +270,7 @@ if $data12_db != 2 then
   return -1
 endi
 
-#sql_error alter database db wal 0     # TD-14436
+sql_error alter database db wal 0     # TD-14436
 sql_error alter database db wal 3
 sql_error alter database db wal 100
 sql_error alter database db wal -1

tests/script/tsim/db/basic6.sim

@@ -15,7 +15,7 @@ $tb = $tbPrefix . $i
 
 print =============== step1
 # quorum  presicion 
-sql create database $db vgroups 8 replica 1 days 20 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
+sql create database $db vgroups 8 replica 1 days 2880 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
 sql show databases
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1 $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
@@ -35,7 +35,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data06 != 20 then 
+if $data06 != 2880 then 
   return -1
 endi
 if $data07 != 3650,3650,3650 then 
@@ -67,7 +67,7 @@ print =============== step4
 sql_error drop database $db
 
 print =============== step5
-sql create database $db replica 1 days 15 keep 1500
+sql create database $db replica 1 days 21600 keep 2160000
 sql show databases
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
 if $data00 != $db then 
@@ -79,7 +79,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data06 != 15 then 
+if $data06 != 21600 then 
   return -1
 endi
 

tests/script/tsim/parser/fourArithmetic-basic.sim

@@ -78,6 +78,10 @@ while $i < $tbNum
   $tstart = 1640966400000
 endw
 
+
+$loop_test = 0
+loop_test_pos:
+
 sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
 print ===> rows: $rows 
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
@@ -107,4 +111,31 @@ endi
 if $data93 != 8.000000000 then
  return -1
 endi
+
+if $loop_test == 0 then
+  print =============== stop and restart taosd
+  system sh/exec.sh -n dnode1 -s stop -x SIGINT
+  system sh/exec.sh -n dnode1 -s start
+  
+  $loop_cnt = 0
+  check_dnode_ready_0:
+  	$loop_cnt = $loop_cnt + 1
+  	sleep 200
+  	if $loop_cnt == 10 then
+  	  print ====> dnode not ready!
+  		return -1
+  	endi
+  sql show dnodes
+  print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+  if $data00 != 1 then
+    return -1
+  endi
+  if $data04 != ready then
+    goto check_dnode_ready_0
+  endi
+
+  $loop_test = 1  
+  goto loop_test_pos
+endi
+
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/groupby-basic.sim

@@ -45,7 +45,7 @@ $tstart = 1640966400000  # 2022-01-01 00:00:00.000
 print ==== create db, stable, ctables, insert data
 sql drop database if exists $db -x step1
 step1:
-sql create database if not exists $db keep 3650
+sql create database if not exists $db
 sql use $db
 
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
@@ -112,25 +112,26 @@ print $data90 $data91 $data92 $data93
 if $rows != 10 then
   return -1
 endi
-#if $data00 != 10 then
+if $data00 != 10 then
-# return -1
+ return -1
-#endi
+endi
 if $data01 != 0 then
   return -1
 endi
-#if $data10 != 10 then
+if $data10 != 10 then
-#  return -1
+  return -1
-#endi
+endi
 if $data11 != 1 then
   return -1
 endi
-#if $data90 != 10 then
+if $data90 != 10 then
-#  return -1
+  return -1
-#endi
+endi
 if $data91 != 9 then
   return -1
 endi
 
+print ==== select first(ts),c1 from group_tb0 group by c1;
 sql select first(ts),c1 from group_tb0 group by c1;
 print rows: $rows
 print $data00 $data01 $data02 $data03
@@ -142,19 +143,29 @@ if $row != 10 then
   return -1
 endi
 
-if $data00 != @2022-01-01 00:00:00.000@ then
+if $data00 != @22-01-01 00:00:00.000@ then
   return -1
 endi
 if $data01 != 0 then
   return -1
 endi
-if $data90 != @2022-01-01 00:00:00.009@ then
+if $data90 != @22-01-01 00:00:00.009@ then
   return -1
 endi
 if $data91 != 9 then
   return -1
 endi
 
+print ==== select first(ts),c1 from interval(5m) group_tb0 group by c1;
+sql select first(ts),c1 from group_tb0 group by c1;
+print rows: $rows
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data80 $data81 $data82 $data83
+print $data90 $data91 $data92 $data93
+
+return 
 
 sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
 if $row != 20 then

tests/script/tsim/query/interval-offset.sim

@@ -73,39 +73,45 @@ sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
 sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
 
 print ================ start query ======================
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
+sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(*) from ct1 interval(10s, 2s)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
+print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(*) from ct1 interval(10s, 2s)
 print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04
+print ===> rows0: $data00 $data01 $data02 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14
+print ===> rows1: $data10 $data11 $data12 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24
+print ===> rows2: $data20 $data21 $data22 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+print ===> rows3: $data30 $data31 $data32 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+print ===> rows4: $data40 $data41 $data42 $data45
 if $rows != 5 then
   return -1
 endi
-if $data00 != 1 then
+if $data00 != @22-01-01 01:00:52.000@ then
   return -1
 endi
-if $data40 != 1 then
+if $data02 != 10000 then
+  return -1
+endi
+if $data45 != 1 then
   return -1
 endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s)
+sql select _wstartts, _wendts, _wduration, _qstartts, _qendts,  count(*) from ct1 interval(10s, 2s) sliding(10s)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s)
+print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts,  count(*) from ct1 interval(10s, 2s) sliding(10s)
 print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04
+print ===> rows0: $data00 $data01 $data02 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14
+print ===> rows1: $data10 $data11 $data12 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24
+print ===> rows2: $data20 $data21 $data22 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+print ===> rows3: $data30 $data31 $data32 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+print ===> rows4: $data40 $data41 $data42 $data45
 if $rows != 5 then
   return -1
 endi
-if $data00 != 1 then
+if $data00 != @22-01-01 01:00:52.000@ then
   return -1
 endi
-if $data40 != 1 then
+if $data02 != 10000 then
+  return -1
+endi
+if $data45 != 1 then
   return -1
 endi
 
@@ -176,107 +182,81 @@ if $data70 != 1 then
   return -1
 endi
 
-sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from ct2 interval(1d, 2h) sliding(12h)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h)
-print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
-print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55
-print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65
-print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75
-if $rows != 8 then
-  return -1
-endi
-if $data05 != 1 then
-  return -1
-endi
-if $data15 != 2 then
-  return -1
-endi
-if $data75 != 1 then
-  return -1
-endi
-if $data02 != 86400000 then
-  return -1
-endi
-
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-# if $rows != 5 then
+  return -1
-#   return -1
+endi
-# endi
+if $data00 != @21-12-08 00:00:00.000@ then
-# f $data00 != 1 then
+  return -1
-#   return -1
+endi
-# endi
+if $data31 != 1 then
-# if $data40 != 1 then
+  return -1
-#   return -1
+endi
-# endi
+if $data34 != $data31 then
+  return -1
+endi
+if $data02 != 2678400000 then
+  return -1
+endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w)
+sql select  _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-#if $rows != 5 then
+  return -1
-#  return -1
+endi
-#endi
+if $data00 != @21-11-30 08:00:00.000@ then
-#if $data00 != 1 then
+  return -1
-#  return -1
+endi
-#endi
+if $data01 != NULL then
-#if $data40 != 1 then
+  return -1
-#  return -1
+endi
-#endi
+if $data31 != $data34 then
+  return -1
+endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*)  from ct3 interval(1n, 1w) sliding(4w)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(4w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-print ===> rows5: $data50 $data51 $data52 $data53 $data54
+  return -1
-print ===> rows6: $data60 $data61 $data62 $data63 $data64
+endi
-print ===> rows7: $data70 $data71 $data72 $data73 $data74
+if $data01 != NULL then
-#if $rows != 8 then
+  return -1
-#  return -1
+endi
-#endi
+if $data04 != 1 then
-#if $data00 != 2 then
+  return -1
-#  return -1
+endi
-#endi
-#if $data70 != 1 then
-#  return -1
-#endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-#if $rows != 5 then
+endi
-#  return -1
+if $data01 != 2 then
-#endi
+  return -1
-#if $data00 != 1 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data40 != 1 then
+endi
-#  return -1
-#endi
 
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n)
@@ -284,38 +264,31 @@ print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-#if $rows != 5 then
+endi
-#  return -1
+if $data00 != 2 then
-#endi
+  return -1
-#if $data00 != 1 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data40 != 1 then
+endi
-#  return -1
-#endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n) sliding(12n)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n) sliding(12n)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-print ===> rows5: $data50 $data51 $data52 $data53 $data54
+endi
-print ===> rows6: $data60 $data61 $data62 $data63 $data64
+if $data01 != 2 then
-print ===> rows7: $data70 $data71 $data72 $data73 $data74
+  return -1
-#if $rows != 8 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data00 != 2 then
+endi
-#  return -1
-#endi
-#if $data70 != 1 then
-#  return -1
-#endi
 
 #=================================================
 print =============== stop and restart taosd

tests/script/tsim/query/scalarFunction.sim

@@ -202,7 +202,7 @@ if $rows != $rowNum then
 endi
 
 print ====> sin
-#sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1  # TD-14426
+sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1
 sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> rows: $rows