merge from 3.0

include/common/tcommon.h

@@ -209,8 +209,8 @@ typedef struct SGroupbyExpr {
 } SGroupbyExpr;
 
 enum {
-  FUNC_PARAM_TYPE_VALUE = 0,
+  FUNC_PARAM_TYPE_VALUE = 0x1,
-  FUNC_PARAM_TYPE_COLUMN,
+  FUNC_PARAM_TYPE_COLUMN= 0x2,
 };
 
 typedef struct SFunctParam {

include/common/tmsg.h

@@ -524,6 +524,7 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
+  int8_t  replications;
 } SAlterDbReq;
 
 int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);

include/common/ttime.h

@@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
 
 int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
+char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/common/ttokendef.h

@@ -82,21 +82,21 @@
 #define TK_SINGLE_STABLE                   64
 #define TK_STREAM_MODE                     65
 #define TK_RETENTIONS                      66
-#define TK_TABLE                           67
+#define TK_NK_COMMA                        67
-#define TK_NK_LP                           68
+#define TK_TABLE                           68
-#define TK_NK_RP                           69
+#define TK_NK_LP                           69
-#define TK_STABLE                          70
+#define TK_NK_RP                           70
-#define TK_ADD                             71
+#define TK_STABLE                          71
-#define TK_COLUMN                          72
+#define TK_ADD                             72
-#define TK_MODIFY                          73
+#define TK_COLUMN                          73
-#define TK_RENAME                          74
+#define TK_MODIFY                          74
-#define TK_TAG                             75
+#define TK_RENAME                          75
-#define TK_SET                             76
+#define TK_TAG                             76
-#define TK_NK_EQ                           77
+#define TK_SET                             77
-#define TK_USING                           78
+#define TK_NK_EQ                           78
-#define TK_TAGS                            79
+#define TK_USING                           79
-#define TK_NK_DOT                          80
+#define TK_TAGS                            80
-#define TK_NK_COMMA                        81
+#define TK_NK_DOT                          81
 #define TK_COMMENT                         82
 #define TK_BOOL                            83
 #define TK_TINYINT                         84
@@ -131,69 +131,91 @@
 #define TK_FUNCTIONS                      113
 #define TK_INDEXES                        114
 #define TK_FROM                           115
-#define TK_LIKE                           116
+#define TK_ACCOUNTS                       116
-#define TK_INDEX                          117
+#define TK_APPS                           117
-#define TK_FULLTEXT                       118
+#define TK_CONNECTIONS                    118
-#define TK_FUNCTION                       119
+#define TK_LICENCE                        119
-#define TK_INTERVAL                       120
+#define TK_QUERIES                        120
-#define TK_TOPIC                          121
+#define TK_SCORES                         121
-#define TK_AS                             122
+#define TK_TOPICS                         122
-#define TK_DESC                           123
+#define TK_VARIABLES                      123
-#define TK_DESCRIBE                       124
+#define TK_LIKE                           124
-#define TK_RESET                          125
+#define TK_INDEX                          125
-#define TK_QUERY                          126
+#define TK_FULLTEXT                       126
-#define TK_EXPLAIN                        127
+#define TK_FUNCTION                       127
-#define TK_ANALYZE                        128
+#define TK_INTERVAL                       128
-#define TK_VERBOSE                        129
+#define TK_TOPIC                          129
-#define TK_NK_BOOL                        130
+#define TK_AS                             130
-#define TK_RATIO                          131
+#define TK_DESC                           131
-#define TK_NULL                           132
+#define TK_DESCRIBE                       132
-#define TK_NK_VARIABLE                    133
+#define TK_RESET                          133
-#define TK_NK_UNDERLINE                   134
+#define TK_QUERY                          134
-#define TK_ROWTS                          135
+#define TK_EXPLAIN                        135
-#define TK_TBNAME                         136
+#define TK_ANALYZE                        136
-#define TK_QSTARTTS                       137
+#define TK_VERBOSE                        137
-#define TK_QENDTS                         138
+#define TK_NK_BOOL                        138
-#define TK_WSTARTTS                       139
+#define TK_RATIO                          139
-#define TK_WENDTS                         140
+#define TK_COMPACT                        140
-#define TK_WDURATION                      141
+#define TK_VNODES                         141
-#define TK_BETWEEN                        142
+#define TK_IN                             142
-#define TK_IS                             143
+#define TK_OUTPUTTYPE                     143
-#define TK_NK_LT                          144
+#define TK_AGGREGATE                      144
-#define TK_NK_GT                          145
+#define TK_BUFSIZE                        145
-#define TK_NK_LE                          146
+#define TK_STREAM                         146
-#define TK_NK_GE                          147
+#define TK_INTO                           147
-#define TK_NK_NE                          148
+#define TK_KILL                           148
-#define TK_MATCH                          149
+#define TK_CONNECTION                     149
-#define TK_NMATCH                         150
+#define TK_MERGE                          150
-#define TK_IN                             151
+#define TK_VGROUP                         151
-#define TK_JOIN                           152
+#define TK_REDISTRIBUTE                   152
-#define TK_INNER                          153
+#define TK_SPLIT                          153
-#define TK_SELECT                         154
+#define TK_SYNCDB                         154
-#define TK_DISTINCT                       155
+#define TK_NULL                           155
-#define TK_WHERE                          156
+#define TK_NK_VARIABLE                    156
-#define TK_PARTITION                      157
+#define TK_NK_UNDERLINE                   157
-#define TK_BY                             158
+#define TK_ROWTS                          158
-#define TK_SESSION                        159
+#define TK_TBNAME                         159
-#define TK_STATE_WINDOW                   160
+#define TK_QSTARTTS                       160
-#define TK_SLIDING                        161
+#define TK_QENDTS                         161
-#define TK_FILL                           162
+#define TK_WSTARTTS                       162
-#define TK_VALUE                          163
+#define TK_WENDTS                         163
-#define TK_NONE                           164
+#define TK_WDURATION                      164
-#define TK_PREV                           165
+#define TK_BETWEEN                        165
-#define TK_LINEAR                         166
+#define TK_IS                             166
-#define TK_NEXT                           167
+#define TK_NK_LT                          167
-#define TK_GROUP                          168
+#define TK_NK_GT                          168
-#define TK_HAVING                         169
+#define TK_NK_LE                          169
-#define TK_ORDER                          170
+#define TK_NK_GE                          170
-#define TK_SLIMIT                         171
+#define TK_NK_NE                          171
-#define TK_SOFFSET                        172
+#define TK_MATCH                          172
-#define TK_LIMIT                          173
+#define TK_NMATCH                         173
-#define TK_OFFSET                         174
+#define TK_JOIN                           174
-#define TK_ASC                            175
+#define TK_INNER                          175
-#define TK_NULLS                          176
+#define TK_SELECT                         176
-#define TK_FIRST                          177
+#define TK_DISTINCT                       177
-#define TK_LAST                           178
+#define TK_WHERE                          178
+#define TK_PARTITION                      179
+#define TK_BY                             180
+#define TK_SESSION                        181
+#define TK_STATE_WINDOW                   182
+#define TK_SLIDING                        183
+#define TK_FILL                           184
+#define TK_VALUE                          185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_NEXT                           189
+#define TK_GROUP                          190
+#define TK_HAVING                         191
+#define TK_ORDER                          192
+#define TK_SLIMIT                         193
+#define TK_SOFFSET                        194
+#define TK_LIMIT                          195
+#define TK_OFFSET                         196
+#define TK_ASC                            197
+#define TK_NULLS                          198
+#define TK_FIRST                          199
+#define TK_LAST                           200
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -207,10 +229,9 @@
 #define TK_NK_COLON                       500
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502
-#define TK_INTO                           503
 #define TK_NOW                            504
 #define TK_VALUES                         507
-#define TK_IMPORT                         507
+#define TK_IMPORT                         509
 #define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535

include/libs/command/command.h

@@ -15,5 +15,10 @@
 
 #include "cmdnodes.h"
 #include "tmsg.h"
+#include "plannodes.h"
 
 int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
+
+

include/libs/nodes/cmdnodes.h

@@ -38,7 +38,9 @@ typedef struct SDatabaseOptions {
   int32_t fsyncPeriod;
   int32_t maxRowsPerBlock;
   int32_t minRowsPerBlock;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t precision;
   int32_t quorum;
   int32_t replica;
@@ -76,7 +78,9 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType type;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t ttl;
   char comments[TSDB_STB_COMMENT_LEN];
   SNodeList* pSma;
@@ -193,6 +197,12 @@ typedef struct SShowStmt {
   SNode* pTbNamePattern; // SValueNode
 } SShowStmt;
 
+typedef struct SShowCreatStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SShowCreatStmt;
+
 typedef enum EIndexType {
   INDEX_TYPE_SMA = 1,
   INDEX_TYPE_FULLTEXT

include/libs/nodes/nodes.h

@@ -103,6 +103,15 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_STMT,
   QUERY_NODE_DESCRIBE_STMT,
   QUERY_NODE_RESET_QUERY_CACHE_STMT,
+  QUERY_NODE_COMPACT_STMT,
+  QUERY_NODE_CREATE_FUNCTION_STMT,
+  QUERY_NODE_DROP_FUNCTION_STMT,
+  QUERY_NODE_CREATE_STREAM_STMT,
+  QUERY_NODE_DROP_STREAM_STMT,
+  QUERY_NODE_MERGE_VGROUP_STMT,
+  QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
+  QUERY_NODE_SPLIT_VGROUP_STMT,
+  QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -115,6 +124,18 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
   QUERY_NODE_SHOW_STREAMS_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_CONNECTIONS_STMT,
+  QUERY_NODE_SHOW_LICENCE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
+  QUERY_NODE_SHOW_QUERIES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TOPICS_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -215,6 +236,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/plannodes.h

@@ -253,7 +253,7 @@ typedef struct SIntervalPhysiNode {
   int64_t    sliding;
   int8_t     intervalUnit;
   int8_t     slidingUnit;
-  uint8_t precision;
+  uint8_t    precision;
   SFillNode* pFill;
 } SIntervalPhysiNode;
 

include/libs/nodes/querynodes.h

@@ -314,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
 void* nodesGetValueFromNode(SValueNode *pNode);
+char* nodesGetStrValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus
 }

include/libs/qcom/query.h

@@ -53,6 +53,7 @@ typedef struct SIndexMeta {
 
 } SIndexMeta;
 
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
     }                                                                    \
   } while (0)
 
+#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+
 #ifdef __cplusplus
 }
 #endif

include/os/osMemory.h

@@ -32,6 +32,7 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 

include/os/osProc.h

@@ -21,9 +21,11 @@ extern "C" {
 #endif
 
 int32_t taosNewProc(char **args);
+void    taosWaitProc(int32_t pid);
+void    taosKillProc(int32_t pid);
+bool    taosProcExist(int32_t pid);
 void    taosSetProcName(int32_t argc, char **argv, const char *name);
 void    taosSetProcPath(int32_t argc, char **argv);
-bool    taosProcExists(int32_t pid);
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -46,6 +46,14 @@ extern "C" {
 #define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
 #define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
 
+#define NANOSECOND_PER_USEC   (1000L)
+#define NANOSECOND_PER_MSEC   (1000000L)
+#define NANOSECOND_PER_SEC    (1000000000L)
+#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60)
+#define NANOSECOND_PER_HOUR   (NANOSECOND_PER_MINUTE * 60)
+#define NANOSECOND_PER_DAY    (NANOSECOND_PER_HOUR * 24)
+#define NANOSECOND_PER_WEEK   (NANOSECOND_PER_DAY * 7)
+
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
 //@return timestamp in second

include/util/taoserror.h

@@ -479,6 +479,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 #define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
+#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
+#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -224,6 +224,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_APP_NAME_LEN    TSDB_UNI_LEN
 #define TSDB_STB_COMMENT_LEN 1024
+
 /**
  *  In some scenarios uint16_t (0~65535) is used to store the row len.
  *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
@@ -303,13 +304,13 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TOTAL_BLOCKS     10000
 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
 
-#define TSDB_MIN_DAYS_PER_FILE     1
+#define TSDB_MIN_DAYS_PER_FILE     (1 * 1440)    // unit minute
-#define TSDB_MAX_DAYS_PER_FILE     3650
+#define TSDB_MAX_DAYS_PER_FILE     (3650 * 1440)
-#define TSDB_DEFAULT_DAYS_PER_FILE 10
+#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
 
-#define TSDB_MIN_KEEP     1       // data in db to be reserved.
+#define TSDB_MIN_KEEP     (1 * 1440)       // data in db to be reserved. unit minute
-#define TSDB_MAX_KEEP     365000  // data in db to be reserved.
+#define TSDB_MAX_KEEP     (365000 * 1440) // data in db to be reserved.
-#define TSDB_DEFAULT_KEEP 3650    // ten years
+#define TSDB_DEFAULT_KEEP (3650 * 1440)    // ten years
 
 #define TSDB_MIN_MIN_ROW_FBLOCK     10
 #define TSDB_MAX_MIN_ROW_FBLOCK     1000
@@ -327,7 +328,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_FSYNC_PERIOD     180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD 3000    // three second
 
-#define TSDB_MIN_WAL_LEVEL     0
+#define TSDB_MIN_WAL_LEVEL     1
 #define TSDB_MAX_WAL_LEVEL     2
 #define TSDB_DEFAULT_WAL_LEVEL 1
 
@@ -388,6 +389,7 @@ typedef enum ELogicConditionType {
 #define TSDB_DEFAULT_EXPLAIN_RATIO   0.001
 
 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
 
 #define TSDB_MAX_JOIN_TABLE_NUM 10
 #define TSDB_MAX_UNION_CLAUSE   5
@@ -479,6 +481,9 @@ enum {
 #define QND_VGID 1
 #define VND_VGID 0
 
+#define MAX_NUM_STR_SIZE 40
+
+
 #ifdef __cplusplus
 }
 #endif

source/common/src/tmsg.c

@@ -1629,6 +1629,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1650,6 +1651,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);

source/common/src/ttime.c

@@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+char getPrecisionUnit(int32_t precision) {
+  static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
+  switch (precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+    case TSDB_TIME_PRECISION_MICRO:
+    case TSDB_TIME_PRECISION_NANO:
+      return units[precision];
+    default:
+      return 0;
+  }
+}
+
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
@@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec
   return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
 }
 
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
+  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+         fromPrecision == TSDB_TIME_PRECISION_NANO);
+  static double factors[3] = {1000000., 1000., 1.};
+  switch (toUnit) {
+    case 's':
+      return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
+    case 'm':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
+    case 'h':
+      return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
+    case 'd':
+      return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
+    case 'w':
+      return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
+    case 'a':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
+    case 'u':
+      return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
+    case 'b':
+      return time * factors[fromPrecision];
+    default: {
+      return -1;
+    }
+  }  
+}
+
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
   switch (unit) {
     case 's':

source/dnode/mgmt/dm/src/dmFile.c

@@ -200,7 +200,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   taosMemoryFree(content);
 
   char realfile[PATH_MAX];
-  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/main/src/dndExec.c

@@ -20,9 +20,9 @@ static bool dndRequireNode(SMgmtWrapper *pWrapper) {
   bool    required = false;
   int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required);
   if (!required) {
-    dDebug("node:%s, no need to start", pWrapper->name);
+    dDebug("node:%s, does not require startup", pWrapper->name);
   } else {
-    dDebug("node:%s, need to start", pWrapper->name);
+    dDebug("node:%s, needs to be started", pWrapper->name);
   }
   return required;
 }
@@ -255,17 +255,39 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
+          taosKillProc(pWrapper->procId);
+        }
+      }
+
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
+          taosWaitProc(pWrapper->procId);
+          dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
+        }
+      }
       break;
-    }
+    } else {
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
 
-    for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
-      SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+          dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
-      if (!pWrapper->required) continue;
+          dndNewProc(pWrapper, n);
-      if (pDnode->ntype == NODE_MAX) continue;
+        }
-
-      if (pWrapper->procId <= 0 || !taosProcExists(pWrapper->procId)) {
-        dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
-        dndNewProc(pWrapper, n);
       }
     }
 
@@ -279,6 +301,12 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
   SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
   dInfo("%s run in child process", pWrapper->name);
 
+  pWrapper->required = dndRequireNode(pWrapper);
+  if (!pWrapper->required) {
+    dError("%s does not require startup", pWrapper->name);
+    return -1;
+  }
+
   SMsgCb msgCb = dndCreateMsgcb(pWrapper);
   tmsgSetDefaultMsgCb(&msgCb);
   pWrapper->procType = PROC_CHILD;

source/dnode/mgmt/vm/src/vmWorker.c

@@ -187,7 +187,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
-    return -1;
+    return terrno;
   }
 
   int32_t code = 0;

source/dnode/mnode/impl/src/mndDb.c

@@ -1309,7 +1309,7 @@ static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMet
   cols++;
 
   pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "days");
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
@@ -1444,7 +1444,7 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
-  *(int16_t *)pWrite = pDb->cfg.daysPerFile;
+  *(int32_t *)pWrite = pDb->cfg.daysPerFile;
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -53,7 +53,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -89,7 +89,6 @@ static const SInfosTableSchema userStbsSchema[] = {
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
 };

source/dnode/mnode/impl/src/mndStb.c

@@ -1652,10 +1652,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
     *(int32_t *)pWrite = pStb->numOfTags;
     cols++;
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = 0; // number of tables
-    cols++;
-
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     *(int64_t *)pWrite = pStb->updateTime; // number of tables
     cols++;

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -42,10 +42,10 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
+  createReq.daysPerFile = 10 * 1440;
-  createReq.daysToKeep0 = 3650;
+  createReq.daysToKeep0 = 3650 * 1440;
-  createReq.daysToKeep1 = 3650;
+  createReq.daysToKeep1 = 3650 * 1440;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -35,10 +35,10 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
+  createReq.daysPerFile = 10 * 1440;
-  createReq.daysToKeep0 = 3650;
+  createReq.daysToKeep0 = 3650 * 1440;
-  createReq.daysToKeep1 = 3650;
+  createReq.daysToKeep1 = 3650 * 1440;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/user/user.cpp

@@ -324,10 +324,10 @@ TEST_F(MndTestUser, 03_Alter_User) {
     createReq.numOfVgroups = 2;
     createReq.cacheBlockSize = 16;
     createReq.totalBlocks = 10;
-    createReq.daysPerFile = 10;
+    createReq.daysPerFile = 10 * 1440;
-    createReq.daysToKeep0 = 3650;
+    createReq.daysToKeep0 = 3650 * 1440;
-    createReq.daysToKeep1 = 3650;
+    createReq.daysToKeep1 = 3650 * 1440;
-    createReq.daysToKeep2 = 3650;
+    createReq.daysToKeep2 = 3650 * 1440;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
     createReq.commitTime = 3600;

source/libs/catalog/inc/catalogInt.h

@@ -112,7 +112,14 @@ typedef struct SCtgRuntimeStat {
 } SCtgRuntimeStat;
 
 typedef struct SCtgCacheStat {
-
+  uint64_t clusterNum;
+  uint64_t dbNum;
+  uint64_t tblNum;
+  uint64_t stblNum;
+  uint64_t vgHitNum;
+  uint64_t vgMissNum;
+  uint64_t tblHitNum;
+  uint64_t tblMissNum;
 } SCtgCacheStat;
 
 typedef struct SCatalogStat {
@@ -186,7 +193,7 @@ typedef struct SCatalogMgmt {
   bool                  exit;
   SRWLatch              lock;
   SCtgQueue             queue;
-  TdThread             updateThread;  
+  TdThread              updateThread;  
   SHashObj             *pCluster;     //key: clusterId, value: SCatalog*
   SCatalogStat          stat;
   SCatalogCfg           cfg;
@@ -204,8 +211,13 @@ typedef struct SCtgAction {
 #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 
-#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1)
+#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n)
-#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1)
+#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n)
+#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
+
+#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n))
+#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n))
+#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n))
 
 #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
 #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
@@ -291,6 +303,9 @@ typedef struct SCtgAction {
 #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); }  } while (0)
 
 
+extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
+extern void ctgdShowClusterCache(SCatalog* pCtg);
+extern int32_t ctgdShowCacheInfo(void);
 
 #ifdef __cplusplus
 }

source/libs/catalog/src/catalog.c

@@ -24,8 +24,8 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action);
 int32_t ctgActRemoveStb(SCtgMetaAction *action);
 int32_t ctgActRemoveTbl(SCtgMetaAction *action);
 
+extern SCtgDebug gCTGDebug;
 SCatalogMgmt gCtgMgmt = {0};
-SCtgDebug gCTGDebug = {0};
 SCtgAction gCtgAction[CTG_ACT_MAX] = {{
                             CTG_ACT_UPDATE_VG,
                             "update vgInfo",
@@ -53,182 +53,6 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{
                           }
 };
 
-int32_t ctgDbgEnableDebug(char *option) {
-  if (0 == strcasecmp(option, "lock")) {
-    gCTGDebug.lockEnable = true;
-    qDebug("lock debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "cache")) {
-    gCTGDebug.cacheEnable = true;
-    qDebug("cache debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "api")) {
-    gCTGDebug.apiEnable = true;
-    qDebug("api debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (0 == strcasecmp(option, "meta")) {
-    gCTGDebug.metaEnable = true;
-    qDebug("api debug enabled");
-    return TSDB_CODE_SUCCESS;
-  }
-
-  qError("invalid debug option:%s", option);
-  
-  return TSDB_CODE_CTG_INTERNAL_ERROR;
-}
-
-int32_t ctgDbgGetStatNum(char *option, void *res) {
-  if (0 == strcasecmp(option, "runtime.qDoneNum")) {
-    *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  qError("invalid stat option:%s", option);
-  
-  return TSDB_CODE_CTG_INTERNAL_ERROR;
-}
-
-int32_t ctgDbgGetTbMetaNum(SCtgDBCache *dbCache) {
-  return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
-}
-
-int32_t ctgDbgGetStbNum(SCtgDBCache *dbCache) {
-  return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
-}
-
-int32_t ctgDbgGetRentNum(SCtgRentMgmt *rent) {
-  int32_t num = 0;
-  for (uint16_t i = 0; i < rent->slotNum; ++i) {
-    SCtgRentSlot *slot = &rent->slots[i];
-    if (NULL == slot->meta) {
-      continue;
-    }
-
-    num += taosArrayGetSize(slot->meta);
-  }
-
-  return num;
-}
-
-int32_t ctgDbgGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
-  if (NULL == pCtg || NULL == pCtg->dbCache) {
-    return 0;
-  }
-
-  switch (type) {
-    case CTG_DBG_DB_NUM:
-      return (int32_t)taosHashGetSize(pCtg->dbCache);
-    case CTG_DBG_DB_RENT_NUM:
-      return ctgDbgGetRentNum(&pCtg->dbRent);
-    case CTG_DBG_STB_RENT_NUM:
-      return ctgDbgGetRentNum(&pCtg->stbRent);
-    default:
-      break;
-  }
-
-  SCtgDBCache *dbCache = NULL;
-  int32_t num = 0;
-  void *pIter = taosHashIterate(pCtg->dbCache, NULL);
-  while (pIter) {
-    dbCache = (SCtgDBCache *)pIter;
-    switch (type) {
-      case CTG_DBG_META_NUM:
-        num += ctgDbgGetTbMetaNum(dbCache);
-        break;
-      case CTG_DBG_STB_NUM:
-        num += ctgDbgGetStbNum(dbCache);
-        break;
-      default:
-        ctgError("invalid type:%d", type);
-        break;
-    }
-    pIter = taosHashIterate(pCtg->dbCache, pIter);
-  }
-
-  return num;
-}
-
-void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
-  if (!gCTGDebug.metaEnable) {
-    return;
-  }
-
-  STableComInfo *c = &p->tableInfo;
-
-  if (TSDB_CHILD_TABLE == p->tableType) {
-    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
-    return;
-  } else {
-    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
-     tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
-  }
-
-  int32_t colNum = c->numOfColumns + c->numOfTags;
-  for (int32_t i = 0; i < colNum; ++i) {
-    SSchema *s = &p->schema[i];
-    ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes);
-  }
-}
-
-void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
-  if (NULL == dbHash || !gCTGDebug.cacheEnable) {
-    return;
-  }
-
-  int32_t i = 0;
-  SCtgDBCache *dbCache = NULL;
-  void *pIter = taosHashIterate(dbHash, NULL);
-  while (pIter) {
-    char *dbFName = NULL;
-    size_t len = 0;
-    
-    dbCache = (SCtgDBCache *)pIter;
-
-    dbFName = taosHashGetKey(pIter, &len);
-
-    int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
-    int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
-    int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
-    int32_t hashMethod = -1;
-    int32_t vgNum = 0;
-
-    if (dbCache->vgInfo) {
-      vgVersion = dbCache->vgInfo->vgVersion;
-      hashMethod = dbCache->vgInfo->hashMethod;
-      if (dbCache->vgInfo->vgHash) {
-        vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
-      }
-    }
-    
-    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
-      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
-
-    pIter = taosHashIterate(dbHash, pIter);
-  }
-}
-
-
-
-
-void ctgDbgShowClusterCache(SCatalog* pCtg) {
-  if (!gCTGDebug.cacheEnable || NULL == pCtg) {
-    return;
-  }
-
-  ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
-  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
-    ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
-  
-  ctgDbgShowDBCache(pCtg, pCtg->dbCache);
-}
-
-
 void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
   if (NULL == mgmt->slots) {
     return;
@@ -249,15 +73,19 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) {
 void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) {
   CTG_LOCK(CTG_WRITE, &cache->stbLock);
   if (cache->stbCache) {
+    int32_t stblNum = taosHashGetSize(cache->stbCache);  
     taosHashCleanup(cache->stbCache);
     cache->stbCache = NULL;
+    CTG_CACHE_STAT_SUB(stblNum, stblNum);
   }
   CTG_UNLOCK(CTG_WRITE, &cache->stbLock);
 
   CTG_LOCK(CTG_WRITE, &cache->metaLock);
   if (cache->metaCache) {
+    int32_t tblNum = taosHashGetSize(cache->metaCache);
     taosHashCleanup(cache->metaCache);
     cache->metaCache = NULL;
+    CTG_CACHE_STAT_SUB(tblNum, tblNum);
   }
   CTG_UNLOCK(CTG_WRITE, &cache->metaLock);
 }
@@ -293,6 +121,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
   ctgFreeMetaRent(&pCtg->stbRent);
   
   if (pCtg->dbCache) {
+    int32_t dbNum = taosHashGetSize(pCtg->dbCache);
+    
     void *pIter = taosHashIterate(pCtg->dbCache, NULL);
     while (pIter) {
       SCtgDBCache *dbCache = pIter;
@@ -305,6 +135,8 @@ void ctgFreeHandle(SCatalog* pCtg) {
     }  
 
     taosHashCleanup(pCtg->dbCache);
+    
+    CTG_CACHE_STAT_SUB(dbNum, dbNum);
   }
   
   taosMemoryFree(pCtg);
@@ -361,7 +193,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) {
   CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock);
 
   CTG_QUEUE_ADD();
-  CTG_STAT_ADD(gCtgMgmt.stat.runtime.qNum);
+  CTG_RUNTIME_STAT_ADD(qNum, 1);
 
   tsem_post(&gCtgMgmt.queue.reqSem);
 
@@ -620,34 +452,45 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache)
 
 
 int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) {
+  SCtgDBCache *dbCache = NULL;
+
   if (NULL == pCtg->dbCache) {
-    *pCache = NULL;
+    ctgDebug("empty db cache, dbFName:%s", dbFName);
-    *inCache = false;
+    goto _return;
-    ctgWarn("empty db cache, dbFName:%s", dbFName);
-    return TSDB_CODE_SUCCESS;
   }
 
-  SCtgDBCache *dbCache = NULL;
   ctgAcquireDBCache(pCtg, dbFName, &dbCache);
   if (NULL == dbCache) {  
-    *pCache = NULL;
+    ctgDebug("db %s not in cache", dbFName);
-    *inCache = false;
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
   
   ctgAcquireVgInfo(pCtg, dbCache, inCache);
   if (!(*inCache)) {
-    ctgReleaseDBCache(pCtg, dbCache);
+    ctgDebug("vgInfo of db %s not in cache", dbFName);
-  
+    goto _return;
-    *pCache = NULL;
-    return TSDB_CODE_SUCCESS;
   }
 
   *pCache = dbCache;
   *inCache = true;
 
+  CTG_CACHE_STAT_ADD(vgHitNum, 1);
+
   ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName);
   
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  if (dbCache) {
+    ctgReleaseDBCache(pCtg, dbCache);
+  }
+
+  *pCache = NULL;
+  *inCache = false;
+
+  CTG_CACHE_STAT_ADD(vgMissNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -763,11 +606,10 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName,
 }
 
 
-int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag, uint64_t *dbId) {
+int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) {
   if (NULL == pCtg->dbCache) {
-    *exist = 0;
+    ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname);
-    ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname);
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
 
   char dbFName[TSDB_DB_FNAME_LEN] = {0};
@@ -782,8 +624,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   SCtgDBCache *dbCache = NULL;
   ctgAcquireDBCache(pCtg, dbFName, &dbCache);
   if (NULL == dbCache) {
-    *exist = 0;
+    ctgDebug("db %s not in cache", pTableName->tname);
-    return TSDB_CODE_SUCCESS;
+    goto _return;
   }
   
   int32_t sz = 0;
@@ -792,13 +634,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
 
   if (NULL == *pTableMeta) {
-    *exist = 0;
     ctgReleaseDBCache(pCtg, dbCache);
     ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
-    return TSDB_CODE_SUCCESS;
+    goto _return;
   }
 
-  *exist = 1;
   if (dbId) {
     *dbId = dbCache->dbId;
   }
@@ -808,6 +648,10 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   if (tbMeta->tableType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
     ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
+
+    *inCache = true;
+    CTG_CACHE_STAT_ADD(tblHitNum, 1);
+    
     return TSDB_CODE_SUCCESS;
   }
   
@@ -819,8 +663,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
     ctgReleaseDBCache(pCtg, dbCache);
     ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid);
     taosMemoryFreeClear(*pTableMeta);
-    *exist = 0;
+    goto _return;
-    return TSDB_CODE_SUCCESS;
   }
 
   if ((*stbMeta)->suid != tbMeta->suid) {    
@@ -846,8 +689,18 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
 
   ctgReleaseDBCache(pCtg, dbCache);
 
+  *inCache = true;
+  CTG_CACHE_STAT_ADD(tblHitNum, 1);
+
   ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname);
   
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  *inCache = false;
+  CTG_CACHE_STAT_ADD(tblMissNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1378,6 +1231,8 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
     CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  CTG_CACHE_STAT_ADD(dbNum, 1);
+ 
   SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1};
   strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName));
 
@@ -1436,6 +1291,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) {
     CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
   }
 
+  CTG_CACHE_STAT_SUB(dbNum, 1);
+
   ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId);
   
   return TSDB_CODE_SUCCESS;
@@ -1568,6 +1425,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
         CTG_LOCK(CTG_WRITE, &tbCache->stbLock);
         if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) {
           ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
+        } else {
+          CTG_CACHE_STAT_SUB(stblNum, 1);
         }
         CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
 
@@ -1594,8 +1453,12 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  if (NULL == orig) {
+    CTG_CACHE_STAT_ADD(tblNum, 1);
+  }
+
   ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType);
-  ctgDbgShowTableMeta(pCtg, tbName, meta);
+  ctgdShowTableMeta(pCtg, tbName, meta);
 
   if (!isStb) {
     CTG_UNLOCK(CTG_READ, &tbCache->metaLock);  
@@ -1616,6 +1479,8 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
   }
 
+  CTG_CACHE_STAT_ADD(stblNum, 1);
+  
   CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
 
   CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
@@ -1874,7 +1739,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
   }
   
-  int32_t exist = 0;
+  bool inCache = false;
   int32_t code = 0;
   uint64_t dbId = 0;
   uint64_t suid = 0;
@@ -1884,11 +1749,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     CTG_FLAG_SET_INF_DB(flag);
   }
 
-  CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag, &dbId));
+  CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &inCache, flag, &dbId));
 
   int32_t tbType = 0;
 
-  if (exist) {
+  if (inCache) {
     if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_INF_DB(flag)))) {
       goto _return;
     }
@@ -1930,8 +1795,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     SName stbName = *pTableName;
     strcpy(stbName.tname, output->tbName);
     
-    CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag, NULL));
+    CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL));
-    if (0 == exist) {
+    if (!inCache) {
       ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
       continue;
     }
@@ -1943,7 +1808,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
 _return:
 
-  if (CTG_TABLE_NOT_EXIST(code) && exist) {
+  if (CTG_TABLE_NOT_EXIST(code) && inCache) {
     char dbFName[TSDB_DB_FNAME_LEN] = {0};
     if (CTG_FLAG_IS_INF_DB(flag)) {
       strcpy(dbFName, pTableName->dbname);
@@ -1962,7 +1827,7 @@ _return:
 
   if (*pTableMeta) {
     ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType);
-    ctgDbgShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
+    ctgdShowTableMeta(pCtg, pTableName->tname, *pTableMeta);
   }
 
   CTG_RET(code);
@@ -2075,11 +1940,15 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) {
   CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock);
   if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) {
     ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
+  } else {
+    CTG_CACHE_STAT_SUB(stblNum, 1);
   }
 
   CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
   if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) {  
     ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid);
+  } else {
+    CTG_CACHE_STAT_SUB(tblNum, 1);
   }
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
   
@@ -2119,6 +1988,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
     CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
     ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
+  } else {
+    CTG_CACHE_STAT_SUB(tblNum, 1);
   }
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
 
@@ -2140,7 +2011,9 @@ void* ctgUpdateThreadFunc(void* param) {
   CTG_LOCK(CTG_READ, &gCtgMgmt.lock);
   
   while (true) {
-    tsem_wait(&gCtgMgmt.queue.reqSem);
+    if (tsem_wait(&gCtgMgmt.queue.reqSem)) {
+      qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    }
     
     if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {
       tsem_post(&gCtgMgmt.queue.rspSem);
@@ -2161,9 +2034,9 @@ void* ctgUpdateThreadFunc(void* param) {
       tsem_post(&gCtgMgmt.queue.rspSem);
     }
 
-    CTG_STAT_ADD(gCtgMgmt.stat.runtime.qDoneNum); 
+    CTG_RUNTIME_STAT_ADD(qDoneNum, 1); 
 
-    ctgDbgShowClusterCache(pCtg);
+    ctgdShowClusterCache(pCtg);
   }
 
   CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock);
@@ -2304,8 +2177,15 @@ int32_t catalogInit(SCatalogCfg *cfg) {
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
-  tsem_init(&gCtgMgmt.queue.reqSem, 0, 0);
+  if (tsem_init(&gCtgMgmt.queue.reqSem, 0, 0)) {
-  tsem_init(&gCtgMgmt.queue.rspSem, 0, 0);
+    qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
+  }
+  
+  if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) {
+    qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+    CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR);
+  }
 
   gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode));
   if (NULL == gCtgMgmt.queue.head) {
@@ -2384,6 +2264,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
 
   *catalogHandle = clusterCtg;
 
+  CTG_CACHE_STAT_ADD(clusterNum, 1);
+  
   return TSDB_CODE_SUCCESS;
 
 _return:
@@ -2403,6 +2285,8 @@ void catalogFreeHandle(SCatalog* pCtg) {
     return;
   }
 
+  CTG_CACHE_STAT_SUB(clusterNum, 1);
+
   uint64_t clusterId = pCtg->clusterId;
   
   ctgFreeHandle(pCtg);
@@ -2417,24 +2301,12 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
     CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
-  if (NULL == pCtg->dbCache) {
-    *version = CTG_DEFAULT_INVALID_VERSION;
-    ctgInfo("empty db cache, dbFName:%s", dbFName);
-    CTG_API_LEAVE(TSDB_CODE_SUCCESS);
-  }
-
   SCtgDBCache *dbCache = NULL;
-  ctgAcquireDBCache(pCtg, dbFName, &dbCache);
-  if (NULL == dbCache) {
-    *version = CTG_DEFAULT_INVALID_VERSION;
-    CTG_API_LEAVE(TSDB_CODE_SUCCESS);
-  }
-
   bool inCache = false;
-  ctgAcquireVgInfo(pCtg, dbCache, &inCache);
+  int32_t code = 0;
-  if (!inCache) {
-    ctgReleaseDBCache(pCtg, dbCache);
 
+  CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache));
+  if (!inCache) {
     *version = CTG_DEFAULT_INVALID_VERSION;
     CTG_API_LEAVE(TSDB_CODE_SUCCESS);
   }
@@ -2449,6 +2321,10 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers
   ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version);
 
   CTG_API_LEAVE(TSDB_CODE_SUCCESS);
+
+_return:
+
+  CTG_API_LEAVE(code);
 }
 
 int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) {
@@ -2549,11 +2425,11 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) {
   }
 
   STableMeta *tblMeta = NULL;
-  int32_t exist = 0;
+  bool inCache = false;
   uint64_t dbId = 0;
-  CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &exist, 0, &dbId));
+  CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId));
 
-  if (0 == exist) {
+  if (!inCache) {
     ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
     goto _return;
   }
@@ -2851,8 +2727,13 @@ void catalogDestroy(void) {
 
   atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
 
-  tsem_post(&gCtgMgmt.queue.reqSem);
+  if (tsem_post(&gCtgMgmt.queue.reqSem)) {
-  tsem_post(&gCtgMgmt.queue.rspSem);
+    qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+  }
+  
+  if (tsem_post(&gCtgMgmt.queue.rspSem)) {
+    qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
+  }
 
   while (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
     taosUsleep(1);

source/libs/catalog/src/catalogDbg.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "trpc.h"
+#include "query.h"
+#include "tname.h"
+#include "catalogInt.h"
+
+extern SCatalogMgmt gCtgMgmt;
+SCtgDebug gCTGDebug = {0};
+
+int32_t ctgdEnableDebug(char *option) {
+  if (0 == strcasecmp(option, "lock")) {
+    gCTGDebug.lockEnable = true;
+    qDebug("lock debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "cache")) {
+    gCTGDebug.cacheEnable = true;
+    qDebug("cache debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "api")) {
+    gCTGDebug.apiEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "meta")) {
+    gCTGDebug.metaEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid debug option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetStatNum(char *option, void *res) {
+  if (0 == strcasecmp(option, "runtime.qDoneNum")) {
+    *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid stat option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+}
+
+int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+}
+
+int32_t ctgdGetRentNum(SCtgRentMgmt *rent) {
+  int32_t num = 0;
+  for (uint16_t i = 0; i < rent->slotNum; ++i) {
+    SCtgRentSlot *slot = &rent->slots[i];
+    if (NULL == slot->meta) {
+      continue;
+    }
+
+    num += taosArrayGetSize(slot->meta);
+  }
+
+  return num;
+}
+
+int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
+  if (NULL == pCtg || NULL == pCtg->dbCache) {
+    return 0;
+  }
+
+  switch (type) {
+    case CTG_DBG_DB_NUM:
+      return (int32_t)taosHashGetSize(pCtg->dbCache);
+    case CTG_DBG_DB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->dbRent);
+    case CTG_DBG_STB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->stbRent);
+    default:
+      break;
+  }
+
+  SCtgDBCache *dbCache = NULL;
+  int32_t num = 0;
+  void *pIter = taosHashIterate(pCtg->dbCache, NULL);
+  while (pIter) {
+    dbCache = (SCtgDBCache *)pIter;
+    switch (type) {
+      case CTG_DBG_META_NUM:
+        num += ctgdGetTbMetaNum(dbCache);
+        break;
+      case CTG_DBG_STB_NUM:
+        num += ctgdGetStbNum(dbCache);
+        break;
+      default:
+        ctgError("invalid type:%d", type);
+        break;
+    }
+    pIter = taosHashIterate(pCtg->dbCache, pIter);
+  }
+
+  return num;
+}
+
+void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
+  if (!gCTGDebug.metaEnable) {
+    return;
+  }
+
+  STableComInfo *c = &p->tableInfo;
+
+  if (TSDB_CHILD_TABLE == p->tableType) {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
+    return;
+  } else {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
+     tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
+  }
+
+  int32_t colNum = c->numOfColumns + c->numOfTags;
+  for (int32_t i = 0; i < colNum; ++i) {
+    SSchema *s = &p->schema[i];
+    ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes);
+  }
+}
+
+void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
+  if (NULL == dbHash || !gCTGDebug.cacheEnable) {
+    return;
+  }
+
+  int32_t i = 0;
+  SCtgDBCache *dbCache = NULL;
+  void *pIter = taosHashIterate(dbHash, NULL);
+  while (pIter) {
+    char *dbFName = NULL;
+    size_t len = 0;
+    
+    dbCache = (SCtgDBCache *)pIter;
+
+    dbFName = taosHashGetKey(pIter, &len);
+
+    int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+    int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+    int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
+    int32_t hashMethod = -1;
+    int32_t vgNum = 0;
+
+    if (dbCache->vgInfo) {
+      vgVersion = dbCache->vgInfo->vgVersion;
+      hashMethod = dbCache->vgInfo->hashMethod;
+      if (dbCache->vgInfo->vgHash) {
+        vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
+      }
+    }
+    
+    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
+      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
+
+    pIter = taosHashIterate(dbHash, pIter);
+  }
+}
+
+
+
+
+void ctgdShowClusterCache(SCatalog* pCtg) {
+  if (!gCTGDebug.cacheEnable || NULL == pCtg) {
+    return;
+  }
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
+  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
+    ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
+  
+  ctgdShowDBCache(pCtg, pCtg->dbCache);
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg);
+}
+
+int32_t ctgdShowCacheInfo(void) {
+  if (!gCTGDebug.cacheEnable) {
+    return TSDB_CODE_CTG_OUT_OF_SERVICE;
+  }
+
+  CTG_API_ENTER();
+  
+  SCatalog *pCtg = NULL;
+  void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+  while (pIter) {
+    pCtg = *(SCatalog **)pIter;
+
+    if (pCtg) {
+      ctgdShowClusterCache(pCtg);
+    }
+    
+    pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
+  }
+
+  CTG_API_LEAVE(TSDB_CODE_SUCCESS);
+}
+

source/libs/catalog/test/catalogTests.cpp

@@ -38,11 +38,11 @@
 namespace {
 
 extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
-                                            int32_t *exist, int32_t flag, uint64_t *dbId);
+                                            bool *inCache, int32_t flag, uint64_t *dbId);
-extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
+extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
 extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action);
-extern "C" int32_t ctgDbgEnableDebug(char *option);
+extern "C" int32_t ctgdEnableDebug(char *option);
-extern "C" int32_t ctgDbgGetStatNum(char *option, void *res);
+extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 
 void ctgTestSetRspTableMeta();
 void ctgTestSetRspCTableMeta();
@@ -140,9 +140,9 @@ void ctgTestInitLogFile() {
   qDebugFlag = 159;
   strcpy(tsLogDir, "/var/log/taos");
 
-  ctgDbgEnableDebug("api");
+  ctgdEnableDebug("api");
-  ctgDbgEnableDebug("meta");
+  ctgdEnableDebug("meta");
-  ctgDbgEnableDebug("cache");
+  ctgdEnableDebug("cache");
   
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) {
   int32_t          code = 0;
   int32_t          n = 0;
   STableMeta      *tbMeta = NULL;
-  int32_t          exist = 0;
+  bool             inCache = false;
 
   SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(cn.dbname, "db1");
   strcpy(cn.tname, ctgTestCTablename);
 
   while (!ctgTestStop) {
-    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL);
+    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL);
-    if (code || 0 == exist) {
+    if (code || !inCache) {
       assert(0);
     }
 
@@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
     taosMsleep(50);
   }
   
@@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (2 != n) {
       taosMsleep(50);
     } else {
@@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(code, 0);
 
   while (true) {
-    int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
-    int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
+    int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
     if (n || m) {
       taosMsleep(50);
     } else {
@@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) {
   }
 
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) {
     }
   }
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
 
   code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
   ASSERT_EQ(code, 0);
@@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) {
     ASSERT_EQ(tableMeta->tableInfo.precision, 1);
     ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
-    while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
+    while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
       taosMsleep(50);
     }
 

source/libs/command/inc/commandInt.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_QUERY_INT_H_
+#define _TD_QUERY_INT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "nodes.h"
+#include "plannodes.h"
+#include "ttime.h"
+
+#define EXPLAIN_MAX_GROUP_NUM 100
+
+//newline area
+#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d"
+#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d"
+#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d"
+#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d"
+#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d"
+#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d"
+#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d"
+#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d"
+#define EXPLAIN_ORDER_FORMAT "Order: %s"
+#define EXPLAIN_FILTER_FORMAT "Filter: "
+#define EXPLAIN_FILL_FORMAT "Fill: %s"
+#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
+#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
+
+//append area
+#define EXPLAIN_GROUPS_FORMAT " groups=%d"
+#define EXPLAIN_WIDTH_FORMAT " width=%d"
+#define EXPLAIN_LOOPS_FORMAT " loops=%d"
+#define EXPLAIN_REVERSE_FORMAT " reverse=%d"
+
+typedef struct SExplainGroup {
+  int32_t   nodeNum;
+  SSubplan *plan;
+  void     *execInfo;  //TODO
+} SExplainGroup;
+
+typedef struct SExplainResNode {
+  SNodeList*  pChildren;
+  SPhysiNode* pNode;
+  void*       pExecInfo;
+} SExplainResNode;
+
+typedef struct SQueryExplainRowInfo {
+  int32_t level;
+  int32_t len;
+  char   *buf;
+} SQueryExplainRowInfo;
+
+typedef struct SExplainCtx {
+  int32_t   totalSize;
+  bool      verbose;
+  char     *tbuf;
+  SArray   *rows;
+  SHashObj *groupHash;
+} SExplainCtx;
+
+#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending")
+#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
+
+#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u)))
+
+#define EXPLAIN_ROW_NEW(level, ...)                                                                               \
+  do {                                                                                                            \
+    if (isVerboseLine) {                                                                                          \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, "");       \
+    } else {                                                                                                      \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+    }                                                                                                             \
+    tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__);         \
+  } while (0)
+  
+#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)
+#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_QUERY_INT_H_*/

source/libs/command/src/explain.c

@@ -0,0 +1,687 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "query.h"
+#include "plannodes.h"
+#include "commandInt.h"
+
+int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes);
+int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level);
+
+
+void qFreeExplainResTree(SExplainResNode *res) {
+  if (NULL == res) {
+    return;
+  }
+
+  taosMemoryFreeClear(res->pExecInfo);
+
+  SNode* node = NULL;
+  FOREACH(node, res->pChildren) {
+    qFreeExplainResTree((SExplainResNode *)node);
+  }  
+  nodesClearList(res->pChildren);
+  
+  taosMemoryFreeClear(res);
+}
+
+void qFreeExplainCtx(void *ctx) {
+  if (NULL == ctx) {
+    return;
+  }
+
+  SExplainCtx *pCtx = (SExplainCtx *)ctx;
+  int32_t rowSize = taosArrayGetSize(pCtx->rows);
+  for (int32_t i = 0; i < rowSize; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    taosMemoryFreeClear(row->buf);
+  }
+
+  taosHashCleanup(pCtx->groupHash);
+  taosArrayDestroy(pCtx->rows);
+  taosMemoryFree(pCtx);
+}
+
+int32_t qInitExplainCtx(void **pCtx, SHashObj *groupHash, bool verbose) {
+  int32_t code = 0;
+  SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx));
+  if (NULL == ctx) {
+    qError("calloc SExplainCtx failed");
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+  
+  SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo));
+  if (NULL == rows) {
+    qError("taosArrayInit SQueryExplainRowInfo failed");
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE);
+  if (NULL == tbuf) {
+    qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE);
+    QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  ctx->verbose = verbose;
+  ctx->tbuf = tbuf;
+  ctx->rows = rows;
+  ctx->groupHash = groupHash;
+  
+  *pCtx = ctx;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  taosArrayDestroy(rows);
+  taosHashCleanup(groupHash);
+  taosMemoryFree(ctx);
+
+  QRY_RET(code);
+}
+
+
+char *qFillModeString(EFillMode mode) {
+  switch (mode) {
+    case FILL_MODE_NONE:
+      return "none";
+    case FILL_MODE_VALUE:
+      return "value";
+    case FILL_MODE_PREV:
+      return "prev";
+    case FILL_MODE_NULL:
+      return "null";
+    case FILL_MODE_LINEAR:
+      return "linear";
+    case FILL_MODE_NEXT:
+      return "next";
+    default:
+      return "unknown";
+  }
+}
+
+char *qGetNameFromColumnNode(SNode *pNode) {
+  if (NULL == pNode || QUERY_NODE_COLUMN != pNode->type) {
+    return "NULL";
+  }
+  
+  return ((SColumnNode *)pNode)->colName;
+}
+
+int32_t qGenerateExplainResChildren(SPhysiNode *pNode, void *pExecInfo, SNodeList **pChildren) {
+  int32_t tlen = 0;
+  SNodeList *pPhysiChildren = NULL;
+  
+  switch (pNode->type) {
+    case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
+      STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
+      pPhysiChildren = pTagScanNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
+      STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
+      pPhysiChildren = pTblScanNode->scan.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
+      SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
+      pPhysiChildren = pSTblScanNode->scan.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
+      SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
+      pPhysiChildren = pPrjNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
+      SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
+      pPhysiChildren = pJoinNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_AGG:{
+      SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
+      pPhysiChildren = pAggNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
+      SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
+      pPhysiChildren = pExchNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:{
+      SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
+      pPhysiChildren = pSortNode->node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
+      SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
+      pPhysiChildren = pIntNode->window.node.pChildren;
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
+      SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode;
+      pPhysiChildren = pSessNode->window.node.pChildren;
+      break;
+    }
+    default:
+      qError("not supported physical node type %d", pNode->type);
+      QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (pPhysiChildren) {
+    *pChildren = nodesMakeList();
+    if (NULL == *pChildren) {
+      qError("nodesMakeList failed");
+      QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+  }
+
+  SNode* node = NULL;
+  SExplainResNode *pResNode = NULL;
+  FOREACH(node, pPhysiChildren) {
+    QRY_ERR_RET(qGenerateExplainResNode((SPhysiNode *)node, pExecInfo, &pResNode));
+    QRY_ERR_RET(nodesListAppend(*pChildren, pResNode));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qGenerateExplainResNode(SPhysiNode *pNode, void *pExecInfo, SExplainResNode **pRes) {
+  if (NULL == pNode) {
+    *pRes = NULL;
+    qError("physical node is NULL");
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  SExplainResNode *res = taosMemoryCalloc(1, sizeof(SExplainResNode));
+  if (NULL == res) {
+    qError("calloc SPhysiNodeExplainRes failed");
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+
+  int32_t code = 0;
+  res->pNode = pNode;
+  res->pExecInfo = pExecInfo;
+  QRY_ERR_JRET(qGenerateExplainResChildren(pNode, pExecInfo, &res->pChildren));
+  
+  *pRes = res;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  qFreeExplainResTree(res);
+  
+  QRY_RET(code);
+}
+
+int32_t qExplainBufAppendExecInfo(void *pExecInfo, char *tbuf, int32_t *len) {
+  int32_t tlen = *len;
+  
+  EXPLAIN_ROW_APPEND("(exec info here)");
+  
+  *len = tlen;
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) {
+  SQueryExplainRowInfo row = {0};
+  row.buf = taosMemoryMalloc(len);
+  if (NULL == row.buf) {
+    qError("taosMemoryMalloc %d failed", len);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  memcpy(row.buf, tbuf, len);
+  row.level = level;
+  row.len = len;
+  ctx->totalSize += len;
+
+  if (NULL == taosArrayPush(ctx->rows, &row)) {
+    qError("taosArrayPush row to explain res rows failed");
+    taosMemoryFree(row.buf);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
+  int32_t tlen = 0;
+  bool isVerboseLine = false;
+  char *tbuf = ctx->tbuf;
+  bool verbose = ctx->verbose;
+  SPhysiNode* pNode = pResNode->pNode;
+  if (NULL == pNode) {
+    qError("pyhsical node in explain res node is NULL");
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+  
+  switch (pNode->type) {
+    case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: {
+      STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_TAG_SCAN_FORMAT, pTagScanNode->tableName.tname, pTagScanNode->pScanCols->length, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count);
+      if (pTagScanNode->reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
+    case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{
+      STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname, pTblScanNode->scan.pScanCols->length, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count);
+      if (pTblScanNode->scan.reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {      
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+        if (pTblScanNode->scan.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{
+      SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname, pSTblScanNode->scan.pScanCols->length, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count);
+      if (pSTblScanNode->scan.reverse) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse);
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {            
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order));
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));   
+
+        if (pSTblScanNode->scan.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+          QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+        
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{
+      SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT, pPrjNode->pProjections->length, pPrjNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pPrjNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_JOIN:{
+      SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType), pJoinNode->pTargets->length, pJoinNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pJoinNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+        
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);      
+        QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+        EXPLAIN_ROW_END();
+        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_AGG:{
+      SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT, pAggNode->pAggFuncs->length);
+      if (pAggNode->pGroupKeys) {
+        EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length);
+      }
+      EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize);
+      
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pAggNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{
+      SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode;
+      SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId));
+      if (NULL == group) {
+        qError("exchange src group %d not in groupHash", pExchNode->srcGroupId);
+        QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum, pExchNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pExchNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+
+      QRY_ERR_RET(qAppendTaskExplainResRows(ctx, pExchNode->srcGroupId, level + 1));
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:{
+      SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT, pSortNode->pSortKeys->length, pSortNode->node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pSortNode->node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
+      SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, qGetNameFromColumnNode(pIntNode->pTspk), pIntNode->window.pFuncs->length, 
+        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision), pIntNode->intervalUnit, 
+        pIntNode->offset, getPrecisionUnit(pIntNode->precision), 
+        INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit, 
+        pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pIntNode->pFill) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, qFillModeString(pIntNode->pFill->mode));      
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+
+        if (pIntNode->window.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{
+      SSessionWinodwPhysiNode *pIntNode = (SSessionWinodwPhysiNode *)pNode;
+      EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT, pIntNode->gap, pIntNode->window.pFuncs->length, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
+      if (pResNode->pExecInfo) {
+        QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+      }
+      EXPLAIN_ROW_END();
+      QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+      if (verbose) {
+        if (pIntNode->window.node.pConditions) {
+          EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);      
+          QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 
+          EXPLAIN_ROW_END();
+          QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+        }
+      }
+      break;
+    }
+    default:
+      qError("not supported physical node type %d", pNode->type);
+      return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
+  if (NULL == pResNode) {
+    qError("explain res node is NULL");
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  int32_t code = 0;
+  QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level));
+
+  SNode* pNode = NULL;
+  FOREACH(pNode, pResNode->pChildren) {
+    QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qAppendTaskExplainResRows(void *pCtx, int32_t groupId, int32_t level) {
+  SExplainResNode *node = NULL;
+  int32_t code = 0;
+  SExplainCtx *ctx = (SExplainCtx *)pCtx;
+
+  SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId));
+  if (NULL == group) {
+    qError("group %d not in groupHash", groupId);
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  QRY_ERR_RET(qGenerateExplainResNode(group->plan->pNode, group->execInfo, &node));
+
+  QRY_ERR_JRET(qExplainResNodeToRows(node, ctx, level));
+
+_return:
+
+  qFreeExplainResTree(node);
+  
+  QRY_RET(code);
+}
+
+
+int32_t qGetExplainRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
+  SExplainCtx *pCtx = (SExplainCtx *)ctx;
+  int32_t rowNum = taosArrayGetSize(pCtx->rows);
+  if (rowNum <= 0) {
+    qError("empty explain res rows");
+    QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  int32_t colNum = 1;
+  int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->totalSize;
+  SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
+  if (NULL == rsp) {
+    qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  rsp->completed = 1;
+  rsp->numOfRows = htonl(rowNum);
+
+  *(int32_t *)rsp->data = htonl(pCtx->totalSize);
+
+  int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t));
+  char *data = (char *)(offset + rowNum);
+  int32_t tOffset = 0;
+  
+  for (int32_t i = 0; i < rowNum; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    *offset = tOffset;
+    tOffset += row->len;
+
+    memcpy(data, row->buf, row->len);
+    
+    ++offset;
+    data += row->len;
+  }
+
+  *pRsp = rsp;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
+  int32_t code = 0;
+  SNodeListNode *plans = NULL;
+  int32_t        taskNum = 0;
+  SExplainGroup *pGroup = NULL;
+  void *pCtx = NULL;
+  int32_t rootGroupId = 0;
+
+  if (pDag->numOfSubplans <= 0) {
+    qError("invalid subplan num:%d", pDag->numOfSubplans);
+    QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans);
+  if (levelNum <= 0) {
+    qError("invalid level num:%d", levelNum);
+    QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  if (NULL == groupHash) {
+    qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM);
+    QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  QRY_ERR_JRET(qInitExplainCtx(&pCtx, groupHash, pDag->explainInfo.verbose));
+
+  for (int32_t i = 0; i < levelNum; ++i) {
+    plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
+    if (NULL == plans) {
+      qError("empty level plan, level:%d", i);
+      QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+
+    taskNum = (int32_t)LIST_LENGTH(plans->pNodeList);
+    if (taskNum <= 0) {
+      qError("invalid level plan number:%d, level:%d", taskNum, i);
+      QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+
+    SSubplan *plan = NULL;
+    for (int32_t n = 0; n < taskNum; ++n) {
+      plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
+      pGroup = taosHashGet(groupHash, &plan->id.groupId, sizeof(plan->id.groupId));
+      if (pGroup) {
+        ++pGroup->nodeNum;
+        continue;
+      }
+
+      SExplainGroup group = {.nodeNum = 1, .plan = plan, .execInfo = NULL};
+      if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) {
+        qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n);
+        QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+      }
+    }
+
+    if (0 == i) {
+      if (taskNum > 1) {
+        qError("invalid taskNum %d for level 0", taskNum);
+        QRY_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+      }
+
+      rootGroupId = plan->id.groupId;
+    }
+
+    qDebug("level %d group handled, taskNum:%d", i, taskNum);
+  }
+
+  QRY_ERR_JRET(qAppendTaskExplainResRows(pCtx, rootGroupId, 0));
+  
+  QRY_ERR_JRET(qGetExplainRspFromCtx(pCtx, pRsp));
+  
+_return:
+
+  qFreeExplainCtx(pCtx);
+
+  QRY_RET(code);
+}
+
+
+

source/libs/executor/inc/executil.h

@@ -73,27 +73,12 @@ typedef struct SResultRowPosition {
 } SResultRowPosition;
 
 typedef struct SResultRowInfo {
-  SList       *pRows;
   SResultRowPosition *pPosition;
-  SResultRow **pResult;    // result list
   int32_t      size;       // number of result set
   int32_t      capacity;   // max capacity
   int32_t      curPos;     // current active result row index of pResult list
 } SResultRowInfo;
 
-typedef struct SResultRowPool {
-  int32_t elemSize;
-  int32_t blockSize;
-  int32_t numOfElemPerBlock;
-
-  struct {
-    int32_t blockIndex;
-    int32_t pos;
-  } position;
-
-  SArray* pData;    // SArray<void*>
-} SResultRowPool;
-
 struct STaskAttr;
 struct STaskRuntimeEnv;
 struct SUdfInfo;
@@ -109,25 +94,33 @@ void    resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo*
 int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo);
 void    closeAllResultRows(SResultRowInfo* pResultRowInfo);
 
-int32_t initResultRow(SResultRow *pResultRow);
+void    initResultRow(SResultRow *pResultRow);
-void    closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot);
+void    closeResultRow(SResultRow* pResultRow);
-bool    isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot);
+bool    isResultRowClosed(SResultRow* pResultRow);
-void    clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow);
 
 struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset);
 
-void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr);
 int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable);
 
-static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) {
-  assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size);
+  ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size);
-  return pResultRowInfo->pResult[slot];
+  SResultRowPosition* pos = &pResultRowInfo->pPosition[slot];
+
+  SFilePage*  bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
+  SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
+  return pRow;
+}
+
+static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
+  SFilePage*  bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
+  SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
+  return pRow;
 }
 
 static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset,
                                              int32_t offset) {
   assert(rowOffset >= 0 && pQueryAttr != NULL);
-
+  ASSERT(0);
 //  int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery);
 //  return ((char *)page->data) + rowOffset + offset * numOfRows;
 }
@@ -139,22 +132,11 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs
   return (char*) page + rowOffset + offset * numOfRows;
 }
 
-//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
-//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
-
-__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr);
-
-SResultRow* getNewResultRow(SResultRowPool* p);
-
 typedef struct {
   SArray* pResult;     // SArray<SResPair>
   int32_t colId;
 } SStddevInterResult;
 
-void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen);
-SArray* interResFromBinary(const char* data, int32_t len);
-void freeInterResult(void* param);
-
 void    initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo);
 void    initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
 

source/libs/executor/inc/executorimpl.h

@@ -69,7 +69,7 @@ enum {
 
 typedef struct SResultRowCell {
   uint64_t    groupId;
-  SResultRow* pRow;
+  SResultRowPosition pos;
 } SResultRowCell;
 
 /**
@@ -277,8 +277,6 @@ typedef struct STaskRuntimeEnv {
   char*           keyBuf;               // window key buffer
   // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
   char**          prevRow;
-  SResultRowPool* pool;
-
   SArray*         prevResult;  // intermediate result, SArray<SInterResult>
   STSBuf*         pTsBuf;      // timestamp filter list
   STSCursor       cur;
@@ -364,6 +362,7 @@ typedef struct SSourceDataInfo {
   int32_t               index;
   SRetrieveTableRsp    *pRsp;
   uint64_t              totalRows;
+  int32_t               code;
   EX_SOURCE_STATUS      status;
 } SSourceDataInfo;
 

source/libs/executor/src/executil.c

@@ -53,15 +53,13 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) {
 
 int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) {
   pResultRowInfo->size     = 0;
-  pResultRowInfo->curPos  = -1;
+  pResultRowInfo->curPos   = -1;
   pResultRowInfo->capacity = size;
-
-  pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES);
   pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition));
-  if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) {
+
+  if (pResultRowInfo->pPosition == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -71,17 +69,17 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
   }
 
   if (pResultRowInfo->capacity == 0) {
-    assert(pResultRowInfo->pResult == NULL);
+//    assert(pResultRowInfo->pResult == NULL);
     return;
   }
 
   for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    if (pResultRowInfo->pResult[i]) {
+//    if (pResultRowInfo->pResult[i]) {
-      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
+//      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
-    }
+//    }
   }
   
-  taosMemoryFreeClear(pResultRowInfo->pResult);
+  taosMemoryFreeClear(pResultRowInfo->pPosition);
 }
 
 void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) {
@@ -90,8 +88,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   }
 
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
+//    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes);
+//    clearResultRow(pRuntimeEnv, pWindowRes);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -101,14 +99,13 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   }
 
   pResultRowInfo->size     = 0;
-  pResultRowInfo->curPos  = -1;
 }
 
 int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) {
   int32_t i = 0;
-  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
+//  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
-    ++i;
+//    ++i;
-  }
+//  }
   
   return i;
 }
@@ -117,21 +114,22 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
   assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    SResultRow* pRow = pResultRowInfo->pResult[i];
+//    ASSERT(0);
-    if (pRow->closed) {
+//    SResultRow* pRow = pResultRowInfo->pResult[i];
-      continue;
+//    if (pRow->closed) {
-    }
+//      continue;
+//    }
     
-    pRow->closed = true;
+//    pRow->closed = true;
   }
 }
 
-bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) {
+bool isResultRowClosed(SResultRow* pRow) {
-  return (getResultRow(pResultRowInfo, slot)->closed == true);
+  return (pRow->closed == true);
 }
 
-void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
+void closeResultRow(SResultRow* pResultRow) {
-  getResultRow(pResultRowInfo, slot)->closed = true;
+  pResultRow->closed = true;
 }
 
 void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) {
@@ -181,29 +179,6 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   return rowSize;
 }
 
-SResultRow* getNewResultRow(SResultRowPool* p) {
-  if (p == NULL) {
-    return NULL;
-  }
-
-  void* ptr = NULL;
-  if (p->position.pos == 0) {
-    ptr = taosMemoryCalloc(1, p->blockSize);
-    taosArrayPush(p->pData, &ptr);
-
-  } else {
-    size_t last = taosArrayGetSize(p->pData);
-
-    void** pBlock = taosArrayGet(p->pData, last - 1);
-    ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos;
-  }
-
-  p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
-  initResultRow(ptr);
-
-  return ptr;
-}
-
 void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
   assert(pGroupResInfo != NULL);
 
@@ -261,8 +236,9 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
   return (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
 }
 
-static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) {
+static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) {
   STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
+  ASSERT(0);
 
   for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) {
     int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId;
@@ -305,25 +281,26 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *
     return -1;
   }
 
+  ASSERT(0);
   STableQueryInfo** pList = supporter->pTableQueryInfo;
-  SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos];
+//  SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos];
 //  SResultRow * pWindowRes1 = getResultRow(&(pList[left]->resInfo), leftPos);
-  TSKEY leftTimestamp = pWindowRes1->win.skey;
+//  TSKEY leftTimestamp = pWindowRes1->win.skey;
 
 //  SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo);
 //  SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
-  SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos];
+//  SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos];
-  TSKEY rightTimestamp = pWindowRes2->win.skey;
+//  TSKEY rightTimestamp = pWindowRes2->win.skey;
 
-  if (leftTimestamp == rightTimestamp) {
+//  if (leftTimestamp == rightTimestamp) {
     return 0;
-  }
+//  }
 
-  if (supporter->order == TSDB_ORDER_ASC) {
+//  if (supporter->order == TSDB_ORDER_ASC) {
-    return (leftTimestamp > rightTimestamp)? 1:-1;
+//    return (leftTimestamp > rightTimestamp)? 1:-1;
-  } else {
+//  } else {
-    return (leftTimestamp < rightTimestamp)? 1:-1;
+//    return (leftTimestamp < rightTimestamp)? 1:-1;
-  }
+//  }
 }
 
 int32_t tsAscOrder(const void* p1, const void* p2) {
@@ -331,11 +308,12 @@ int32_t tsAscOrder(const void* p1, const void* p2) {
   SResultRowCell* pc2 = (SResultRowCell*) p2;
 
   if (pc1->groupId == pc2->groupId) {
-    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
+    ASSERT(0);
-      return 0;
+//    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
-    } else {
+//      return 0;
-      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1;
+//    } else {
-    }
+//      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1;
+//    }
   } else {
     return (pc1->groupId < pc2->groupId)? -1:1;
   }
@@ -346,11 +324,12 @@ int32_t tsDescOrder(const void* p1, const void* p2) {
   SResultRowCell* pc2 = (SResultRowCell*) p2;
 
   if (pc1->groupId == pc2->groupId) {
-    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
+    ASSERT(0);
-      return 0;
+//    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
-    } else {
+//      return 0;
-      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1;
+//    } else {
-    }
+//      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1;
+//    }
   } else {
     return (pc1->groupId < pc2->groupId)? -1:1;
   }
@@ -384,13 +363,13 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe
       break;
     }
 
-    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pResultRowCell->pRow, rowCellInfoOffset);
+    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset);
     if (num <= 0) {
       continue;
     }
 
-    taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pRow);
+    taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pos);
-    pResultRowCell->pRow->numOfRows = (uint32_t) num;
+//    pResultRowCell->pRow->numOfRows = (uint32_t) num;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -449,9 +428,10 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
     int32_t tableIndex = tMergeTreeGetChosenIndex(pTree);
 
     SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo;
-    SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
+    ASSERT(0);
+    SResultRow  *pWindowRes = NULL;//getResultRow(pBuf, pWindowResInfo, cs.rowIndex[tableIndex]);
 
-    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset);
+    int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset);
     if (num <= 0) {
       cs.rowIndex[tableIndex] += 1;
 

source/libs/executor/src/executorimpl.c

@@ -426,18 +426,10 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env)
     newCapacity += 4;
   }
 
-  char* t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES));
-  if (t == NULL) {
-    longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY);
-  }
-
   pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition));
-  pResultRowInfo->pResult = (SResultRow**)t;
 
   int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity;
-  memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc);
   memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition));
-
   pResultRowInfo->capacity = (int32_t)newCapacity;
 }
 
@@ -458,9 +450,8 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR
     if (p1 != NULL) {
       if (pResultRowInfo->size == 0) {
         existed = false;
-        assert(pResultRowInfo->curPos == -1);
       } else if (pResultRowInfo->size == 1) {
-        existed = (pResultRowInfo->pResult[0] == (*p1));
+//        existed = (pResultRowInfo->pResult[0] == (*p1));
       } else {  // check if current pResultRowInfo contains the existed pResultRow
         SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo);
         int64_t* index =
@@ -479,6 +470,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR
   return p1 != NULL;
 }
 
+#if 0
 static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid,
                                           char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) {
   bool existed = false;
@@ -496,16 +488,16 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR
     if (p1 != NULL) {
       if (pResultRowInfo->size == 0) {
         existed = false;
-        assert(pResultRowInfo->curPos == -1);
+//        assert(pResultRowInfo->curPos == -1);
       } else if (pResultRowInfo->size == 1) {
-        existed = (pResultRowInfo->pResult[0] == (*p1));
+//        existed = (pResultRowInfo->pResult[0] == (*p1));
-        pResultRowInfo->curPos = 0;
+//        pResultRowInfo->curPos = 0;
       } else {  // check if current pResultRowInfo contains the existed pResultRow
         SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo);
         int64_t* index =
             taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
         if (index != NULL) {
-          pResultRowInfo->curPos = (int32_t)*index;
+//          pResultRowInfo->curPos = (int32_t)*index;
           existed = true;
         } else {
           existed = false;
@@ -555,6 +547,7 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR
 
   return pResultRowInfo->pResult[pResultRowInfo->curPos];
 }
+#endif
 
 SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) {
   SFilePage* pData = NULL;
@@ -599,65 +592,75 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
 static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t tid,
                                              char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId,
                                              SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) {
-  bool existed = false;
+  bool existInCurrentResusltRowInfo = false;
   SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId);
 
-  SResultRow** p1 = (SResultRow**)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
+  SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
 
   // in case of repeat scan/reverse scan, no new time window added.
   if (isIntervalQuery) {
     if (!masterscan) {  // the *p1 may be NULL in case of sliding+offset exists.
-      return (p1 != NULL) ? *p1 : NULL;
+      if (p1 != NULL) {
+        return getResultRowByPos(pResultBuf, p1);
+      } else {
+        return NULL;
+      }
     }
 
     if (p1 != NULL) {
       if (pResultRowInfo->size == 0) {
-        existed = false;
+        existInCurrentResusltRowInfo = false;  // this time window created by other timestamp that does not belongs to current table.
         assert(pResultRowInfo->curPos == -1);
       } else if (pResultRowInfo->size == 1) {
-        existed = (pResultRowInfo->pResult[0] == (*p1));
+        ASSERT(0);
-        pResultRowInfo->curPos = 0;
+//        existInCurrentResusltRowInfo = (pResultRowInfo->pResult[0] == (*p1));
-      } else {  // check if current pResultRowInfo contains the existed pResultRow
+      } else {  // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow
         SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
         int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
         if (index != NULL) {
-          pResultRowInfo->curPos = (int32_t)*index;
+          // TODO check the scan order for current opened time window
-          existed = true;
+//          pResultRowInfo->curPos = (int32_t)*index;
+          existInCurrentResusltRowInfo = true;
         } else {
-          existed = false;
+          existInCurrentResusltRowInfo = false;
         }
       }
     }
   } else {
-    // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object.
+    // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the pResultRowInfo object.
     if (p1 != NULL) {
-      return *p1;
+      return getResultRowByPos(pResultBuf, p1);
     }
   }
 
-  if (!existed) {
+  SResultRow* pResult = NULL;
-    prepareResultListBuffer(pResultRowInfo, pTaskInfo->env);
+  if (!existInCurrentResusltRowInfo) {
-
+    // 1. close current opened time window
-    SResultRow* pResult = NULL;
+    if (pResultRowInfo->curPos != -1) {  // todo extract function
-    if (p1 == NULL) {
+      SResultRowPosition* pos = &pResultRowInfo->pPosition[pResultRowInfo->curPos];
-      pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize);
+      SFilePage* pPage = getBufPage(pResultBuf, pos->pageId);
-      int32_t ret = initResultRow(pResult);
+      SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset);
-      if (ret != TSDB_CODE_SUCCESS) {
+      closeResultRow(pRow);
-        longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+      releaseBufPage(pResultBuf, pPage);
-      }
-
-      // add a new result set for a new group
-      taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES);
-      SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult};
-      taosArrayPush(pSup->pResultRowArrayList, &cell);
-    } else {
-      pResult = *p1;
     }
 
+    prepareResultListBuffer(pResultRowInfo, pTaskInfo->env);
+    if (p1 == NULL) {
+      pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize);
+      initResultRow(pResult);
+
+      // add a new result set for a new group
+      SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
+      taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, POINTER_BYTES);
+      SResultRowCell cell = {.groupId = tableGroupId, .pos = pos};
+      taosArrayPush(pSup->pResultRowArrayList, &cell);
+    } else {
+      pResult = getResultRowByPos(pResultBuf, p1);
+    }
+
+    // 2. set the new time window to be the new active time window
     pResultRowInfo->curPos = pResultRowInfo->size;
-    pResultRowInfo->pPosition[pResultRowInfo->size] =
+    pResultRowInfo->pPosition[pResultRowInfo->size++] = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
-        (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
-    pResultRowInfo->pResult[pResultRowInfo->size++] = pResult;
 
     int64_t index = pResultRowInfo->curPos;
     SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo);
@@ -669,7 +672,7 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR
     longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW);
   }
 
-  return pResultRowInfo->pResult[pResultRowInfo->curPos];
+  return pResult;
 }
 
 static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey,
@@ -693,7 +696,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T
 }
 
 // get the correct time window according to the handled timestamp
-static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
+static STimeWindow getActiveTimeWindow(SDiskbasedBuf * pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
                                        int32_t precision, STimeWindow* win) {
   STimeWindow w = {0};
 
@@ -701,7 +704,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t
     getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true);
     w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1;
   } else {
-    w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win;
+    w = getResultRow(pBuf, pResultRowInfo, pResultRowInfo->curPos)->win;
   }
 
   if (w.skey > ts || w.ekey < ts) {
@@ -730,7 +733,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t
 // get the correct time window according to the handled timestamp
 static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) {
   STimeWindow w = {0};
-
+#if 0
   if (pResultRowInfo->curPos == -1) {  // the first window, from the previous stored value
                                        //    getInitialStartTimeWindow(pQueryAttr, ts, &w);
 
@@ -742,7 +745,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in
       w.ekey = w.skey + pQueryAttr->interval.interval - 1;
     }
   } else {
-    w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win;
+    w = pRow->win;
   }
 
   /*
@@ -752,6 +755,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in
   if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) {
     w.ekey = pQueryAttr->window.ekey;
   }
+#endif
 
   return w;
 }
@@ -816,8 +820,8 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowI
   assert(win->skey <= win->ekey);
   SDiskbasedBuf* pResultBuf = pRuntimeEnv->pResultBuf;
 
-  SResultRow* pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE,
+  SResultRow* pResultRow = NULL;//doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE,
-                                                  masterscan, tableGroupId);
+//                                                  masterscan, tableGroupId);
   if (pResultRow == NULL) {
     *pResult = NULL;
     return TSDB_CODE_SUCCESS;
@@ -840,8 +844,7 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowI
   return TSDB_CODE_SUCCESS;
 }
 
-static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, SqlFunctionCtx* pCtx,
+static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
-                                            int32_t numOfOutput, int32_t* rowCellInfoOffset);
 
 static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan,
                                           SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
@@ -859,7 +862,7 @@ static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_
   // set time window for current result
   pResultRow->win = (*win);
   *pResult = pResultRow;
-  setResultRowOutputBufInitCtx_rv(pAggSup->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -909,9 +912,9 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
   return forwardStep;
 }
 
-static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery,
+static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) {
-                                   bool timeWindowInterpo) {
   int64_t skey = TSKEY_INITIAL_VAL;
+#if 0
   int32_t i = 0;
   for (i = pResultRowInfo->size - 1; i >= 0; --i) {
     SResultRow* pResult = pResultRowInfo->pResult[i];
@@ -963,6 +966,7 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey
       pResultRowInfo->curPos = i + 1;  // current not closed result object
     }
   }
+#endif
 }
 
 static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey,
@@ -1041,7 +1045,9 @@ static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin) {
 static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol,
                              int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
   SScalarParam intervalParam = {.numOfRows = 5, .columnData = pTimeWindowData};  //TODO move out of this function
-  updateTimeWindowInfo(pTimeWindowData, pWin);
+  if (pTimeWindowData != NULL) {
+    updateTimeWindowInfo(pTimeWindowData, pWin);
+  }
 
   for (int32_t k = 0; k < numOfOutput; ++k) {
     pCtx[k].startTs = pWin->skey;
@@ -1253,8 +1259,8 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx,
         ASSERT(pCtx[i].input.pData[j] != NULL);
       }
     }
-    //    setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns);
 
+    //    setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns);
     //      uint32_t flag = pOperator->pExpr[i].base.pParam[0].pCol->flag;
     //      if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) ||
     //          (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) {
@@ -1551,14 +1557,14 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
   if (pSDataBlock->pDataBlock != NULL) {
     SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0);
     tsCols = (int64_t*)pColDataInfo->pData;
-//    assert(tsCols[0] == pSDataBlock->info.window.skey &&
+//    assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] ==
-//           tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey);
+//           pSDataBlock->info.window.ekey);
   }
 
   int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1);
   TSKEY   ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan);
 
-  STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win);
+  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win);
   bool        masterScan = true;
 
   SResultRow* pResult = NULL;
@@ -1581,6 +1587,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
 
   // prev time window not interpolation yet.
   int32_t curIndex = pResultRowInfo->curPos;
+
+#if 0
   if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) {
     for (int32_t j = prevIndex; j < curIndex; ++j) {  // previous time window may be all closed already.
       SResultRow* pRes = getResultRow(pResultRowInfo, j);
@@ -1615,6 +1623,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
   }
+#endif
 
   // window start key interpolation
   doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep,
@@ -1886,9 +1895,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     /*int32_t ret = */ generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals);
-    int32_t ret =
+    int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
-        setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len,
-                                   0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup);
     if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
     }
@@ -2010,12 +2017,11 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo
   SqlFunctionCtx* pCtx = binfo->pCtx;
 
   SResultRow* pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId,
-                                                     pTaskInfo, true, pAggSup);
+                                                     pTaskInfo, false, pAggSup);
   assert(pResultRow != NULL);
 
   setResultRowKey(pResultRow, pData, type);
-
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset);
-  setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2170,8 +2176,8 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num
         }
       }
       pCtx->resDataInfo.interBufSize = env.calcMemSize;
-    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN) {
+    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) {
-    } else if (pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) {
+      pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element.
     }
 
     pCtx->input.numOfInputCols = pFunct->numOfParams;
@@ -3430,10 +3436,8 @@ void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   }
 }
 
-// TODO fix this bug.
+void initResultRow(SResultRow* pResultRow) {
-int32_t initResultRow(SResultRow* pResultRow) {
   pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow));
-  return TSDB_CODE_SUCCESS;
 }
 
 /*
@@ -3449,7 +3453,9 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t
   SqlFunctionCtx* pCtx = pInfo->pCtx;
   SSDataBlock*    pDataBlock = pInfo->pRes;
   int32_t*        rowCellInfoOffset = pInfo->rowCellInfoOffset;
+
   SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo;
+  initResultRowInfo(pResultRowInfo, 16);
 
   int64_t     tid = 0;
   int64_t     groupId = 0;
@@ -3610,9 +3616,11 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
 
     SFilePage*  bufPage = getBufPage(pBuf, pPos->pageId);
     SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset);
-    if (!isResultRowClosed(pResultRowInfo, i)) {
+
-      continue;
+    // TODO ignore the close status anyway.
-    }
+//    if (!isResultRowClosed(pRow)) {
+//      continue;
+//    }
 
     for (int32_t j = 0; j < numOfOutput; ++j) {
       pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset);
@@ -3622,7 +3630,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
         continue;
       }
 
-      if (pCtx[j].fpSet.process) {  // TODO set the dummy function.
+      if (pCtx[j].fpSet.process) {  // TODO set the dummy function, to avoid the check for null ptr.
         pCtx[j].fpSet.finalize(&pCtx[j]);
       }
 
@@ -3765,8 +3773,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pRes
   }
 }
 
-void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, SqlFunctionCtx* pCtx,
+void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) {
-                                     int32_t numOfOutput, int32_t* rowCellInfoOffset) {
   for (int32_t i = 0; i < numOfOutput; ++i) {
     pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
 
@@ -3779,18 +3786,13 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, S
       continue;
     }
 
-    //    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
-    //      if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput;
-    //    }
-
     if (!pResInfo->initialized && pCtx[i].functionId != -1) {
       pCtx[i].fpSet.init(&pCtx[i], pResInfo);
     }
   }
 }
 
-void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId,
+void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) {
-                              SExecTaskInfo* pTaskInfo) {
   // for simple group by query without interval, all the tables belong to one group result.
   int64_t uid = 0;
   int64_t tid = 0;
@@ -3809,14 +3811,13 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i
    * all group belong to one result set, and each group result has different group id so set the id to be one
    */
   if (pResultRow->pageId == -1) {
-    int32_t ret =
+    int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize);
-        addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.pRes->info.rowSize);
     if (ret != TSDB_CODE_SUCCESS) {
       return;
     }
   }
 
-  setResultRowOutputBufInitCtx_rv(pAggInfo->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
+  setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset);
 }
 
 void setExecutionContext(int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey, SExecTaskInfo* pTaskInfo,
@@ -4132,7 +4133,7 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu
   //  if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
   //    return;
   //  }
-
+#if 0
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
     SResultRow* pResult = pResultRowInfo->pResult[i];
 
@@ -4146,6 +4147,8 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu
       pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes));
     }
   }
+#endif
+
 }
 
 static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) {
@@ -5001,12 +5004,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
 
 int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*)param;
-  pSourceDataInfo->pRsp = pMsg->pData;
+  if (code == TSDB_CODE_SUCCESS) {
+    pSourceDataInfo->pRsp = pMsg->pData;
 
-  SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
+    SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
-  pRsp->numOfRows = htonl(pRsp->numOfRows);
+    pRsp->numOfRows = htonl(pRsp->numOfRows);
-  pRsp->useconds = htobe64(pRsp->useconds);
+    pRsp->compLen   = htonl(pRsp->compLen);
-  pRsp->compLen = htonl(pRsp->compLen);
+    pRsp->useconds  = htobe64(pRsp->useconds);
+  } else {
+    pSourceDataInfo->code = code;
+  }
 
   pSourceDataInfo->status = EX_SOURCE_DATA_READY;
   tsem_post(&pSourceDataInfo->pEx->ready);
@@ -5262,7 +5269,6 @@ static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo* pOperator) {
          totalSources, endTs - startTs);
 
   tsem_wait(&pExchangeInfo->ready);
-
   pOperator->status = OP_RES_TO_RETURN;
   return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
 }
@@ -5306,18 +5312,22 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     }
 
     doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
-
     tsem_wait(&pExchangeInfo->ready);
 
-    SSourceDataInfo*       pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current);
+    SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current);
     SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
 
+    if (pDataInfo->code != TSDB_CODE_SUCCESS) {
+      qError("%s vgId:%d, taskID:0x%" PRIx64 " error happens, code:%s",
+             GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, tstrerror(pDataInfo->code));
+      pOperator->pTaskInfo->code = pDataInfo->code;
+      return NULL;
+    }
+
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
-
     if (pRsp->numOfRows == 0) {
-      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
+      qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next",
-             " try next",
              GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
              pDataInfo->totalRows, pLoadInfo->totalRows);
 
@@ -7585,10 +7595,10 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
 int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey) {
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
 
-  pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
+  pAggSup->resultRowSize       = getResultRowSize(pCtx, numOfOutput);
-  pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES);
+  pAggSup->keyBuf              = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES);
   pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK);
-  pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
+  pAggSup->pResultRowListSet   = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
   pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell));
 
   if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL ||
@@ -8761,8 +8771,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
 
       pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
 
-      pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
+//      pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
-      pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType);
+//      pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType);
     } else {
       ASSERT(0);
     }

source/libs/index/inc/indexFstRegex.h

@@ -63,9 +63,10 @@ typedef struct {
 
 FstRegex *regexCreate(const char *str);
 
-void regexSetup(FstRegex *regex, uint32_t size, const char *str);
+uint32_t regexAutomStart(FstRegex *regex);
-
+bool     regexAutomIsMatch(FstRegex *regex, uint32_t state);
-// uint32_t regexStart()
+bool     regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null);
+bool     regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result);
 
 #ifdef __cplusplus
 }

source/libs/index/inc/indexFstSparse.h

@@ -23,9 +23,9 @@ extern "C" {
 #endif
 
 typedef struct FstSparseSet {
-  SArray *dense;
+  uint32_t *dense;
-  SArray *sparse;
+  uint32_t *sparse;
-  int32_t size;
+  int32_t   size;
 } FstSparseSet;
 
 FstSparseSet *sparSetCreate(int32_t sz);

source/libs/index/src/index.c

@@ -27,7 +27,7 @@
 #endif
 
 #define INDEX_NUM_OF_THREADS 4
-#define INDEX_QUEUE_SIZE 200
+#define INDEX_QUEUE_SIZE     200
 
 void* indexQhandle = NULL;
 

source/libs/index/src/indexCache.c

@@ -21,8 +21,8 @@
 
 #define MAX_INDEX_KEY_LEN 256  // test only, change later
 
-#define MEM_TERM_LIMIT 10 * 10000
+#define MEM_TERM_LIMIT     10 * 10000
-#define MEM_THRESHOLD 1024 * 1024
+#define MEM_THRESHOLD      1024 * 1024
 #define MEM_ESTIMATE_RADIO 1.5
 
 static void indexMemRef(MemTable* tbl);

source/libs/index/src/indexFstCommon.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+YAML:9:25: error: unknown key 'AlignConsecutiveMacros' * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
  * it under the terms of the GNU Affero General Public License, version 3

source/libs/index/src/indexFstRegex.c

@@ -14,6 +14,7 @@
  */
 
 #include "indexFstRegex.h"
+#include "indexFstDfa.h"
 #include "indexFstSparse.h"
 
 FstRegex *regexCreate(const char *str) {
@@ -26,9 +27,35 @@ FstRegex *regexCreate(const char *str) {
   memcpy(orig, str, sz);
 
   regex->orig = orig;
+
+  // construct insts based on str
+  SArray *insts = NULL;
+
+  FstDfaBuilder *builder = dfaBuilderCreate(insts);
+  regex->dfa = dfaBuilderBuild(builder);
+  return regex;
 }
 
-void regexSetup(FstRegex *regex, uint32_t size, const char *str) {
+uint32_t regexAutomStart(FstRegex *regex) {
-  // return
+  ///// no nothing
-  // return;
+  return 0;
+}
+bool regexAutomIsMatch(FstRegex *regex, uint32_t state) {
+  if (regex->dfa != NULL && dfaIsMatch(regex->dfa, state)) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null) {
+  // make frame happy
+  return null;
+}
+
+bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result) {
+  if (regex->dfa == NULL) {
+    return false;
+  }
+  return dfaAccept(regex->dfa, state, byte, result);
 }

source/libs/index/src/indexFstSparse.c

@@ -21,47 +21,44 @@ FstSparseSet *sparSetCreate(int32_t sz) {
     return NULL;
   }
 
-  ss->dense = taosArrayInit(sz, sizeof(uint32_t));
+  ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
-  ss->sparse = taosArrayInit(sz, sizeof(uint32_t));
+  ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
-  ss->size = sz;
+  ss->size = 0;
   return ss;
 }
 void sparSetDestroy(FstSparseSet *ss) {
   if (ss == NULL) {
     return;
   }
-  taosArrayDestroy(ss->dense);
+  taosMemoryFree(ss->dense);
-  taosArrayDestroy(ss->sparse);
+  taosMemoryFree(ss->sparse);
   taosMemoryFree(ss);
 }
-uint32_t sparSetLen(FstSparseSet *ss) { return ss == NULL ? 0 : ss->size; }
+uint32_t sparSetLen(FstSparseSet *ss) {
+  // Get occupied size
+  return ss == NULL ? 0 : ss->size;
+}
 uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) {
   if (ss == NULL) {
     return 0;
   }
   uint32_t i = ss->size;
-  taosArraySet(ss->dense, i, &ip);
+  ss->dense[i] = ip;
-  taosArraySet(ss->sparse, ip, &i);
+  ss->sparse[ip] = i;
   ss->size += 1;
   return i;
 }
 uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) {
-  if (i >= taosArrayGetSize(ss->dense)) {
+  // check later
-    return 0;
+  return ss->dense[i];
-  }
-  uint32_t *v = taosArrayGet(ss->dense, i);
-  return *v;
 }
 bool sparSetContains(FstSparseSet *ss, uint32_t ip) {
-  if (ip >= taosArrayGetSize(ss->sparse)) {
+  uint32_t i = ss->sparse[ip];
+  if (i < ss->size && ss->dense[i] == ip) {
+    return true;
+  } else {
     return false;
   }
-  uint32_t i = *(uint32_t *)taosArrayGet(ss->sparse, ip);
-  if (i >= taosArrayGetSize(ss->dense)) {
-    return false;
-  }
-  uint32_t v = *(uint32_t *)taosArrayGet(ss->dense, i);
-  return v == ip;
 }
 void sparSetClear(FstSparseSet *ss) {
   if (ss == NULL) {

source/libs/nodes/inc/nodesUtil.h

@@ -20,12 +20,16 @@
 extern "C" {
 #endif
 
-#define nodesFatal(param, ...)  qFatal("NODES: " param, __VA_ARGS__)
+#define nodesFatal(...)  qFatal("NODES: " __VA_ARGS__)
-#define nodesError(param, ...)  qError("NODES: " param, __VA_ARGS__)
+#define nodesError(...)  qError("NODES: " __VA_ARGS__)
-#define nodesWarn(param, ...)   qWarn("NODES: " param, __VA_ARGS__)
+#define nodesWarn(...)   qWarn("NODES: " __VA_ARGS__)
-#define nodesInfo(param, ...)   qInfo("NODES: " param, __VA_ARGS__)
+#define nodesInfo(...)   qInfo("NODES: " __VA_ARGS__)
-#define nodesDebug(param, ...)  qDebug("NODES: " param, __VA_ARGS__)
+#define nodesDebug(...)  qDebug("NODES: " __VA_ARGS__)
-#define nodesTrace(param, ...)  qTrace("NODES: " param, __VA_ARGS__)
+#define nodesTrace(...)  qTrace("NODES: " __VA_ARGS__)
+
+#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
 #ifdef __cplusplus
 }

source/libs/nodes/src/nodesToSQLFuncs.c

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdnodes.h"
+#include "nodesUtil.h"
+#include "plannodes.h"
+#include "querynodes.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "thash.h"
+
+char *gOperatorStr[] = {NULL, "+", "-", "*", "/", "%", "&", "|", ">", ">=", "<", "<=", "=", "<>", 
+                        "IN", "NOT IN", "LIKE", "NOT LIKE", "MATCH", "NMATCH", "IS NULL", "IS NOT NULL",
+                        "IS TRUE", "IS FALSE", "IS UNKNOWN", "IS NOT TRUE", "IS NOT FALSE", "IS NOT UNKNOWN"};
+char *gLogicConditionStr[] = {"AND", "OR", "NOT"};
+
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
+  switch (pNode->type) {
+    case QUERY_NODE_COLUMN: {
+      SColumnNode *colNode = (SColumnNode *)pNode;
+      if (colNode->dbName[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->dbName);
+      }
+      
+      if (colNode->tableAlias[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableAlias);
+      } else if (colNode->tableName[0]) {
+        *len += snprintf(buf + *len, bufSize - *len, "`%s`.", colNode->tableName);
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName);
+      
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_VALUE:{
+      SValueNode *colNode = (SValueNode *)pNode;
+      char *t = nodesGetStrValueFromNode(colNode);
+      if (NULL == t) {
+        nodesError("fail to get str value from valueNode");
+        NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, "%s", t);
+      taosMemoryFree(t);
+      
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_OPERATOR: {
+      SOperatorNode* pOpNode = (SOperatorNode*)pNode;
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      if (pOpNode->pLeft) {
+        NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len));
+      }
+
+      if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) {
+        nodesError("unknown operation type:%d", pOpNode->opType);
+        NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]);
+
+      if (pOpNode->pRight) {
+        NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len));
+      }    
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_LOGIC_CONDITION:{
+      SLogicConditionNode* pLogicNode = (SLogicConditionNode*)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      
+      FOREACH(node, pLogicNode->pParameterList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]);
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_FUNCTION:{
+      SFunctionNode* pFuncNode = (SFunctionNode*)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "%s(", pFuncNode->functionName);
+      
+      FOREACH(node, pFuncNode->pParameterList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, ", ");
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    case QUERY_NODE_NODE_LIST:{
+      SNodeListNode* pListNode = (SNodeListNode *)pNode;
+      SNode* node = NULL;
+      bool first = true;
+      
+      *len += snprintf(buf + *len, bufSize - *len, "(");
+      
+      FOREACH(node, pListNode->pNodeList) {
+        if (!first) {
+          *len += snprintf(buf + *len, bufSize - *len, ", ");
+        }
+        NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
+        first = false;
+      }
+      
+      *len += snprintf(buf + *len, bufSize - *len, ")");
+
+      return TSDB_CODE_SUCCESS;
+    }
+    default:
+      break;
+  }
+  
+  nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type));
+  NODES_RET(TSDB_CODE_QRY_APP_ERROR);
+}

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -17,7 +17,8 @@
 
 typedef enum ETraversalOrder {
   TRAVERSAL_PREORDER = 1,
-  TRAVERSAL_POSTORDER
+  TRAVERSAL_INORDER,
+  TRAVERSAL_POSTORDER,
 } ETraversalOrder;
 
 static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -790,6 +790,71 @@ void* nodesGetValueFromNode(SValueNode *pNode) {
   return NULL;
 }
 
+char* nodesGetStrValueFromNode(SValueNode *pNode) {
+  switch (pNode->node.resType.type) {
+    case TSDB_DATA_TYPE_BOOL: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%s", pNode->datum.b ? "true" : "false");
+      return buf;
+    }
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%" PRId64, pNode->datum.i);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%" PRIu64, pNode->datum.u);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE: {
+      void *buf = taosMemoryMalloc(MAX_NUM_STR_SIZE);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      sprintf(buf, "%e", pNode->datum.d);
+      return buf;
+    }
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_VARCHAR:
+    case TSDB_DATA_TYPE_VARBINARY: {
+      int32_t bufSize = varDataLen(pNode->datum.p) + 2 + 1;
+      void *buf = taosMemoryMalloc(bufSize);
+      if (NULL == buf) {
+        return NULL;
+      }
+      
+      snprintf(buf, bufSize, "'%s'", varDataVal(pNode->datum.p));
+      return buf;
+    }
+    default:
+      break;
+  }
+
+  return NULL;
+}
+
 bool nodesIsExprNode(const SNode* pNode) {
   ENodeType type = nodeType(pNode);
   return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);

source/libs/parser/inc/parAst.h

@@ -71,6 +71,7 @@ typedef enum ETableOptionType {
 typedef struct SAlterOption {
   int32_t type;
   SToken val;
+  SNodeList* pKeep;
 } SAlterOption;
 
 extern SToken nil_token;
@@ -121,6 +122,8 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
 SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt);
 SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt);
 SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, const SToken* pVal);
+SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
+SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
 SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions);
 SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pDbName);
 SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions);
@@ -129,6 +132,8 @@ SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt);
 SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, const SToken* pVal);
 SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pSma);
 SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs);
+SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep);
+SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption);
 SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment);
 SDataType createDataType(uint8_t type);
 SDataType createVarLenDataType(uint8_t type, const SToken* pLen);
@@ -145,6 +150,8 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int
 SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal);
 SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbNamePattern);
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName);
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword);
 SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal);
 SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
@@ -165,6 +172,16 @@ SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* p
 SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery);
 SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable);
 SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt);
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups);
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize);
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName);
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery);
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName);
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName);
 
 #ifdef __cplusplus
 }

source/libs/parser/inc/parInsertData.h

@@ -78,8 +78,6 @@ typedef struct STableDataBlocks {
   char       *pData;
   bool        cloned;
   STagData    tagData; 
-  char        tableName[TSDB_TABLE_NAME_LEN];
-  char        dbFName[TSDB_DB_FNAME_LEN];
   
   SParsedDataColInfo boundColumnInfo;
   SRowBuilder        rowBuilder;

source/libs/parser/inc/sql.y

@@ -137,7 +137,7 @@ db_options(A) ::= db_options(B) DAYS NK_INTEGER(C).
 db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C).                              { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); }
 db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); }
 db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); }
-db_options(A) ::= db_options(B) KEEP NK_INTEGER(C).                               { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, &C); }
+db_options(A) ::= db_options(B) KEEP integer_list(C).                             { A = setDatabaseKeepOption(pCxt, B, C); }
 db_options(A) ::= db_options(B) PRECISION NK_STRING(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); }
 db_options(A) ::= db_options(B) QUORUM NK_INTEGER(C).                             { A = setDatabaseOption(pCxt, B, DB_OPTION_QUORUM, &C); }
 db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C).                            { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); }
@@ -148,17 +148,23 @@ db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C).
 db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C).                        { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
 db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C).                          { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
 
-alter_db_options(A) ::= alter_db_option(B).                                       { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
+alter_db_options(A) ::= alter_db_option(B).                                       { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseAlterOption(pCxt, A, &B); }
-alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
+alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setDatabaseAlterOption(pCxt, B, &C); }
 
 %type alter_db_option                                                             { SAlterOption }
 %destructor alter_db_option                                                       { }
 alter_db_option(A) ::= BLOCKS NK_INTEGER(B).                                      { A.type = DB_OPTION_BLOCKS; A.val = B; }
 alter_db_option(A) ::= FSYNC NK_INTEGER(B).                                       { A.type = DB_OPTION_FSYNC; A.val = B; }
-alter_db_option(A) ::= KEEP NK_INTEGER(B).                                        { A.type = DB_OPTION_KEEP; A.val = B; }
+alter_db_option(A) ::= KEEP integer_list(B).                                      { A.type = DB_OPTION_KEEP; A.pKeep = B; }
 alter_db_option(A) ::= WAL NK_INTEGER(B).                                         { A.type = DB_OPTION_WAL; A.val = B; }
 alter_db_option(A) ::= QUORUM NK_INTEGER(B).                                      { A.type = DB_OPTION_QUORUM; A.val = B; }
 alter_db_option(A) ::= CACHELAST NK_INTEGER(B).                                   { A.type = DB_OPTION_CACHELAST; A.val = B; }
+alter_db_option(A) ::= REPLICA NK_INTEGER(B).                                     { A.type = DB_OPTION_REPLICA; A.val = B; }
+
+%type integer_list                                                                { SNodeList* }
+%destructor integer_list                                                          { nodesDestroyList($$); }
+integer_list(A) ::= NK_INTEGER(B).                                                { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C).                       { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
 
 /************************************************ create/drop table/stable ********************************************/
 cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
@@ -259,20 +265,20 @@ tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP.
 
 table_options(A) ::= .                                                            { A = createDefaultTableOptions(pCxt); }
 table_options(A) ::= table_options(B) COMMENT NK_STRING(C).                       { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); }
-table_options(A) ::= table_options(B) KEEP NK_INTEGER(C).                         { A = setTableOption(pCxt, B, TABLE_OPTION_KEEP, &C); }
+table_options(A) ::= table_options(B) KEEP integer_list(C).                       { A = setTableKeepOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) TTL NK_INTEGER(C).                          { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
 table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP.           { A = setTableSmaOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP.       { A = setTableRollupOption(pCxt, B, C); }
 table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C).                    { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); }
 table_options(A) ::= table_options(B) DELAY NK_INTEGER(C).                        { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); }
 
-alter_table_options(A) ::= alter_table_option(B).                                 { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
+alter_table_options(A) ::= alter_table_option(B).                                 { A = createDefaultAlterTableOptions(pCxt); A = setTableAlterOption(pCxt, A, &B); }
-alter_table_options(A) ::= alter_table_options(B) alter_table_option(C).          { A = setTableOption(pCxt, B, C.type, &C.val); }
+alter_table_options(A) ::= alter_table_options(B) alter_table_option(C).          { A = setTableAlterOption(pCxt, B, &C); }
 
 %type alter_table_option                                                          { SAlterOption }
 %destructor alter_table_option                                                    { }
 alter_table_option(A) ::= COMMENT NK_STRING(B).                                   { A.type = TABLE_OPTION_COMMENT; A.val = B; }
-alter_table_option(A) ::= KEEP NK_INTEGER(B).                                     { A.type = TABLE_OPTION_KEEP; A.val = B; }
+alter_table_option(A) ::= KEEP integer_list(B).                                   { A.type = TABLE_OPTION_KEEP; A.pKeep = B; }
 alter_table_option(A) ::= TTL NK_INTEGER(B).                                      { A.type = TABLE_OPTION_TTL; A.val = B; }
 
 %type col_name_list                                                               { SNodeList* }
@@ -295,6 +301,17 @@ cmd ::= SHOW QNODES.
 cmd ::= SHOW FUNCTIONS.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); }
 cmd ::= SHOW INDEXES FROM table_name_cond(A) from_db_opt(B).                      { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, A, B); }
 cmd ::= SHOW STREAMS.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); }
+cmd ::= SHOW ACCOUNTS.                                                            { pCxt->valid = false; generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); }
+cmd ::= SHOW APPS.                                                                { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT, NULL, NULL); }
+cmd ::= SHOW CONNECTIONS.                                                         { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT, NULL, NULL); }
+cmd ::= SHOW LICENCE.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); }
+cmd ::= SHOW CREATE DATABASE db_name(A).                                          { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
+cmd ::= SHOW CREATE TABLE full_table_name(A).                                     { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
+cmd ::= SHOW CREATE STABLE full_table_name(A).                                    { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }
+cmd ::= SHOW QUERIES.                                                             { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); }
+cmd ::= SHOW SCORES.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT, NULL, NULL); }
+cmd ::= SHOW TOPICS.                                                              { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT, NULL, NULL); }
+cmd ::= SHOW VARIABLES.                                                           { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT, NULL, NULL); }
 
 db_name_cond_opt(A) ::= .                                                         { A = createDefaultDatabaseCondValue(pCxt); }
 db_name_cond_opt(A) ::= db_name(B) NK_DOT.                                        { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
@@ -358,6 +375,45 @@ explain_options(A) ::= .
 explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C).                     { A = setExplainVerbose(pCxt, B, &C); }
 explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C).                      { A = setExplainRatio(pCxt, B, &C); }
 
+/************************************************ compact *************************************************************/
+cmd ::= COMPACT VNODES IN NK_LP integer_list(A) NK_RP.                            { pCxt->pRootNode = createCompactStmt(pCxt, A); }
+
+/************************************************ create/drop function ************************************************/
+cmd ::= CREATE agg_func_opt(A) FUNCTION function_name(B) 
+  AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E).                         { pCxt->pRootNode = createCreateFunctionStmt(pCxt, A, &B, &C, D, E); }
+cmd ::= DROP FUNCTION function_name(A).                                           { pCxt->pRootNode = createDropFunctionStmt(pCxt, &A); }
+
+%type agg_func_opt                                                                { bool }
+%destructor agg_func_opt                                                          { }
+agg_func_opt(A) ::= .                                                             { A = false; }
+agg_func_opt(A) ::= AGGREGATE.                                                    { A = true; }
+
+%type bufsize_opt                                                                 { int32_t }
+%destructor bufsize_opt                                                           { }
+bufsize_opt(A) ::= .                                                              { A = 0; }
+bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B).                                         { A = strtol(B.z, NULL, 10); }
+
+/************************************************ create/drop stream **************************************************/
+cmd ::= CREATE STREAM stream_name(A) INTO table_name(B) AS query_expression(C).   { pCxt->pRootNode = createCreateStreamStmt(pCxt, &A, &B, C); }
+cmd ::= DROP STREAM stream_name(A).                                               { pCxt->pRootNode = createDropStreamStmt(pCxt, &A); }
+
+/************************************************ kill connection/query ***********************************************/
+cmd ::= KILL CONNECTION NK_INTEGER(A).                                            { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); }
+cmd ::= KILL QUERY NK_INTEGER(A).                                                 { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &A); }
+
+/************************************************ merge/redistribute/ vgroup ******************************************/
+cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B).                                 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
+cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B).                          { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
+cmd ::= SPLIT VGROUP NK_INTEGER(A).                                               { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
+
+%type dnode_list                                                                  { SNodeList* }
+%destructor dnode_list                                                            { nodesDestroyList($$); }
+dnode_list(A) ::= DNODE NK_INTEGER(B).                                            { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+dnode_list(A) ::= dnode_list(B) DNODE NK_INTEGER(C).                              { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); }
+
+/************************************************ syncdb **************************************************************/
+cmd ::= SYNCDB db_name(A) REPLICA.                                                { pCxt->pRootNode = createSyncdbStmt(pCxt, &A); }
+
 /************************************************ select **************************************************************/
 cmd ::= query_expression(A).                                                      { pCxt->pRootNode = A; }
 
@@ -436,6 +492,10 @@ index_name(A) ::= NK_ID(B).
 %destructor topic_name                                                            { }
 topic_name(A) ::= NK_ID(B).                                                       { A = B; }
 
+%type stream_name                                                                 { SToken }
+%destructor stream_name                                                           { }
+stream_name(A) ::= NK_ID(B).                                                      { A = B; }
+
 /************************************************ expression **********************************************************/
 expression(A) ::= literal(B).                                                     { A = B; }
 //expression(A) ::= NK_QUESTION(B).                                                 { A = B; }

source/libs/parser/src/parAstCreater.c

@@ -138,18 +138,6 @@ static SDatabaseOptions* setDbMinRows(SAstCreateContext* pCxt, SDatabaseOptions*
   return pOptions;
 }
 
-static SDatabaseOptions* setDbKeep(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
-  int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
-    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid db option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-    pCxt->valid = false;
-    return pOptions;
-  }
-  pOptions->keep = val;
-  return pOptions;
-}
-
 static SDatabaseOptions* setDbPrecision(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
   char val[10] = {0};
   trimString(pVal->z, pVal->n, val, sizeof(val));
@@ -180,9 +168,9 @@ static SDatabaseOptions* setDbQuorum(SAstCreateContext* pCxt, SDatabaseOptions*
 
 static SDatabaseOptions* setDbReplica(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
   int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_DB_REPLICA_OPTION || val > TSDB_MAX_DB_REPLICA_OPTION) {
+  if (!(val == TSDB_MIN_DB_REPLICA_OPTION || val == TSDB_MAX_DB_REPLICA_OPTION)) {
     snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid db option replications: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+        "invalid db option replications: %"PRId64", only 1, 3 allowed", val);
     pCxt->valid = false;
     return pOptions;
   }
@@ -291,7 +279,6 @@ static void initSetDatabaseOptionFp() {
   setDbOptionFuncs[DB_OPTION_FSYNC] = setDbFsync;
   setDbOptionFuncs[DB_OPTION_MAXROWS] = setDbMaxRows;
   setDbOptionFuncs[DB_OPTION_MINROWS] = setDbMinRows;
-  setDbOptionFuncs[DB_OPTION_KEEP] = setDbKeep;
   setDbOptionFuncs[DB_OPTION_PRECISION] = setDbPrecision;
   setDbOptionFuncs[DB_OPTION_QUORUM] = setDbQuorum;
   setDbOptionFuncs[DB_OPTION_REPLICA] = setDbReplica;
@@ -303,18 +290,6 @@ static void initSetDatabaseOptionFp() {
   setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
 }
 
-static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
-  int64_t val = strtol(pVal->z, NULL, 10);
-  if (val < TSDB_MIN_KEEP || val > TSDB_MAX_KEEP) {
-    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
-        "invalid table option keep: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-    pCxt->valid = false;
-    return pOptions;
-  }
-  pOptions->keep = val;
-  return pOptions;
-}
-
 static STableOptions* setTableTtl(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
   int64_t val = strtol(pVal->z, NULL, 10);
   if (val < TSDB_MIN_DB_TTL_OPTION) {
@@ -363,7 +338,6 @@ static STableOptions* setTableDelay(SAstCreateContext* pCxt, STableOptions* pOpt
 }
 
 static void initSetTableOptionFp() {
-  setTableOptionFuncs[TABLE_OPTION_KEEP] = setTableKeep;
   setTableOptionFuncs[TABLE_OPTION_TTL] = setTableTtl;
   setTableOptionFuncs[TABLE_OPTION_COMMENT] = setTableComment;
   setTableOptionFuncs[TABLE_OPTION_FILE_FACTOR] = setTableFileFactor;
@@ -397,7 +371,9 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) {
       pCxt->valid = false;
     }
   }
-  trimEscape(pUserName);
+  if (pCxt->valid) {
+    trimEscape(pUserName);
+  }
   return pCxt->valid;
 }
 
@@ -472,45 +448,50 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t
 
 static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) {
   if (NULL == pDbName) {
-    pCxt->valid = (query ? NULL != pCxt->pQueryCxt->db : true);
+    if (query && NULL == pCxt->pQueryCxt->db) {
-    if (!pCxt->valid) {
+      generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED);
-      snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified");
+      pCxt->valid = false;
     }
   } else {
-    pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false;
+    if (pDbName->n >= TSDB_DB_NAME_LEN) {
+      generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z);
+      pCxt->valid = false;
+    }
+  }
+  if (pCxt->valid) {
+    trimEscape(pDbName);
   }
-  trimEscape(pDbName);
   return pCxt->valid;
 }
 
 static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
-  if (NULL == pTableName) {
+  if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) {
-    pCxt->valid = true;
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z);
-  } else {
+    pCxt->valid = false;
-    pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false;
+    return false;
   }
   trimEscape(pTableName);
-  return pCxt->valid;
+  return true;
 }
 
 static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) {
-  if (NULL == pColumnName) {
+  if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) {
-    pCxt->valid = true;
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z);
-  } else {
+    pCxt->valid = false;
-    pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false;
+    return false;
   }
   trimEscape(pColumnName);
-  return pCxt->valid;
+  return true;
 }
 
 static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
-  if (NULL == pIndexName) {
+  if (NULL != pIndexName && pIndexName->n >= TSDB_INDEX_NAME_LEN) {
+    generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pIndexName->z);
     pCxt->valid = false;
-  } else {
+    return false;
-    pCxt->valid = pIndexName->n < TSDB_INDEX_NAME_LEN ? true : false;
   }
   trimEscape(pIndexName);
-  return pCxt->valid;
+  return true;
 }
 
 SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) {
@@ -885,7 +866,9 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
   pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
   pOptions->maxRowsPerBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
   pOptions->minRowsPerBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
-  pOptions->keep = TSDB_DEFAULT_KEEP;
+  pOptions->keep0 = TSDB_DEFAULT_KEEP;
+  pOptions->keep1 = TSDB_DEFAULT_KEEP;
+  pOptions->keep2 = TSDB_DEFAULT_KEEP;
   pOptions->precision = TSDB_TIME_PRECISION_MILLI;
   pOptions->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
   pOptions->replica = TSDB_DEFAULT_DB_REPLICA_OPTION;
@@ -908,7 +891,9 @@ SNode* createDefaultAlterDatabaseOptions(SAstCreateContext* pCxt) {
   pOptions->fsyncPeriod = -1;
   pOptions->maxRowsPerBlock = -1;
   pOptions->minRowsPerBlock = -1;
-  pOptions->keep = -1;
+  pOptions->keep0 = -1;
+  pOptions->keep1 = -1;
+  pOptions->keep2= -1;
   pOptions->precision = -1;
   pOptions->quorum = -1;
   pOptions->replica = -1;
@@ -924,6 +909,48 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti
   return (SNode*)setDbOptionFuncs[type](pCxt, (SDatabaseOptions*)pOptions, pVal);
 }
 
+static bool checkAndSetKeepOption(SAstCreateContext* pCxt, SNodeList* pKeep, int32_t* pKeep0, int32_t* pKeep1, int32_t* pKeep2) {
+  int32_t numOfKeep = LIST_LENGTH(pKeep);
+  if (numOfKeep > 3 || numOfKeep < 1) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid number of keep options");
+    return false;
+  }
+
+  int32_t daysToKeep0 = strtol(((SValueNode*)nodesListGetNode(pKeep, 0))->literal, NULL, 10);
+  int32_t daysToKeep1 = numOfKeep > 1 ? strtol(((SValueNode*)nodesListGetNode(pKeep, 1))->literal, NULL, 10) : daysToKeep0;
+  int32_t daysToKeep2 = numOfKeep > 2 ? strtol(((SValueNode*)nodesListGetNode(pKeep, 2))->literal, NULL, 10) : daysToKeep1;
+  if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP ||
+      daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
+        "invalid option keep: %d, %d, %d valid range: [%d, %d]", daysToKeep0, daysToKeep1, daysToKeep2, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
+    return false;
+  }
+
+  if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) {
+    snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid keep value, should be keep0 <= keep1 <= keep2");
+    return false;
+  }
+
+  *pKeep0 = daysToKeep0;
+  *pKeep1 = daysToKeep1;
+  *pKeep2 = daysToKeep2;
+  return true;
+}
+
+SNode* setDatabaseKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
+  SDatabaseOptions* pOp = (SDatabaseOptions*)pOptions;
+  pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
+  return pOptions;
+}
+
+SNode* setDatabaseAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
+  if (DB_OPTION_KEEP == pAlterOption->type) {
+    return setDatabaseKeepOption(pCxt, pOptions, pAlterOption->pKeep);
+  } else {
+    return setDatabaseOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val);
+  }
+}
+
 SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) {
   if (!checkDbName(pCxt, pDbName, false)) {
     return NULL;
@@ -961,7 +988,9 @@ SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode*
 SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
   STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
   CHECK_OUT_OF_MEM(pOptions);
-  pOptions->keep = TSDB_DEFAULT_KEEP;
+  pOptions->keep0 = TSDB_DEFAULT_KEEP;
+  pOptions->keep1 = TSDB_DEFAULT_KEEP;
+  pOptions->keep2 = TSDB_DEFAULT_KEEP;
   pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
   pOptions->filesFactor = TSDB_DEFAULT_DB_FILE_FACTOR;
   pOptions->delay = TSDB_DEFAULT_DB_DELAY;
@@ -971,7 +1000,9 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
 SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
   STableOptions* pOptions = nodesMakeNode(QUERY_NODE_TABLE_OPTIONS);
   CHECK_OUT_OF_MEM(pOptions);
-  pOptions->keep = -1;
+  pOptions->keep0 = -1;
+  pOptions->keep1 = -1;
+  pOptions->keep2 = -1;
   pOptions->ttl = -1;
   pOptions->filesFactor = -1;
   pOptions->delay = -1;
@@ -997,6 +1028,20 @@ SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList*
   return pOptions;
 }
 
+SNode* setTableKeepOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pKeep) {
+  STableOptions* pOp = (STableOptions*)pOptions;
+  pCxt->valid = checkAndSetKeepOption(pCxt, pKeep, &pOp->keep0, &pOp->keep1, &pOp->keep2);
+  return pOptions;
+}
+
+SNode* setTableAlterOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) {
+  if (TABLE_OPTION_KEEP == pAlterOption->type) {
+    return setTableKeepOption(pCxt, pOptions, pAlterOption->pKeep);
+  } else {
+    return setTableOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val);
+  }
+}
+
 SNode* createColumnDefNode(SAstCreateContext* pCxt, const SToken* pColName, SDataType dataType, const SToken* pComment) {
   SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF);
   CHECK_OUT_OF_MEM(pCol);
@@ -1176,6 +1221,18 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, S
   return (SNode*)pStmt;
 }
 
+SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SHOW_CREATE_DATABASE_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
 SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) {
   char password[TSDB_USET_PASSWORD_LEN] = {0};
   if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) {
@@ -1388,3 +1445,63 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) {
   CHECK_OUT_OF_MEM(pStmt);
   return pStmt;
 }
+
+SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_COMPACT_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropFunctionStmt(SAstCreateContext* pCxt, const SToken* pFuncName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_FUNCTION_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createCreateStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName, const SToken* pTableName, SNode* pQuery) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createDropStreamStmt(SAstCreateContext* pCxt, const SToken* pStreamName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) {
+  SNode* pStmt = nodesMakeNode(type);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SPLIT_VGROUP_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}
+
+SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
+  SNode* pStmt = nodesMakeNode(QUERY_NODE_SYNCDB_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  return pStmt;
+}

source/libs/parser/src/parInsert.c

@@ -52,8 +52,6 @@ typedef struct SInsertParseContext {
   SParseContext* pComCxt;       // input
   char          *pSql;          // input
   SMsgBuf        msg;           // input
-  char           dbFName[TSDB_DB_FNAME_LEN];
-  char           tableName[TSDB_TABLE_NAME_LEN];
   STableMeta* pTableMeta;       // each table
   SParsedDataColInfo tags;      // each table
   SKVRowBuilder tagsBuilder;    // each table
@@ -231,9 +229,6 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
   SVgroupInfo vg;
   CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg));
   CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-  pCxt->pTableMeta->vgId = vg.vgId; // todo remove
-  strcpy(pCxt->tableName, name.tname);
-  tNameGetFullDbName(&name, pCxt->dbFName);
   
   return TSDB_CODE_SUCCESS;
 }
@@ -977,8 +972,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     STableDataBlocks *dataBuf = NULL;
     CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE,
         sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL));
-    strcpy(dataBuf->tableName, pCxt->tableName);
-    strcpy(dataBuf->dbFName, pCxt->dbFName);
     
     if (TK_NK_LP == sToken.type) {
       // pSql -> field1_name, ...)

source/libs/parser/src/parTokenizer.c

@@ -29,10 +29,14 @@ typedef struct SKeyword {
 // keywords in sql string
 static SKeyword keywordTable[] = {
     {"ACCOUNT",       TK_ACCOUNT},
+    {"ACCOUNTS",      TK_ACCOUNTS},
+    {"ADD",           TK_ADD},
+    {"AGGREGATE",     TK_AGGREGATE},
     {"ALL",           TK_ALL},
     {"ALTER",         TK_ALTER},
     {"ANALYZE",       TK_ANALYZE},
     {"AND",           TK_AND},
+    {"APPS",          TK_APPS},
     {"AS",            TK_AS},
     {"ASC",           TK_ASC},
     {"BETWEEN",       TK_BETWEEN},
@@ -40,15 +44,22 @@ static SKeyword keywordTable[] = {
     {"BIGINT",        TK_BIGINT},
     {"BLOCKS",        TK_BLOCKS},
     {"BOOL",          TK_BOOL},
+    {"BUFSIZE",       TK_BUFSIZE},
     {"BY",            TK_BY},
     {"CACHE",         TK_CACHE},
     {"CACHELAST",     TK_CACHELAST},
+    {"COLUMN",        TK_COLUMN},
     {"COMMENT",       TK_COMMENT},
     {"COMP",          TK_COMP},
+    {"COMPACT",       TK_COMPACT},
+    {"CONNS",         TK_CONNS},
+    {"CONNECTION",    TK_CONNECTION},
+    {"CONNECTIONS",   TK_CONNECTIONS},
     {"CREATE",        TK_CREATE},
     {"DATABASE",      TK_DATABASE},
     {"DATABASES",     TK_DATABASES},
     {"DAYS",          TK_DAYS},
+    {"DBS",           TK_DBS},
     {"DELAY",         TK_DELAY},
     {"DESC",          TK_DESC},
     {"DESCRIBE",      TK_DESCRIBE},
@@ -83,14 +94,18 @@ static SKeyword keywordTable[] = {
     {"JOIN",          TK_JOIN},
     {"JSON",          TK_JSON},
     {"KEEP",          TK_KEEP},
+    {"KILL",          TK_KILL},
+    {"LICENCE",       TK_LICENCE},
     {"LIKE",          TK_LIKE},
     {"LIMIT",         TK_LIMIT},
     {"LINEAR",        TK_LINEAR},
+    {"LOCAL",         TK_LOCAL},
     {"MATCH",         TK_MATCH},
     {"MAXROWS",       TK_MAXROWS},
     {"MINROWS",       TK_MINROWS},
     {"MINUS",         TK_MINUS},
     {"MNODES",        TK_MNODES},
+    {"MODIFY",        TK_MODIFY},
     {"MODULES",       TK_MODULES},
     {"NCHAR",         TK_NCHAR},
     {"NMATCH",        TK_NMATCH},
@@ -102,9 +117,11 @@ static SKeyword keywordTable[] = {
     {"ON",            TK_ON},
     {"OR",            TK_OR},
     {"ORDER",         TK_ORDER},
+    {"OUTPUTTYPE",    TK_OUTPUTTYPE},
     {"PARTITION",     TK_PARTITION},
     {"PASS",          TK_PASS},
     {"PORT",          TK_PORT},
+    {"PPS",           TK_PPS},
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
@@ -112,6 +129,8 @@ static SKeyword keywordTable[] = {
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
     {"QSTARTTS",      TK_QSTARTTS},
+    {"QTIME",         TK_QTIME},
+    {"QUERIES",       TK_QUERIES},
     {"QUERY",         TK_QUERY},
     {"QUORUM",        TK_QUORUM},
     {"RATIO",         TK_RATIO},
@@ -120,8 +139,10 @@ static SKeyword keywordTable[] = {
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
     {"ROWTS",         TK_ROWTS},
+    {"SCORES",        TK_SCORES},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
+    {"SET",           TK_SET},
     {"SHOW",          TK_SHOW},
     {"SINGLE_STABLE", TK_SINGLE_STABLE},
     {"SLIDING",       TK_SLIDING},
@@ -131,16 +152,23 @@ static SKeyword keywordTable[] = {
     {"SOFFSET",       TK_SOFFSET},
     {"STABLE",        TK_STABLE},
     {"STABLES",       TK_STABLES},
+    {"STATE",         TK_STATE},
     {"STATE_WINDOW",  TK_STATE_WINDOW},
+    {"STORAGE",       TK_STORAGE},
+    {"STREAM",       TK_STREAM},
     {"STREAMS",       TK_STREAMS},
     {"STREAM_MODE",   TK_STREAM_MODE},
+    {"SYNCDB",        TK_SYNCDB},
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
+    {"TAG",           TK_TAG},
     {"TAGS",          TK_TAGS},
     {"TBNAME",        TK_TBNAME},
     {"TIMESTAMP",     TK_TIMESTAMP},
     {"TINYINT",       TK_TINYINT},
     {"TOPIC",         TK_TOPIC},
+    {"TOPICS",        TK_TOPICS},
+    {"TSERIES",       TK_TSERIES},
     {"TTL",           TK_TTL},
     {"UNION",         TK_UNION},
     {"UNSIGNED",      TK_UNSIGNED},
@@ -150,8 +178,10 @@ static SKeyword keywordTable[] = {
     {"USING",         TK_USING},
     {"VALUES",        TK_VALUES},
     {"VARCHAR",       TK_VARCHAR},
+    {"VARIABLES",     TK_VARIABLES},
     {"VERBOSE",       TK_VERBOSE},
     {"VGROUPS",       TK_VGROUPS},
+    {"VNODES",        TK_VNODES},
     {"WAL",           TK_WAL},
     {"WDURATION",     TK_WDURATION},
     {"WENDTS",        TK_WENDTS},
@@ -182,22 +212,8 @@ static SKeyword keywordTable[] = {
     // {"UMINUS",       TK_UMINUS},
     // {"UPLUS",        TK_UPLUS},
     // {"BITNOT",       TK_BITNOT},
-    // {"ACCOUNTS",     TK_ACCOUNTS},
-    // {"QUERIES",      TK_QUERIES},
-    // {"CONNECTIONS",  TK_CONNECTIONS},
-    // {"VARIABLES",    TK_VARIABLES},
-    // {"SCORES",       TK_SCORES},
     // {"GRANTS",       TK_GRANTS},
     // {"DOT",          TK_DOT},
-    // {"SYNCDB",       TK_SYNCDB},
-    // {"LOCAL",        TK_LOCAL},
-    // {"PPS",          TK_PPS},
-    // {"TSERIES",      TK_TSERIES},
-    // {"DBS",          TK_DBS},
-    // {"STORAGE",      TK_STORAGE},
-    // {"QTIME",        TK_QTIME},
-    // {"CONNS",        TK_CONNS},
-    // {"STATE",        TK_STATE},
     // {"CTIME",        TK_CTIME},
     // {"LP",           TK_LP},
     // {"RP",           TK_RP},
@@ -205,15 +221,8 @@ static SKeyword keywordTable[] = {
     // {"EVERY",        TK_EVERY},
     // {"VARIABLE",     TK_VARIABLE},
     // {"UPDATE",       TK_UPDATE},
-    // {"ADD",          TK_ADD},
-    // {"COLUMN",       TK_COLUMN},
-    // {"TAG",          TK_TAG},
     // {"CHANGE",       TK_CHANGE},
-    // {"SET",          TK_SET},
-    // {"KILL",         TK_KILL},
-    // {"CONNECTION",   TK_CONNECTION},
     // {"COLON",        TK_COLON},
-    // {"STREAM",       TK_STREAM},
     // {"ABORT",        TK_ABORT},
     // {"AFTER",        TK_AFTER},
     // {"ATTACH",       TK_ATTACH},
@@ -244,14 +253,7 @@ static SKeyword keywordTable[] = {
     // {"TRIGGER",      TK_TRIGGER},
     // {"VIEW",         TK_VIEW},
     // {"SEMI",         TK_SEMI},
-    // {"VNODES",       TK_VNODES},
+    // {"PARTITIONS",   TK_PARTITIONS},
-//    {"PARTITIONS",   TK_PARTITIONS},
-    // {"TOPICS",       TK_TOPICS},
-    // {"COMPACT",      TK_COMPACT},
-    // {"MODIFY",       TK_MODIFY},
-    // {"OUTPUTTYPE",   TK_OUTPUTTYPE},
-    // {"AGGREGATE",    TK_AGGREGATE},
-    // {"BUFSIZE",      TK_BUFSIZE},
     // {"MODE",         TK_MODE},
 };
 

source/libs/parser/src/parTranslater.c

@@ -240,7 +240,11 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   if (NULL != pTable) {
     strcpy(pCol->tableAlias, pTable->tableAlias);
   } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
-    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
+    SColumnNode* pProjCol = (SColumnNode*)pExpr;
+    strcpy(pCol->tableAlias, pProjCol->tableAlias);
+    pCol->tableId = pProjCol->tableId;
+    pCol->colId = pProjCol->colId;
+    pCol->colType = pProjCol->colType;
   }
   strcpy(pCol->colName, pExpr->aliasName);
   pCol->node.resType = pExpr->resType;
@@ -956,9 +960,9 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS
   pReq->cacheBlockSize = pStmt->pOptions->cacheBlockSize;
   pReq->totalBlocks = pStmt->pOptions->numOfBlocks;
   pReq->daysPerFile = pStmt->pOptions->daysPerFile;
-  pReq->daysToKeep0 = pStmt->pOptions->keep;
+  pReq->daysToKeep0 = pStmt->pOptions->keep0;
-  pReq->daysToKeep1 = -1;
+  pReq->daysToKeep1 = pStmt->pOptions->keep1;
-  pReq->daysToKeep2 = -1;
+  pReq->daysToKeep2 = pStmt->pOptions->keep2;
   pReq->minRows = pStmt->pOptions->minRowsPerBlock;
   pReq->maxRows = pStmt->pOptions->maxRowsPerBlock;
   pReq->commitTime = -1;
@@ -1041,13 +1045,14 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt,
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   tNameGetFullDbName(&name, pReq->db);
   pReq->totalBlocks = pStmt->pOptions->numOfBlocks;
-  pReq->daysToKeep0 = pStmt->pOptions->keep;
+  pReq->daysToKeep0 = pStmt->pOptions->keep0;
-  pReq->daysToKeep1 = -1;
+  pReq->daysToKeep1 = pStmt->pOptions->keep1;
-  pReq->daysToKeep2 = -1;
+  pReq->daysToKeep2 = pStmt->pOptions->keep2;
   pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod;
   pReq->walLevel = pStmt->pOptions->walLevel;
   pReq->quorum = pStmt->pOptions->quorum;
   pReq->cacheLastRow = pStmt->pOptions->cachelast;
+  pReq->replications = pStmt->pOptions->replica;
   return;
 }
 
@@ -1094,10 +1099,6 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
 }
 
 static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
-  if (NULL == pList) {
-    return TSDB_CODE_SUCCESS;
-  }
-
   *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
   SNode* pNode;
   FOREACH(pNode, pList) {
@@ -1119,7 +1120,7 @@ static const SColumnDefNode* findColDef(const SNodeList* pCols, const SColumnNod
   return NULL;
 }
 
-static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+static int32_t checkCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
   if (NULL != pStmt->pOptions->pSma) {
     SNode* pNode = NULL;
     FOREACH(pNode, pStmt->pOptions->pSma) {
@@ -1148,7 +1149,7 @@ static int32_t getAggregationMethod(SNodeList* pFuncs) {
 }
 
 static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
-  int32_t code = checkCreateTable(pCxt, pStmt);
+  int32_t code = checkCreateSuperTable(pCxt, pStmt);
   if (TSDB_CODE_SUCCESS != code) {
     return code;
   }
@@ -1160,10 +1161,15 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt
   createReq.delay = pStmt->pOptions->delay;
   columnDefNodeToField(pStmt->pCols, &createReq.pColumns);
   columnDefNodeToField(pStmt->pTags, &createReq.pTags);
-  columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
   createReq.numOfColumns = LIST_LENGTH(pStmt->pCols);
   createReq.numOfTags = LIST_LENGTH(pStmt->pTags);
-  createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  if (NULL == pStmt->pOptions->pSma) {
+    columnDefNodeToField(pStmt->pCols, &createReq.pSmas);
+    createReq.numOfSmas = createReq.numOfColumns;
+  } else {
+    columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
+    createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
+  }
 
   SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
   strcpy(tableName.dbname, pStmt->dbName);
@@ -1469,20 +1475,20 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
 
 static int32_t nodeTypeToShowType(ENodeType nt) {
   switch (nt) {
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-      return TSDB_MGMT_TABLE_DB;
+      return 0; // todo
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-      return TSDB_MGMT_TABLE_STB;
+      return TSDB_MGMT_TABLE_CONNS;
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-      return TSDB_MGMT_TABLE_USER;
+      return 0; // todo
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-      return TSDB_MGMT_TABLE_DNODE;
+      return TSDB_MGMT_TABLE_QUERIES;
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-      return TSDB_MGMT_TABLE_VGROUP;
+      return 0; // todo
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-      return TSDB_MGMT_TABLE_MNODE;
+      return 0; // todo
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
-      return TSDB_MGMT_TABLE_QNODE;
+      return TSDB_MGMT_TABLE_VARIABLES;
     default:
       break;
   }
@@ -1508,30 +1514,6 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateShowTables(STranslateContext* pCxt) {
-  SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq));
-
-  SArray* array = NULL;
-  int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array);
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
-  }
-  SVgroupInfo* info = taosArrayGet(array, 0);
-  pShowReq->head.vgId = htonl(info->vgId);
-
-  pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
-  if (NULL == pCxt->pCmdMsg) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  pCxt->pCmdMsg->epSet = info->epSet;
-  pCxt->pCmdMsg->msgType = TDMT_VND_SHOW_TABLES;
-  pCxt->pCmdMsg->msgLen = sizeof(SVShowTablesReq);
-  pCxt->pCmdMsg->pMsg = pShowReq;
-  pCxt->pCmdMsg->pExtension = array;
-
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) {
   SVgroupInfo vg = {0};
   int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg);
@@ -1867,17 +1849,19 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_ALTER_DNODE_STMT:
       code = translateAlterDnode(pCxt, (SAlterDnodeStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_DATABASES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
-    case QUERY_NODE_SHOW_STABLES_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-    case QUERY_NODE_SHOW_USERS_STMT:
+    case QUERY_NODE_SHOW_LICENCE_STMT:
-    case QUERY_NODE_SHOW_DNODES_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
-    case QUERY_NODE_SHOW_MNODES_STMT:
+    case QUERY_NODE_SHOW_TOPICS_STMT:
-    case QUERY_NODE_SHOW_QNODES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
       code = translateShow(pCxt, (SShowStmt*)pNode);
       break;
-    case QUERY_NODE_SHOW_TABLES_STMT:
+    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
-      code = translateShowTables(pCxt);
+    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
+      // todo
       break;
     case QUERY_NODE_CREATE_INDEX_STMT:
       code = translateCreateIndex(pCxt, (SCreateIndexStmt*)pNode);
@@ -1952,6 +1936,7 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema)
   }
   (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
   (*pSchema)[0].bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+  strcpy((*pSchema)[0].name, TSDB_EXPLAIN_RESULT_COLUMN_NAME);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2447,9 +2432,19 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) {
+  if (0 != strcmp(pStmt->dbName, pStmt->useDbName)) {
+    return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR);;
+  }
+  return TSDB_CODE_SUCCESS;
+}
 static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt, SHashObj* pVgroupHashmap) {
+  int32_t code = checkCreateSubTable(pCxt, pStmt);
+
   STableMeta* pSuperTableMeta = NULL;
-  int32_t code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
+  }
 
   SKVRowBuilder kvRowBuilder = {0};
   if (TSDB_CODE_SUCCESS == code) {

source/libs/parser/src/parUtil.c

@@ -61,6 +61,12 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "This statement is no longer supported";
     case TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL:
       return "This interval value is too small : %s";
+    case TSDB_CODE_PAR_DB_NOT_SPECIFIED:
+      return "db not specified";
+    case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME:
+      return "Invalid identifier name : %s";
+    case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR:
+      return "corresponding super table not in this db";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/planner/src/planLogicCreater.c

@@ -288,7 +288,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   // set the output
   if (TSDB_CODE_SUCCESS == code) {
     pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
-    if (NULL == pJoin->pOnConditions) {
+    if (NULL == pJoin->node.pTargets) {
       code = TSDB_CODE_OUT_OF_MEMORY;
     }
     if (TSDB_CODE_SUCCESS == code) {

source/libs/qcom/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
     qcom
-    PRIVATE os util transport
+    PRIVATE os util transport nodes
 )
 
 if(${BUILD_TEST})

source/libs/qcom/inc/queryInt.h

@@ -21,7 +21,6 @@ extern "C" {
 #endif
 
 
-
 #ifdef __cplusplus
 }
 #endif

source/libs/scalar/inc/filterInt.h

@@ -36,8 +36,6 @@ extern "C" {
 
 #define FILTER_DUMMY_EMPTY_OPTR  127
 
-#define MAX_NUM_STR_SIZE 40
-
 #define FILTER_RM_UNIT_MIN_ROWS 100
 
 enum {

source/libs/scalar/src/sclfunc.c

@@ -23,7 +23,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -36,7 +36,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -49,7 +49,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -62,7 +62,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -75,7 +75,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }
@@ -88,7 +88,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
           colDataSetNull_f(pOutputData->nullbitmap, i);
           continue;
         }
-        out[i] = (in[i] > 0)? in[i] : -in[i];
+        out[i] = (in[i] >= 0)? in[i] : -in[i];
       }
       break;
     }

source/libs/scalar/src/sclvector.c

@@ -720,7 +720,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
 
       double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
       double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
-      if (compareDoubleVal(&zero, &rx)) {
+      if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
         colDataAppend(pOutputCol, i, NULL, true);
         continue;
       }
@@ -729,7 +729,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
     }
   } else if (pLeft->numOfRows == 1) {
     double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0);
-    if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+    if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
       // TODO set numOfRows NULL value
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
@@ -739,7 +739,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
         }
 
         double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
-        if (compareDoubleVal(&zero, &rx)) {
+        if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
           colDataAppend(pOutputCol, i, NULL, true);
           continue;
         }
@@ -749,17 +749,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
     }
   } else if (pRight->numOfRows == 1) {
     double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0);
-    if (colDataIsNull_f(pRightCol->nullbitmap, 0)) {  // Set pLeft->numOfRows NULL value
+    if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
       // TODO set numOfRows NULL value
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        if (colDataIsNull_f(pRightCol->nullbitmap, i)) {
+        if (colDataIsNull_f(pLeftCol->nullbitmap, i)) {
           colDataAppend(pOutputCol, i, NULL, true);
           continue;
         }
 
-        double lx = getVectorDoubleValueFnLeft(pRightCol->pData, i);
+        double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
-        if (compareDoubleVal(&zero, &lx)) {
+        if (isnan(lx) || isinf(lx)) {
           colDataAppend(pOutputCol, i, NULL, true);
           continue;
         }

source/libs/scheduler/CMakeLists.txt

@@ -9,7 +9,7 @@ target_include_directories(
 
 target_link_libraries(
         scheduler
-        PUBLIC os util nodes planner qcom common catalog transport
+        PUBLIC os util nodes planner qcom common catalog transport command
 )
 
 if(${BUILD_TEST})

source/libs/scheduler/inc/schedulerInt.h

@@ -142,10 +142,10 @@ typedef struct SSchTask {
 } SSchTask;
 
 typedef struct SSchJobAttr {
-  bool needFetch;
+  EExplainMode explainMode;
-  bool syncSchedule;
+  bool         syncSchedule;
-  bool queryJob;
+  bool         queryJob;
-  bool needFlowCtrl;
+  bool         needFlowCtrl;
 } SSchJobAttr;
 
 typedef struct SSchJob {

source/libs/scheduler/src/scheduler.c

@@ -19,6 +19,7 @@
 #include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
+#include "command.h"
 
 SSchedulerMgmt schMgmt = {0};
 
@@ -2103,6 +2104,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
     SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  pJob->attr.explainMode = pDag->explainInfo.mode;
   pJob->attr.syncSchedule = syncSchedule;
   pJob->transport = transport;
   pJob->sql = sql;
@@ -2136,19 +2138,24 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD
 
   tsem_init(&pJob->rspSem, 0, 0);
 
-  pJob->refId = taosAddRef(schMgmt.jobRef, pJob);
+  int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
-  if (pJob->refId < 0) {
+  if (refId < 0) {
-    SCH_JOB_ELOG("taosHashPut job failed, error:%s", tstrerror(terrno));
+    SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
     SCH_ERR_JRET(terrno);
   }
 
+  if (NULL == schAcquireJob(refId)) {
+    SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
+    SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
+  }
+
+  pJob->refId = refId;
+
   SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
 
   pJob->status = JOB_TASK_STATUS_NOT_START;
   SCH_ERR_JRET(schLaunchJob(pJob));
 
-  schAcquireJob(pJob->refId);
-
   *job = pJob->refId;
 
   if (syncSchedule) {
@@ -2168,6 +2175,54 @@ _return:
   SCH_RET(code);
 }
 
+int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql,
+                              bool syncSchedule) {
+  qDebug("QID:0x%" PRIx64 " job started", pDag->queryId);
+
+  int32_t  code = 0;
+  SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
+  if (NULL == pJob) {
+    qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob));
+    SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  pJob->sql = sql;
+  pJob->attr.queryJob = true;
+  pJob->attr.explainMode = pDag->explainInfo.mode;
+  pJob->queryId = pDag->queryId;
+  pJob->subPlans = pDag->pSubplans;
+
+  SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData));
+
+  int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
+  if (refId < 0) {
+    SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
+    SCH_ERR_JRET(terrno);
+  }
+
+  if (NULL == schAcquireJob(refId)) {
+    SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
+    SCH_RET(TSDB_CODE_SCH_STATUS_ERROR);
+  }
+
+  pJob->refId = refId;
+
+  SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
+
+  pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
+  *job = pJob->refId;
+  SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
+
+  schReleaseJob(pJob->refId);
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  schFreeJobImpl(pJob);
+  SCH_RET(code);
+}
+
 int32_t schedulerInit(SSchedulerCfg *cfg) {
   if (schMgmt.jobRef) {
     qError("scheduler already initialized");
@@ -2216,7 +2271,11 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
-  SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
+  if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) {
+    SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true));
+  } else {
+    SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true));
+  }
 
   SSchJob *job = schAcquireJob(*pJob);
 
@@ -2399,11 +2458,15 @@ int32_t schedulerFetchRows(int64_t job, void **pData) {
     SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status));
     goto _return;
   } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
-    SCH_ERR_JRET(schFetchFromRemote(pJob));
+    if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) {
+      SCH_ERR_JRET(schFetchFromRemote(pJob));
+      tsem_wait(&pJob->rspSem);
+    } 
+  } else {
+    SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status));
+    SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
   }
 
-  tsem_wait(&pJob->rspSem);
-
   status = SCH_GET_JOB_STATUS(pJob);
 
   if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) {

source/libs/tdb/src/db/tdbBtree.c

@@ -29,15 +29,15 @@ struct SBTree {
   int            minLocal;
   int            maxLeaf;
   int            minLeaf;
-  u8            *pTmp;
+  void          *pBuf;
 };
 
 #define TDB_BTREE_PAGE_COMMON_HDR u8 flags;
 
-#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0]
+#define TDB_BTREE_PAGE_GET_FLAGS(PAGE)        (PAGE)->pData[0]
 #define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags))
-#define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
+#define TDB_BTREE_PAGE_IS_ROOT(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
-#define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
+#define TDB_BTREE_PAGE_IS_LEAF(PAGE)          (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
 #define TDB_BTREE_ASSERT_FLAG(flags)                                                 \
   ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
          TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0))
@@ -101,7 +101,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
   // pBt->kcmpr
   pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn;
   // pBt->pageSize
-  pBt->pageSize = tdbPagerGetPageSize(pPager);
+  pBt->pageSize = pPager->pageSize;
   // pBt->maxLocal
   pBt->maxLocal = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr)) / 4;
   // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData]
@@ -127,132 +127,144 @@ int tdbBtreeClose(SBTree *pBt) {
   return 0;
 }
 
-int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen) {
-  int     ret;
+  SBTC   btc;
-  int     idx;
+  SCell *pCell;
-  SPager *pPager;
+  void  *pBuf;
-  SCell  *pCell;
+  int    szCell;
-  int     szCell;
+  int    szBuf;
-  int     cret;
+  int    ret;
-  SBTree *pBt;
+  int    idx;
+  int    c;
 
-  ret = tdbBtcMoveTo(pBtc, pKey, kLen, &cret);
+  tdbBtcOpen(&btc, pBt);
+
+  // move to the position to insert
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
   if (ret < 0) {
-    // TODO: handle error
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  if (pBtc->idx == -1) {
+  if (btc.idx == -1) {
-    ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
     idx = 0;
   } else {
-    if (cret > 0) {
+    if (c > 0) {
-      idx = pBtc->idx + 1;
+      idx = btc.idx + 1;
-    } else if (cret < 0) {
+    } else if (c < 0) {
-      idx = pBtc->idx;
+      idx = btc.idx;
     } else {
-      /* TODO */
+      // TDB does NOT allow same key
+      tdbBtcClose(&btc);
       ASSERT(0);
-    }
-  }
-
-  // TODO: refact code here
-  pBt = pBtc->pBt;
-  if (!pBt->pTmp) {
-    pBt->pTmp = (u8 *)tdbOsMalloc(pBt->pageSize);
-    if (pBt->pTmp == NULL) {
       return -1;
     }
   }
 
-  pCell = pBt->pTmp;
+  // make sure enough space to hold the cell
+  szBuf = kLen + vLen + 14;
+  pBuf = TDB_REALLOC(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize);
+  if (pBuf == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+  pBt->pBuf = pBuf;
+  pCell = (SCell *)pBt->pBuf;
 
-  // Encode the cell
+  // encode cell
-  ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
+  ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // Insert the cell to the index
+  // mark the page dirty
-  ret = tdbPageInsertCell(pBtc->pPage, idx, pCell, szCell, 0);
+  ret = tdbPagerWrite(pBt->pPager, btc.pPage);
   if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
     return -1;
   }
 
-  // If page is overflow, balance the tree
+  // insert the cell
-  if (pBtc->pPage->nOverflow > 0) {
+  ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0);
-    ret = tdbBtreeBalance(pBtc);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
+
+  // check if need balance
+  if (btc.pPage->nOverflow > 0) {
+    ret = tdbBtreeBalance(&btc);
     if (ret < 0) {
+      tdbBtcClose(&btc);
+      ASSERT(0);
       return -1;
     }
   }
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) {
-  SBTC         btc;
+  return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen);
-  SCell       *pCell;
-  int          cret;
-  void        *pVal;
-  SCellDecoder cd;
-
-  tdbBtcOpen(&btc, pBt);
-
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
-
-  if (cret) {
-    return cret;
-  }
-
-  pCell = tdbPageGetCell(btc.pPage, btc.idx);
-  tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
-
-  *vLen = cd.vLen;
-  pVal = TDB_REALLOC(*ppVal, *vLen);
-  if (pVal == NULL) {
-    return -1;
-  }
-
-  *ppVal = pVal;
-  memcpy(*ppVal, cd.pVal, cd.vLen);
-  return 0;
 }
 
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
   SBTC         btc;
   SCell       *pCell;
   int          cret;
-  void        *pTKey;
+  int          ret;
-  void        *pTVal;
+  void        *pTKey = NULL;
+  void        *pTVal = NULL;
   SCellDecoder cd;
 
   tdbBtcOpen(&btc, pBt);
 
-  tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret);
+  if (ret < 0) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+  }
+
   if (cret) {
-    return cret;
+    tdbBtcClose(&btc);
+    return -1;
   }
 
   pCell = tdbPageGetCell(btc.pPage, btc.idx);
   tdbBtreeDecodeCell(btc.pPage, pCell, &cd);
 
-  pTKey = TDB_REALLOC(*ppKey, cd.kLen);
+  if (ppKey) {
-  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+    pTKey = TDB_REALLOC(*ppKey, cd.kLen);
-
+    if (pTKey == NULL) {
-  if (pTKey == NULL || pTVal == NULL) {
+      tdbBtcClose(&btc);
-    TDB_FREE(pTKey);
+      ASSERT(0);
-    TDB_FREE(pTVal);
+      return -1;
+    }
+    *ppKey = pTKey;
+    *pkLen = cd.kLen;
+    memcpy(*ppKey, cd.pKey, cd.kLen);
   }
 
-  *ppKey = pTKey;
+  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+  if (pTVal == NULL) {
+    tdbBtcClose(&btc);
+    ASSERT(0);
+    return -1;
+  }
   *ppVal = pTVal;
-  *pkLen = cd.kLen;
   *vLen = cd.vLen;
-
-  memcpy(*ppKey, cd.pKey, cd.kLen);
   memcpy(*ppVal, cd.pVal, cd.vLen);
 
+  tdbBtcClose(&btc);
+
   return 0;
 }
 
@@ -300,7 +312,8 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
     return -1;
   }
 
-  // TODO: Unref the page
+  // TODO: here still has problem
+  tdbPagerReturnPage(pBt->pPager, pPage);
 
   ASSERT(pgno != 0);
   pBt->root = pgno;
@@ -371,17 +384,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
   return 0;
 }
 
-#ifndef TDB_BTREE_BALANCE
+// TDB_BTREE_BALANCE =====================
-typedef struct {
-  SBTree *pBt;
-  SPage  *pParent;
-  int     idx;
-  i8      nOld;
-  SPage  *pOldPages[3];
-  i8      nNewPages;
-  SPage  *pNewPages[5];
-} SBtreeBalanceHelper;
-
 static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
   SPager           *pPager;
   SPage            *pChild;
@@ -408,6 +411,13 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
     ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno;
   }
 
+  ret = tdbPagerWrite(pPager, pChild);
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
+  }
+
   // Copy the root page content to the child page
   tdbPageCopy(pRoot, pChild);
 
@@ -472,6 +482,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         ASSERT(0);
         return -1;
       }
+
+      ret = tdbPagerWrite(pBt->pPager, pOlds[i]);
+      if (ret < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
     }
     // copy the parent key out if child pages are not leaf page
     childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]);
@@ -492,6 +509,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
       }
       rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno;
     }
+
+    ret = tdbPagerWrite(pBt->pPager, pParent);
+    if (ret < 0) {
+      // TODO
+      ASSERT(0);
+      return -1;
+    }
+
     // drop the cells on parent page
     for (int i = 0; i < nOlds; i++) {
       nCells = TDB_PAGE_TOTAL_CELLS(pParent);
@@ -619,6 +644,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         if (ret < 0) {
           ASSERT(0);
         }
+
+        ret = tdbPagerWrite(pBt->pPager, pNews[iNew]);
+        if (ret < 0) {
+          // TODO
+          ASSERT(0);
+          return -1;
+        }
       }
     }
 
@@ -732,14 +764,24 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
     }
   }
 
+  // TODO: here is not corrent for drop case
+  for (int i = 0; i < nNews; i++) {
+    if (i < nOlds) {
+      tdbPagerReturnPage(pBt->pPager, pOlds[i]);
+    } else {
+      tdbPagerReturnPage(pBt->pPager, pNews[i]);
+    }
+  }
+
   return 0;
 }
 
 static int tdbBtreeBalance(SBTC *pBtc) {
   int    iPage;
+  int    ret;
+  int    nFree;
   SPage *pParent;
   SPage *pPage;
-  int    ret;
   u8     flags;
   u8     leaf;
   u8     root;
@@ -750,10 +792,11 @@ static int tdbBtreeBalance(SBTC *pBtc) {
     pPage = pBtc->pPage;
     leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
     root = TDB_BTREE_PAGE_IS_ROOT(pPage);
+    nFree = TDB_PAGE_FREE_SIZE(pPage);
 
     // when the page is not overflow and not too empty, the balance work
     // is finished. Just break out the balance loop.
-    if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) {
+    if (pPage->nOverflow == 0 && nFree < TDB_PAGE_USABLE_SIZE(pPage) * 2 / 3) {
       break;
     }
 
@@ -781,6 +824,8 @@ static int tdbBtreeBalance(SBTC *pBtc) {
         return -1;
       }
 
+      tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+
       pBtc->iPage--;
       pBtc->pPage = pBtc->pgStack[pBtc->iPage];
     }
@@ -788,7 +833,7 @@ static int tdbBtreeBalance(SBTC *pBtc) {
 
   return 0;
 }
-#endif
+// TDB_BTREE_BALANCE
 
 // TDB_BTREE_CELL =====================
 static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const void *pKey, int kLen, const void *pVal,
@@ -1028,12 +1073,11 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
     // move upward
     for (;;) {
-      if (pBtc->iPage == 0) {
+      if (pBtc->iPage == iPage) {
         pBtc->idx = 0;
         break;
       }
 
-      if (pBtc->iPage < iPage) break;
       tdbBtcMoveUpward(pBtc);
     }
   }
@@ -1056,6 +1100,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
 int tdbBtcMoveToLast(SBTC *pBtc) {
   int     ret;
+  int     nCells;
   SBTree *pBt;
   SPager *pPager;
   SPgno   pgno;
@@ -1071,27 +1116,56 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
       return -1;
     }
 
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
     pBtc->iPage = 0;
+    if (nCells > 0) {
+      pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells;
+    } else {
+      // no data at all, point to an invalid position
+      ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
+      pBtc->idx = -1;
+      return 0;
+    }
   } else {
-    // move from a position
+    int iPage = 0;
-    ASSERT(0);
+
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage]));
+      nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]);
+      if (pBtc->idxStack[iPage] != nCells) break;
+    }
+
+    // move upward
+    for (;;) {
+      if (pBtc->iPage == iPage) {
+        if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
+        } else {
+          pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+        }
+        break;
+      }
+
+      tdbBtcMoveUpward(pBtc);
+    }
   }
 
   // move downward
   for (;;) {
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
-      // TODO: handle empty case
-      ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0);
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1;
-      break;
-    } else {
-      pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-      ret = tdbBtcMoveDownward(pBtc);
+    ret = tdbBtcMoveDownward(pBtc);
-      if (ret < 0) {
+    if (ret < 0) {
-        ASSERT(0);
+      ASSERT(0);
-        return -1;
+      return -1;
-      }
+    }
+
+    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
+      pBtc->idx = nCells - 1;
+    } else {
+      pBtc->idx = nCells;
     }
   }
 
@@ -1104,6 +1178,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   void        *pKey, *pVal;
   int          ret;
 
+  // current cursor points to an invalid position
   if (pBtc->idx < 0) {
     return -1;
   }
@@ -1134,12 +1209,17 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   memcpy(pVal, cd.pVal, cd.vLen);
 
   ret = tdbBtcMoveToNext(pBtc);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
   return 0;
 }
 
 static int tdbBtcMoveToNext(SBTC *pBtc) {
   int    nCells;
+  int    ret;
   SCell *pCell;
 
   ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
@@ -1151,39 +1231,35 @@ static int tdbBtcMoveToNext(SBTC *pBtc) {
     return 0;
   }
 
-  if (pBtc->iPage == 0) {
+  // move upward
-    pBtc->idx = -1;
-    return 0;
-  }
-
-  // Move upward
   for (;;) {
-    tdbBtcMoveUpward(pBtc);
-    pBtc->idx++;
-
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
-    if (pBtc->idx <= nCells) {
-      break;
-    }
-
     if (pBtc->iPage == 0) {
       pBtc->idx = -1;
       return 0;
     }
-  }
 
-  // Move downward
+    tdbBtcMoveUpward(pBtc);
-  for (;;) {
+    pBtc->idx++;
-    nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage);
 
-    tdbBtcMoveDownward(pBtc);
+    ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage));
-    pBtc->idx = 0;
+    if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) {
-
-    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) {
       break;
     }
   }
 
+  // move downward
+  for (;;) {
+    if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break;
+
+    ret = tdbBtcMoveDownward(pBtc);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    pBtc->idx = 0;
+  }
+
   return 0;
 }
 
@@ -1230,91 +1306,145 @@ static int tdbBtcMoveUpward(SBTC *pBtc) {
 }
 
 static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
-  int     ret;
+  int          ret;
-  SBTree *pBt;
+  int          nCells;
-  SPager *pPager;
+  int          c;
+  SBTree      *pBt;
+  SCell       *pCell;
+  SPager      *pPager;
+  SCellDecoder cd = {0};
 
   pBt = pBtc->pBt;
   pPager = pBt->pPager;
 
   if (pBtc->iPage < 0) {
-    ASSERT(pBtc->iPage == -1);
+    // move from a clear cursor
-    ASSERT(pBtc->idx == -1);
-
-    // Move from the root
     ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
     if (ret < 0) {
+      // TODO
       ASSERT(0);
-      return -1;
-    }
-
-    pBtc->iPage = 0;
-
-    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) {
-      // Current page is empty
-      // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF));
       return 0;
     }
 
-    for (;;) {
+    pBtc->iPage = 0;
-      int          lidx, ridx, midx, c, nCells;
+    pBtc->idx = -1;
-      SCell       *pCell;
+    // for empty tree, just return with an invalid position
-      SPage       *pPage;
+    if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0;
-      SCellDecoder cd = {0};
+  } else {
+    SPage *pPage;
+    int    idx;
+    int    iPage = 0;
 
-      pPage = pBtc->pPage;
+    // downward search
+    for (; iPage < pBtc->iPage; iPage++) {
+      pPage = pBtc->pgStack[iPage];
+      idx = pBtc->idxStack[iPage];
       nCells = TDB_PAGE_TOTAL_CELLS(pPage);
-      lidx = 0;
-      ridx = nCells - 1;
 
-      ASSERT(nCells > 0);
+      ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage));
 
-      for (;;) {
+      // check if key <= current position
-        if (lidx > ridx) break;
+      if (idx < nCells) {
-
+        pCell = tdbPageGetCell(pPage, idx);
-        midx = (lidx + ridx) >> 1;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-
-        pCell = tdbPageGetCell(pPage, midx);
-        ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
-        if (ret < 0) {
-          // TODO: handle error
-          ASSERT(0);
-          return -1;
-        }
-
-        // Compare the key values
         c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        if (c < 0) {
+        if (c > 0) break;
-          /* input-key < cell-key */
-          ridx = midx - 1;
-        } else if (c > 0) {
-          /* input-key > cell-key */
-          lidx = midx + 1;
-        } else {
-          /* input-key == cell-key */
-          break;
-        }
       }
 
-      // Move downward or break
+      // check if key > current - 1 position
-      u8 leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
+      if (idx > 0) {
-      if (leaf) {
+        pCell = tdbPageGetCell(pPage, idx - 1);
-        pBtc->idx = midx;
+        tdbBtreeDecodeCell(pPage, pCell, &cd);
-        *pCRst = c;
+        c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
-        break;
+        if (c <= 0) break;
-      } else {
-        if (c <= 0) {
-          pBtc->idx = midx;
-        } else {
-          pBtc->idx = midx + 1;
-        }
-        tdbBtcMoveDownward(pBtc);
       }
     }
 
-  } else {
+    // move upward
-    // TODO: Move the cursor from a some position instead of a clear state
+    for (;;) {
-    ASSERT(0);
+      if (pBtc->iPage == iPage) break;
+      tdbBtcMoveUpward(pBtc);
+    }
+  }
+
+  // search downward to the leaf
+  for (;;) {
+    int    lidx, ridx, midx;
+    SPage *pPage;
+
+    pPage = pBtc->pPage;
+    nCells = TDB_PAGE_TOTAL_CELLS(pPage);
+    lidx = 0;
+    ridx = nCells - 1;
+
+    ASSERT(nCells > 0);
+    ASSERT(pBtc->idx == -1);
+
+    // compare first cell
+    midx = lidx;
+    pCell = tdbPageGetCell(pPage, midx);
+    tdbBtreeDecodeCell(pPage, pCell, &cd);
+    c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+    if (c <= 0) {
+      ridx = lidx - 1;
+    } else {
+      lidx = lidx + 1;
+    }
+
+    // compare last cell
+    if (lidx <= ridx) {
+      midx = ridx;
+      pCell = tdbPageGetCell(pPage, midx);
+      tdbBtreeDecodeCell(pPage, pCell, &cd);
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c >= 0) {
+        lidx = ridx + 1;
+      } else {
+        ridx = ridx - 1;
+      }
+    }
+
+    // binary search
+    for (;;) {
+      if (lidx > ridx) break;
+
+      midx = (lidx + ridx) >> 1;
+
+      pCell = tdbPageGetCell(pPage, midx);
+      ret = tdbBtreeDecodeCell(pPage, pCell, &cd);
+      if (ret < 0) {
+        // TODO: handle error
+        ASSERT(0);
+        return -1;
+      }
+
+      // Compare the key values
+      c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen);
+      if (c < 0) {
+        // pKey < cd.pKey
+        ridx = midx - 1;
+      } else if (c > 0) {
+        // pKey > cd.pKey
+        lidx = midx + 1;
+      } else {
+        // pKey == cd.pKey
+        break;
+      }
+    }
+
+    // keep search downward or break
+    if (TDB_BTREE_PAGE_IS_LEAF(pPage)) {
+      pBtc->idx = midx;
+      *pCRst = c;
+      break;
+    } else {
+      if (c <= 0) {
+        pBtc->idx = midx;
+      } else {
+        pBtc->idx = midx + 1;
+      }
+      tdbBtcMoveDownward(pBtc);
+    }
   }
 
   return 0;

source/libs/tdb/src/db/tdbDb.c

@@ -49,6 +49,8 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
     if (ret < 0) {
       return -1;
     }
+
+    tdbEnvAddPager(pEnv, pPager);
   }
 
   ASSERT(pPager != NULL);
@@ -74,22 +76,7 @@ int tdbDbDrop(TDB *pDb) {
 }
 
 int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  SBTC  btc;
+  return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen);
-  SBTC *pCur;
-  int   ret;
-
-  pCur = &btc;
-  ret = tdbBtcOpen(pCur, pDb->pBt);
-  if (ret < 0) {
-    return -1;
-  }
-
-  ret = tdbBtCursorInsert(pCur, pKey, keyLen, pVal, valLen);
-  if (ret < 0) {
-    return -1;
-  }
-
-  return 0;
 }
 
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {

source/libs/tdb/src/db/tdbEnv.c

@@ -19,6 +19,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
   TENV *pEnv;
   int   dsize;
   int   zsize;
+  int   tsize;
   u8   *pPtr;
   int   ret;
 
@@ -53,6 +54,14 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
     return -1;
   }
 
+  pEnv->nPgrHash = 8;
+  tsize = sizeof(SPager *) * pEnv->nPgrHash;
+  pEnv->pgrHash = TDB_REALLOC(pEnv->pgrHash, tsize);
+  if (pEnv->pgrHash == NULL) {
+    return -1;
+  }
+  memset(pEnv->pgrHash, 0, tsize);
+
   mkdir(rootDir, 0755);
 
   *ppEnv = pEnv;
@@ -64,7 +73,99 @@ int tdbEnvClose(TENV *pEnv) {
   return 0;
 }
 
-SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
+int tdbBegin(TENV *pEnv) {
-  // TODO
+  SPager *pPager;
-  return NULL;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerBegin(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbCommit(TENV *pEnv) {
+  SPager *pPager;
+  int     ret;
+
+  for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
+    ret = tdbPagerCommit(pPager);
+    if (ret < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tdbRollback(TENV *pEnv) {
+  ASSERT(0);
+  return 0;
+}
+
+SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
+  u32      hash;
+  SPager **ppPager;
+
+  hash = tdbCstringHash(fname);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
+  }
+
+  return *ppPager;
+}
+
+void tdbEnvAddPager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // rehash if neccessary
+  if (pEnv->nPager + 1 > pEnv->nPgrHash) {
+    // TODO
+  }
+
+  // add to list
+  pPager->pNext = pEnv->pgrList;
+  pEnv->pgrList = pPager;
+
+  // add to hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  pPager->pHashNext = *ppPager;
+  *ppPager = pPager;
+
+  // increase the counter
+  pEnv->nPager++;
+}
+
+void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) {
+  u32      hash;
+  SPager **ppPager;
+
+  // remove from the list
+  for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // remove from hash
+  hash = tdbCstringHash(pPager->dbFileName);
+  ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
+  for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
+  }
+  ASSERT(*ppPager == pPager);
+  *ppPager = pPager->pNext;
+
+  // decrease the counter
+  pEnv->nPager--;
+
+  // rehash if necessary
+  if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) {
+    // TODO
+  }
 }

source/libs/tdb/src/db/tdbPCache.c

@@ -34,18 +34,6 @@ struct SPCache {
   })
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
-// For page ref
-#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
-#if 0
-#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
-#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
-#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
-#else
-#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
-#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
-#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
-#endif
-
 static int    tdbPCacheOpenImpl(SPCache *pCache);
 static void   tdbPCacheInitLock(SPCache *pCache);
 static void   tdbPCacheClearLock(SPCache *pCache);
@@ -107,12 +95,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
   ASSERT(nRef >= 0);
 
   if (nRef == 0) {
-    if (1 /*TODO: page still clean*/) {
+    tdbPCacheUnpinPage(pCache, pPage);
-      tdbPCacheUnpinPage(pCache, pPage);
-    } else {
-      // TODO
-      ASSERT(0);
-    }
   }
 }
 
@@ -192,6 +175,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
 
   tdbPCacheLock(pCache);
 
+  ASSERT(!pPage->isDirty);
+
   nRef = TDB_GET_PAGE_REF(pPage);
   ASSERT(nRef >= 0);
   if (nRef == 0) {

source/libs/tdb/src/db/tdbPager.c

@@ -15,20 +15,6 @@
 
 #include "tdbInt.h"
 
-struct SPager {
-  char    *dbFileName;
-  char    *jFileName;
-  int      pageSize;
-  uint8_t  fid[TDB_FILE_ID_LEN];
-  tdb_fd_t fd;
-  tdb_fd_t jfd;
-  SPCache *pCache;
-  SPgno    dbFileSize;
-  SPgno    dbOrigSize;
-  SPage   *pDirty;
-  u8       inTran;
-};
-
 typedef struct __attribute__((__packed__)) {
   u8    hdrString[16];
   u16   pageSize;
@@ -41,9 +27,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
 
 #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
 static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
 static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
 static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
 
@@ -131,26 +116,36 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
 }
 
 int tdbPagerWrite(SPager *pPager, SPage *pPage) {
-  int ret;
+  int     ret;
+  SPage **ppPage;
 
+  ASSERT(pPager->inTran);
+#if 0
   if (pPager->inTran == 0) {
     ret = tdbPagerBegin(pPager);
     if (ret < 0) {
       return -1;
     }
   }
+#endif
 
   if (pPage->isDirty) return 0;
 
+  // ref page one more time so the page will not be release
+  TDB_REF_PAGE(pPage);
+
   // Set page as dirty
   pPage->isDirty = 1;
 
-  // Add page to dirty list
+  // Add page to dirty list(TODO: NOT use O(n^2) algorithm)
-  // TODO: sort the list according to the page number
+  for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
-  pPage->pDirtyNext = pPager->pDirty;
+       ppPage = &((*ppPage)->pDirtyNext)) {
-  pPager->pDirty = pPage;
+  }
+  ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
+  pPage->pDirtyNext = *ppPage;
+  *ppPage = pPage;
 
-  // Write page to journal
+  // Write page to journal if neccessary
   if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
     ret = tdbPagerWritePageToJournal(pPager, pPage);
     if (ret < 0) {
@@ -184,54 +179,46 @@ int tdbPagerCommit(SPager *pPager) {
   SPage *pPage;
   int    ret;
 
-  // Begin commit
+  // sync the journal file
-  {
+  ret = tdbOsFSync(pPager->jfd);
-    // TODO: Sync the journal file (Here or when write ?)
+  if (ret < 0) {
+    // TODO
+    ASSERT(0);
+    return 0;
   }
 
-  for (;;) {
+  // loop to write the dirty pages to file
-    pPage = pPager->pDirty;
+  for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
-
+    // TODO: update the page footer
-    if (pPage == NULL) break;
-
     ret = tdbPagerWritePageToDB(pPager, pPage);
     if (ret < 0) {
       ASSERT(0);
       return -1;
     }
+  }
 
+  // release the page
+  for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
     pPager->pDirty = pPage->pDirtyNext;
     pPage->pDirtyNext = NULL;
 
-    // TODO: release the page
+    pPage->isDirty = 0;
+
+    tdbPCacheRelease(pPager->pCache, pPage);
   }
 
+  // sync the db file
   tdbOsFSync(pPager->fd);
 
+  // remote the journal file
   tdbOsClose(pPager->jfd);
   tdbOsRemove(pPager->jFileName);
-  // pPager->jfd = -1;
+  pPager->dbOrigSize = pPager->dbFileSize;
+  pPager->inTran = 0;
 
   return 0;
 }
 
-static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
-  i64 offset;
-  int ret;
-
-  ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
-
-  offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
-  ret = tdbOsPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
-  if (ret < 0) {
-    // TODO: handle error
-    return -1;
-  }
-  return 0;
-}
-
-int tdbPagerGetPageSize(SPager *pPager) { return pPager->pageSize; }
-
 int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
   SPage *pPage;
   SPgid  pgid;
@@ -247,7 +234,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
 
   // Initialize the page if need
   if (!TDB_PAGE_INITIALIZED(pPage)) {
-    ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+    ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 1);
     if (ret < 0) {
       return -1;
     }
@@ -284,7 +271,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   ASSERT(!TDB_PAGE_INITIALIZED(pPage));
 
   // Initialize the page if need
-  ret = tdbPagerInitPage(pPager, pPage, initPage, arg);
+  ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
   if (ret < 0) {
     return -1;
   }
@@ -332,10 +319,11 @@ static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
   return 0;
 }
 
-static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg) {
+static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage) {
   int ret;
   int lcode;
   int nLoops;
+  i64 nRead;
 
   lcode = TDB_TRY_LOCK_PAGE(pPage);
   if (lcode == P_LOCK_SUCC) {
@@ -344,6 +332,19 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
       return 0;
     }
 
+    if (loadPage) {
+      nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * TDB_PAGE_PGNO(pPage));
+      if (nRead < 0) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      } else if (nRead < pPage->pageSize) {
+        // TODO
+        ASSERT(0);
+        return -1;
+      }
+    }
+
     ret = (*initPage)(pPage, arg);
     if (ret < 0) {
       TDB_UNLOCK_PAGE(pPage);

source/libs/tdb/src/db/tdbTxn.c

@@ -15,17 +15,29 @@
 
 #include "tdbInt.h"
 
-int tdbTxnBegin(TENV *pEnv) {
+// int tdbTxnBegin(TENV *pEnv) {
-  // TODO
+//   // TODO
-  return 0;
+//   return 0;
-}
+// }
 
-int tdbTxnCommit(TENV *pEnv) {
+// int tdbTxnCommit(TENV *pEnv) {
-  // TODO
+//   SPager *pPager = NULL;
-  return 0;
+//   int     ret;
-}
 
-int tdbTxnRollback(TENV *pEnv) {
+//   for (;;) {
-  // TODO
+//     break;
-  return 0;
+//     ret = tdbPagerCommit(pPager);
-}
+//     if (ret < 0) {
+//       ASSERT(0);
+//       return -1;
+//     }
+//   }
+
+//   // TODO
+//   return 0;
+// }
+
+// int tdbTxnRollback(TENV *pEnv) {
+//   // TODO
+//   return 0;
+// }

source/libs/tdb/src/inc/tdbBtree.h

@@ -40,7 +40,7 @@ struct SBTC {
 // SBTree
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
-int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen);
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 

source/libs/tdb/src/inc/tdbEnv.h

@@ -25,11 +25,20 @@ typedef struct STEnv {
   char    *jfname;
   int      jfd;
   SPCache *pCache;
+  SPager  *pgrList;
+  int      nPager;
+  int      nPgrHash;
+  SPager **pgrHash;
 } TENV;
 
 int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
 int tdbEnvClose(TENV *pEnv);
+int tdbBegin(TENV *pEnv);
+int tdbCommit(TENV *pEnv);
+int tdbRollback(TENV *pEnv);
 
+void    tdbEnvAddPager(TENV *pEnv, SPager *pPager);
+void    tdbEnvRemovePager(TENV *pEnv, SPager *pPager);
 SPager *tdbEnvGetPager(TENV *pEnv, const char *fname);
 
 #ifdef __cplusplus

source/libs/tdb/src/inc/tdbInt.h

@@ -16,8 +16,6 @@
 #ifndef _TD_TDB_INTERNAL_H_
 #define _TD_TDB_INTERNAL_H_
 
-#include "os.h"
-
 #include "tdb.h"
 
 #ifdef __cplusplus
@@ -91,23 +89,6 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
 // dbname
 #define TDB_MAX_DBNAME_LEN 24
 
-// tdb_log
-#define tdbError(var)
-
-#define TERR_A(val, op, flag)  \
-  do {                         \
-    if (((val) = (op)) != 0) { \
-      goto flag;               \
-    }                          \
-  } while (0)
-
-#define TERR_B(val, op, flag)     \
-  do {                            \
-    if (((val) = (op)) == NULL) { \
-      goto flag;                  \
-    }                             \
-  } while (0)
-
 #define TDB_VARIANT_LEN ((int)-1)
 
 typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);

source/libs/tdb/src/inc/tdbOs.h

@@ -24,6 +24,8 @@ extern "C" {
 #define TDB_FOR_TDENGINE
 
 #ifdef TDB_FOR_TDENGINE
+#include "os.h"
+#include "thash.h"
 
 // For memory -----------------
 #define tdbOsMalloc  taosMemoryMalloc
@@ -95,7 +97,11 @@ typedef int tdb_fd_t;
 
 #define tdbOsOpen(PATH, OPTION, MODE) open((PATH), (OPTION), (MODE))
 
-#define tdbOsClose close
+#define tdbOsClose(FD) \
+  do {                 \
+    close(FD);         \
+    (FD) = -1;         \
+  } while (0)
 
 i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes);
 i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset);

source/libs/tdb/src/inc/tdbPCache.h

@@ -33,6 +33,18 @@ extern "C" {
   SPager *pPager;       \
   SPgid   pgid;
 
+// For page ref
+#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
+#if 0
+#define TDB_REF_PAGE(pPage)     (++(pPage)->nRef)
+#define TDB_UNREF_PAGE(pPage)   (--(pPage)->nRef)
+#define TDB_GET_PAGE_REF(pPage) ((pPage)->nRef)
+#else
+#define TDB_REF_PAGE(pPage)     atomic_add_fetch_32(&((pPage)->nRef), 1)
+#define TDB_UNREF_PAGE(pPage)   atomic_sub_fetch_32(&((pPage)->nRef), 1)
+#define TDB_GET_PAGE_REF(pPage) atomic_load_32(&((pPage)->nRef))
+#endif
+
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
 SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);

source/libs/tdb/src/inc/tdbPage.h

@@ -100,6 +100,7 @@ struct SPage {
 // APIs
 #define TDB_PAGE_TOTAL_CELLS(pPage)        ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
 #define TDB_PAGE_USABLE_SIZE(pPage)        ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
+#define TDB_PAGE_FREE_SIZE(pPage)          (*(pPage)->pPageMethods->getFreeBytes)(pPage)
 #define TDB_PAGE_PGNO(pPage)               ((pPage)->pgid.pgno)
 #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset)
 #define TDB_PAGE_OFFSET_SIZE(pPage)        ((pPage)->pPageMethods->szOffset)

source/libs/tdb/src/inc/tdbPager.h

@@ -20,13 +20,28 @@
 extern "C" {
 #endif
 
+struct SPager {
+  char    *dbFileName;
+  char    *jFileName;
+  int      pageSize;
+  uint8_t  fid[TDB_FILE_ID_LEN];
+  tdb_fd_t fd;
+  tdb_fd_t jfd;
+  SPCache *pCache;
+  SPgno    dbFileSize;
+  SPgno    dbOrigSize;
+  SPage   *pDirty;
+  u8       inTran;
+  SPager  *pNext;      // used by TENV
+  SPager  *pHashNext;  // used by TENV
+};
+
 int  tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
 int  tdbPagerClose(SPager *pPager);
 int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
 int  tdbPagerWrite(SPager *pPager, SPage *pPage);
 int  tdbPagerBegin(SPager *pPager);
 int  tdbPagerCommit(SPager *pPager);
-int  tdbPagerGetPageSize(SPager *pPager);
 int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage);

source/libs/tdb/src/inc/tdbTxn.h

@@ -28,10 +28,6 @@ struct STxn {
   void *xArg;
 };
 
-int tdbTxnBegin(TENV *pEnv);
-int tdbTxnCommit(TENV *pEnv);
-int tdbTxnRollback(TENV *pEnv);
-
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/src/inc/tdbUtil.h

@@ -101,6 +101,8 @@ static inline int tdbGetVarInt(const u8 *p, int *v) {
   return n;
 }
 
+static inline u32 tdbCstringHash(const char *s) { return MurmurHash3_32(s, strlen(s)); }
+
 #ifdef __cplusplus
 }
 #endif

source/libs/tdb/test/tdbTest.cpp

@@ -118,10 +118,10 @@ TEST(tdb_test, simple_test) {
   TENV          *pEnv;
   TDB           *pDb;
   FKeyComparator compFunc;
-  int            nData = 1000000;
+  int            nData = 50000000;
 
   // Open Env
-  ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv);
+  ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
   GTEST_ASSERT_EQ(ret, 0);
 
   // Create a database
@@ -134,36 +134,23 @@ TEST(tdb_test, simple_test) {
     char val[64];
 
     {  // Insert some data
-      int       i = 1;
+      for (int i = 1; i <= nData;) {
-      SPoolMem *pPool;
+        tdbBegin(pEnv);
-      int       memPoolCapacity = 16 * 1024;
 
-      pPool = openPool();
+        for (int k = 0; k < 2000; k++) {
-
+          sprintf(key, "key%d", i);
-      tdbTxnBegin(pEnv);
+          sprintf(val, "value%d", i);
-
+          ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-      for (;;) {
+          GTEST_ASSERT_EQ(ret, 0);
-        if (i > nData) break;
+          i++;
-
-        sprintf(key, "key%d", i);
-        sprintf(val, "value%d", i);
-        ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-        GTEST_ASSERT_EQ(ret, 0);
-
-        if (pPool->size >= memPoolCapacity) {
-          tdbTxnCommit(pEnv);
-
-          clearPool(pPool);
-
-          tdbTxnBegin(pEnv);
         }
 
-        i++;
+        tdbCommit(pEnv);
       }
-
-      closePool(pPool);
     }
 
+    tdbCommit(pEnv);
+
     {  // Query the data
       void *pVal = NULL;
       int   vLen;
@@ -173,6 +160,7 @@ TEST(tdb_test, simple_test) {
         sprintf(val, "value%d", i);
 
         ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen);
+        ASSERT(ret == 0);
         GTEST_ASSERT_EQ(ret, 0);
 
         GTEST_ASSERT_EQ(vLen, strlen(val));

source/os/src/osFile.c

@@ -294,7 +294,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&((*ppFile)->rwlock));
 #endif
-  if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) {
+  if (ppFile == NULL || *ppFile == NULL) {
     return 0;
   }
   if ((*ppFile)->fp != NULL) {

source/os/src/osMemory.c

@@ -23,15 +23,22 @@
 
 #define TD_MEMORY_STACK_TRACE_DEPTH 10
 
+typedef struct TdMemoryInfo *TdMemoryInfoPtr;
+
 typedef struct TdMemoryInfo {
   int32_t symbol;
   int32_t memorySize;
   void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH];     // gdb: disassemble /m 0xXXX
-} *TdMemoryInfoPtr , TdMemoryInfo;
+  // TdMemoryInfoPtr pNext;
+  // TdMemoryInfoPtr pPrev;
+} TdMemoryInfo;
+
+// static TdMemoryInfoPtr GlobalMemoryPtr = NULL;
 
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
-
+  #define tstrdup(str) _strdup(str)
 #else
+  #define tstrdup(str) strdup(str)
 
 #include<execinfo.h>
 
@@ -129,6 +136,26 @@ void *taosMemoryRealloc(void *ptr, int32_t size) {
 #endif
 }
 
+void *taosMemoryStrDup(void *ptr) {
+#ifdef USE_TD_MEMORY
+  if (ptr == NULL) return NULL;
+  
+  TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
+  assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL);
+
+  void *tmp = tstrdup((const char *)pTdMemoryInfo);
+  if (tmp == NULL) return NULL;
+  
+  memcpy(tmp, pTdMemoryInfo, sizeof(TdMemoryInfo));
+  taosBackTrace(((TdMemoryInfoPtr)tmp)->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH);
+
+  return (char*)tmp  + sizeof(TdMemoryInfo);
+#else
+  return tstrdup((const char *)ptr);
+#endif
+}
+
+
 void taosMemoryFree(const void *ptr) {
   if (ptr == NULL) return;
 

source/os/src/osProc.c

@@ -32,6 +32,18 @@ int32_t taosNewProc(char **args) {
   }
 }
 
+void taosWaitProc(int32_t pid) {
+  int32_t status = 0;
+  waitpid(pid, &status, 0);
+}
+
+void taosKillProc(int32_t pid) { kill(pid, SIGINT); }
+
+bool taosProcExist(int32_t pid) {
+  int32_t p = getpgid(pid);
+  return p >= 0;
+}
+
 // the length of the new name must be less than the original name to take effect
 void taosSetProcName(int32_t argc, char **argv, const char *name) {
   prctl(PR_SET_NAME, name);
@@ -48,8 +60,3 @@ void taosSetProcName(int32_t argc, char **argv, const char *name) {
 }
 
 void taosSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; }
-
-bool taosProcExists(int32_t pid) {
-  int32_t p = getpgid(pid);
-  return p >= 0;
-}

source/util/src/tcompare.c

@@ -173,6 +173,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
   if (isnan(p2)) {
     return 1;
   }
+
   if (FLT_EQUAL(p1, p2)) {
     return 0;
   }

source/util/src/tconfig.c

@@ -590,7 +590,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
 }
 
 int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
-  uInfo("load from global env variables not implemented yet");
+  uInfo("load from env variables not implemented yet");
   return 0;
 }
 

tests/script/jenkins/basic.txt

@@ -23,8 +23,8 @@
 ./test.sh -f tsim/insert/null.sim
 
 # ---- parser
-#./test.sh -f tsim/parser/groupby-basic.sim
+./test.sh -f tsim/parser/groupby-basic.sim
-#./test.sh -f tsim/parser/fourArithmetic-basic.sim
+./test.sh -f tsim/parser/fourArithmetic-basic.sim
 
 # ---- query
 ./test.sh -f tsim/query/interval.sim

tests/script/runAllSimCases.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##################################################
+# 
+# Do simulation test 
+#
+##################################################
+
+set -e
+#set -x
+
+while read line
+do
+  firstChar=`echo ${line:0:1}`
+  if [[ -n "$line" ]]  && [[ $firstChar != "#" ]]; then
+    echo "======== $line ========"
+    $line 
+  fi
+done < ./jenkins/basic.txt
+
+

tests/script/tsim/db/alter_option.sim

@@ -66,7 +66,7 @@ print ============= create database
 #  | REPLICA value      [1 | 3]
 #  | WAL value          [1 | 2]
 
-sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1
+sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1
 sql show databases
 print rows: $rows
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
@@ -92,10 +92,10 @@ endi
 if $data5_db != 1 then  # quorum
   return -1
 endi
-if $data6_db != 240 then  # days
+if $data6_db != 345600 then  # days
   return -1
 endi
-if $data7_db != 1000,1000,1000 then # keep
+if $data7_db != 1440000,1440000,1440000 then # keep
   return -1
 endi
 if $data8_db != 3 then  # cache
@@ -138,7 +138,7 @@ sql_error alter database db ntables 0
 sql_error alter database db ntables 1
 sql_error alter database db ntables 10
 
-#print ============== modify replica
+#print ============== modify replica        # TD-14409
 sql_error alter database db replica 2
 sql_error alter database db replica 5
 sql_error alter database db replica -1
@@ -186,7 +186,7 @@ print ============== modify keep
 sql alter database db keep 2000
 sql show databases
 print keep $data7_db
-if $data7_db != 1000,1000,2000 then
+if $data7_db != 2000,2000,2000 then
   return -1
 endi
 
@@ -270,7 +270,7 @@ if $data12_db != 2 then
   return -1
 endi
 
-#sql_error alter database db wal 0     # TD-14436
+sql_error alter database db wal 0     # TD-14436
 sql_error alter database db wal 3
 sql_error alter database db wal 100
 sql_error alter database db wal -1

tests/script/tsim/db/basic6.sim

@@ -15,7 +15,7 @@ $tb = $tbPrefix . $i
 
 print =============== step1
 # quorum  presicion 
-sql create database $db vgroups 8 replica 1 days 20 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
+sql create database $db vgroups 8 replica 1 days 2880 keep 3650 cache 32 blocks 12 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us'
 sql show databases
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1 $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
@@ -35,7 +35,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data06 != 20 then 
+if $data06 != 2880 then 
   return -1
 endi
 if $data07 != 3650,3650,3650 then 
@@ -67,7 +67,7 @@ print =============== step4
 sql_error drop database $db
 
 print =============== step5
-sql create database $db replica 1 days 15 keep 1500
+sql create database $db replica 1 days 21600 keep 2160000
 sql show databases
 print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
 if $data00 != $db then 
@@ -79,7 +79,7 @@ endi
 if $data04 != 1 then 
   return -1
 endi
-if $data06 != 15 then 
+if $data06 != 21600 then 
   return -1
 endi
 

tests/script/tsim/parser/fourArithmetic-basic.sim

@@ -78,6 +78,10 @@ while $i < $tbNum
   $tstart = 1640966400000
 endw
 
+
+$loop_test = 0
+loop_test_pos:
+
 sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
 print ===> rows: $rows 
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
@@ -107,4 +111,31 @@ endi
 if $data93 != 8.000000000 then
  return -1
 endi
+
+if $loop_test == 0 then
+  print =============== stop and restart taosd
+  system sh/exec.sh -n dnode1 -s stop -x SIGINT
+  system sh/exec.sh -n dnode1 -s start
+  
+  $loop_cnt = 0
+  check_dnode_ready_0:
+  	$loop_cnt = $loop_cnt + 1
+  	sleep 200
+  	if $loop_cnt == 10 then
+  	  print ====> dnode not ready!
+  		return -1
+  	endi
+  sql show dnodes
+  print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+  if $data00 != 1 then
+    return -1
+  endi
+  if $data04 != ready then
+    goto check_dnode_ready_0
+  endi
+
+  $loop_test = 1  
+  goto loop_test_pos
+endi
+
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/groupby-basic.sim

@@ -45,7 +45,7 @@ $tstart = 1640966400000  # 2022-01-01 00:00:00.000
 print ==== create db, stable, ctables, insert data
 sql drop database if exists $db -x step1
 step1:
-sql create database if not exists $db keep 3650
+sql create database if not exists $db
 sql use $db
 
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
@@ -131,6 +131,7 @@ if $data91 != 9 then
   return -1
 endi
 
+print ==== select first(ts),c1 from group_tb0 group by c1;
 sql select first(ts),c1 from group_tb0 group by c1;
 print rows: $rows
 print $data00 $data01 $data02 $data03
@@ -142,19 +143,38 @@ if $row != 10 then
   return -1
 endi
 
-if $data00 != @2022-01-01 00:00:00.000@ then
+if $data00 != @22-01-01 00:00:00.000@ then
   return -1
 endi
 if $data01 != 0 then
   return -1
 endi
-if $data90 != @2022-01-01 00:00:00.009@ then
+if $data90 != @22-01-01 00:00:00.009@ then
   return -1
 endi
 if $data91 != 9 then
   return -1
 endi
 
+print ==== select first(ts),c1 from interval(5m) group_tb0 group by c1;
+sql select first(ts),c1 from group_tb0 group by c1;
+print rows: $rows
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data80 $data81 $data82 $data83
+print $data90 $data91 $data92 $data93
+
+return 
+
+
+
+
+
+
+
+
+
 
 sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
 if $row != 20 then

tests/script/tsim/query/interval-offset.sim

@@ -73,39 +73,45 @@ sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
 sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
 
 print ================ start query ======================
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
+sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(*) from ct1 interval(10s, 2s)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
+print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(*) from ct1 interval(10s, 2s)
 print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04
+print ===> rows0: $data00 $data01 $data02 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14
+print ===> rows1: $data10 $data11 $data12 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24
+print ===> rows2: $data20 $data21 $data22 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+print ===> rows3: $data30 $data31 $data32 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+print ===> rows4: $data40 $data41 $data42 $data45
 if $rows != 5 then
   return -1
 endi
-if $data00 != 1 then
+if $data00 != @22-01-01 01:00:52.000@ then
   return -1
 endi
-if $data40 != 1 then
+if $data02 != 10000 then
+  return -1
+endi
+if $data45 != 1 then
   return -1
 endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s)
+sql select _wstartts, _wendts, _wduration, _qstartts, _qendts,  count(*) from ct1 interval(10s, 2s) sliding(10s)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s)
+print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts,  count(*) from ct1 interval(10s, 2s) sliding(10s)
 print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04
+print ===> rows0: $data00 $data01 $data02 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14
+print ===> rows1: $data10 $data11 $data12 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24
+print ===> rows2: $data20 $data21 $data22 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+print ===> rows3: $data30 $data31 $data32 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+print ===> rows4: $data40 $data41 $data42 $data45
 if $rows != 5 then
   return -1
 endi
-if $data00 != 1 then
+if $data00 != @22-01-01 01:00:52.000@ then
   return -1
 endi
-if $data40 != 1 then
+if $data02 != 10000 then
+  return -1
+endi
+if $data45 != 1 then
   return -1
 endi
 
@@ -176,107 +182,81 @@ if $data70 != 1 then
   return -1
 endi
 
-sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from ct2 interval(1d, 2h) sliding(12h)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h)
-print ===> rows: $rows
-print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05
-print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15
-print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25
-print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35
-print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
-print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55
-print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65
-print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75
-if $rows != 8 then
-  return -1
-endi
-if $data05 != 1 then
-  return -1
-endi
-if $data15 != 2 then
-  return -1
-endi
-if $data75 != 1 then
-  return -1
-endi
-if $data02 != 86400000 then
-  return -1
-endi
-
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-# if $rows != 5 then
+  return -1
-#   return -1
+endi
-# endi
+if $data00 != @21-12-08 00:00:00.000@ then
-# f $data00 != 1 then
+  return -1
-#   return -1
+endi
-# endi
+if $data31 != 1 then
-# if $data40 != 1 then
+  return -1
-#   return -1
+endi
-# endi
+if $data34 != $data31 then
+  return -1
+endi
+if $data02 != 2678400000 then
+  return -1
+endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w)
+sql select  _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-#if $rows != 5 then
+  return -1
-#  return -1
+endi
-#endi
+if $data00 != @21-11-30 08:00:00.000@ then
-#if $data00 != 1 then
+  return -1
-#  return -1
+endi
-#endi
+if $data01 != NULL then
-#if $data40 != 1 then
+  return -1
-#  return -1
+endi
-#endi
+if $data31 != $data34 then
+  return -1
+endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*)  from ct3 interval(1n, 1w) sliding(4w)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(4w)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
 print ===> rows3: $data30 $data31 $data32 $data33 $data34
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+if $rows != 4 then
-print ===> rows5: $data50 $data51 $data52 $data53 $data54
+  return -1
-print ===> rows6: $data60 $data61 $data62 $data63 $data64
+endi
-print ===> rows7: $data70 $data71 $data72 $data73 $data74
+if $data01 != NULL then
-#if $rows != 8 then
+  return -1
-#  return -1
+endi
-#endi
+if $data04 != 1 then
-#if $data00 != 2 then
+  return -1
-#  return -1
+endi
-#endi
-#if $data70 != 1 then
-#  return -1
-#endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-#if $rows != 5 then
+endi
-#  return -1
+if $data01 != 2 then
-#endi
+  return -1
-#if $data00 != 1 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data40 != 1 then
+endi
-#  return -1
-#endi
 
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n)
@@ -284,38 +264,31 @@ print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-#if $rows != 5 then
+endi
-#  return -1
+if $data00 != 2 then
-#endi
+  return -1
-#if $data00 != 1 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data40 != 1 then
+endi
-#  return -1
-#endi
 
-sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n)
+sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n) sliding(12n)
-print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n)
+print ===> select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct4 interval(1y, 6n) sliding(12n)
 print ===> rows: $rows
 print ===> rows0: $data00 $data01 $data02 $data03 $data04
 print ===> rows1: $data10 $data11 $data12 $data13 $data14
 print ===> rows2: $data20 $data21 $data22 $data23 $data24
-print ===> rows3: $data30 $data31 $data32 $data33 $data34
+if $rows != 3 then
-print ===> rows4: $data40 $data41 $data42 $data43 $data44
+  return -1
-print ===> rows5: $data50 $data51 $data52 $data53 $data54
+endi
-print ===> rows6: $data60 $data61 $data62 $data63 $data64
+if $data01 != 2 then
-print ===> rows7: $data70 $data71 $data72 $data73 $data74
+  return -1
-#if $rows != 8 then
+endi
-#  return -1
+if $data04 != 2 then
-#endi
+  return -1
-#if $data00 != 2 then
+endi
-#  return -1
-#endi
-#if $data70 != 1 then
-#  return -1
-#endi
 
 #=================================================
 print =============== stop and restart taosd

tests/script/tsim/query/scalarFunction.sim

@@ -202,7 +202,7 @@ if $rows != $rowNum then
 endi
 
 print ====> sin
-#sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1  # TD-14426
+sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1
 sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1
 print ====> rows: $rows

tests/script/tsim/tmq/multiTopic.sim

@@ -0,0 +1,224 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene2 and scene4
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 1
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 1 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
+print =============== will support: * from stb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== insert data
+$rowNum = 100
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , $c , $x , $binary )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$totalMsgCnt = $rowNum * $tbNum
+print inserted totalMsgCnt: $totalMsgCnt
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+$numOfTopics = 2
+$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $totalMsgCntOfmultiTopics
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2"
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2"
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 20000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+$numOfTopics = 3
+$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $totalMsgCntOfmultiTopics
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 300, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+$numOfTopics = 3
+$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $totalMsgCntOfmultiTopics
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 30000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/oneTopic.sim

@@ -0,0 +1,264 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+######## This test case include scene1 and scene3
+######## ######## ######## ######## ######## ######## ######## ######## ######## ######## 
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$loop_cnt = 0
+$vgroups  = 1
+$dbNamme  = d0
+loop_vgroups:
+print =============== create database $dbNamme vgroups $vgroups
+sql create database $dbNamme vgroups $vgroups
+sql show databases
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+
+if $loop_cnt == 0 then 
+  if $rows != 2 then 
+    return -1
+  endi
+  if $data02 != 1 then  # vgroups
+    print vgroups: $data02
+    return -1
+  endi
+else
+  if $rows != 3 then 
+    return -1
+  endi
+  if $data00 == d1 then 
+    if $data02 != 4 then  # vgroups
+      print vgroups: $data02
+      return -1
+    endi
+  else  
+    if $data12 != 4 then  # vgroups
+      print vgroups: $data12
+      return -1
+    endi
+  endi
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+$tbPrefix = ct
+$tbNum = 100
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i )
+  $i = $i + 1
+endw
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
+print =============== will support: * from stb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+
+sql show tables
+if $rows != 101 then 
+  return -1
+endi
+
+print =============== insert data
+$rowNum = 100
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , $c , $x , $binary )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$totalMsgCnt = $rowNum * $tbNum
+print inserted totalMsgCnt: $totalMsgCnt
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $totalMsgCnt
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#print cmd result----> $system_content
+##if $system_content != @{consume success: 10000, 0}@ then
+#if $system_content != $expect_result then
+#  return -1
+#endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $rowNum
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 100, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 100, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 100, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+$expect_result = @{consume success: @
+$expect_result = $expect_result . $totalMsgCnt
+$expect_result = $expect_result . @, @ 
+$expect_result = $expect_result . 0}
+print expect_result----> $expect_result
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
+print cmd result----> $system_content
+#if $system_content != @{consume success: 10000, 0}@ then
+if $system_content != $expect_result then
+  return -1
+endi
+
+if $loop_cnt == 0 then
+  $loop_cnt = 1
+  $vgroups  = 4
+  $dbNamme  = d1
+  goto loop_vgroups
+endi
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT