Merge branch '3.0' of github.com:taosdata/TDengine into szhou/tbname-prune-vgroup

Jenkinsfile2

@@ -424,7 +424,7 @@ pipeline {
                             echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
                         }
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 130, unit: 'MINUTES'){
+                            timeout(time: 150, unit: 'MINUTES'){
                                 pre_test()
                                 script {
                                     sh '''

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG main
+        GIT_TAG 3.0
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/12-taos-sql/10-function.md

@@ -483,6 +483,93 @@ return_timestamp: {
 - The precision of the returned timestamp is same as the precision set for the current data base in use
 - return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT.
 
+#### TO_CHAR
+
+```sql
+TO_CHAR(ts, format_str_literal)
+```
+
+**Description**: Convert a ts column to string as the format specified
+
+**Return value type**: VARCHAR
+
+**Applicable column types**: TIMESTAMP
+
+**Nested query**: It can be used in both the outer query and inner query in a nested query.
+
+**Applicable table types**: standard tables and supertables
+
+**Supported Formats**
+
+| **Format** | **Comment**| **example** |
+| --- | --- | --- |
+|AM,am,PM,pm| Meridiem indicator(without periods) | 07:00:00am|
+|A.M.,a.m.,P.M.,p.m.| Meridiem indicator(with periods)| 07:00:00a.m.|
+|YYYY,yyyy|year, 4 or more digits| 2023-10-10|
+|YYY,yyy| year, last 3 digits| 023-10-10|
+|YY,yy| year, last 2 digits| 23-10-10|
+|Y,y| year, last digit| 3-10-10|
+|MONTH|full uppercase of month| 2023-JANUARY-01|
+|Month|full capitalized month| 2023-January-01|
+|month|full lowercase of month| 2023-january-01|
+|MON| abbreviated uppercase of month(3 char)| JAN, SEP|
+|Mon| abbreviated capitalized month| Jan, Sep|
+|mon|abbreviated lowercase of month| jan, sep|
+|MM,mm|month number 01-12|2023-01-01|
+|DD,dd|month day, 01-31||
+|DAY|full uppercase of week day|MONDAY|
+|Day|full capitalized week day|Monday|
+|day|full lowercase of week day|monday|
+|DY|abbreviated uppercase of week day|MON|
+|Dy|abbreviated capitalized week day|Mon|
+|dy|abbreviated lowercase of week day|mon|
+|DDD|year day, 001-366||
+|D,d|week day number, 1-7, Sunday(1) to Saturday(7)||
+|HH24,hh24|hour of day, 00-23|2023-01-30 23:59:59|
+|hh12,HH12, hh, HH| hour of day, 01-12|2023-01-30 12:59:59PM|
+|MI,mi|minute, 00-59||
+|SS,ss|second, 00-59||
+|MS,ms|milli second, 000-999||
+|US,us|micro second, 000000-999999||
+|NS,ns|nano second, 000000000-999999999||
+|TZH,tzh|time zone hour|2023-01-30 11:59:59PM +08|
+
+**More explanations**:
+- The output format of `Month`, `Day` are left aligined, like`2023-OCTOBER  -01`, `2023-SEPTEMBER-01`, `September` is the longest, no paddings. Week days are slimilar.
+- When `ms`,`us`,`ns` are used in `to_char`, like `to_char(ts, 'yyyy-mm-dd hh:mi:ss.ms.us.ns')`, The time of `ms`,`us`,`ns` corresponds to the same fraction seconds. When ts is `1697182085123`, the output of `ms` is `123`, `us` is `123000`, `ns` is `123000000`.
+- If we want to output some characters of format without converting, surround it with double quotes. `to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. If want to output double quotes, add a back slash before double quote, like `to_char(ts, '\"yyyy-mm-dd\"')` will output `"2023-10-10"`.
+- For formats that output digits, the uppercase and lowercase formats are the same.
+- It's recommended to put time zone in the format, if not, the default time zone will be that in server or client.
+- The precision of the input timestamp will be recognized automatically according to the precision of the table used, milliseconds will be used if no table is specified.
+
+#### TO_TIMESTAMP
+
+```sql
+TO_TIMESTAMP(ts_str_literal, format_str_literal)
+```
+
+**Description**: Convert a formated timestamp string to a timestamp
+
+**Return value type**: TIMESTAMP
+
+**Applicable column types**: VARCHAR
+
+**Nested query**: It can be used in both the outer query and inner query in a nested query.
+
+**Applicable table types**: standard tables and supertables
+
+**Supported Formats**: The same as `TO_CHAR`.
+
+**More explanations**:
+- When `ms`, `us`, `ns` are used in `to_timestamp`, if multi of them are specified, the results are accumulated. For example, `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` will output the timestamp of `2023-10-10 10:10:10.123456789`.
+- The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored.
+- If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`.
+- To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client.
+- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone.
+- If `AM` or `PM` is specified in formats, the Hour must between `1-12`.
+- In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically.
+- The precision of the output timestamp will be the same as the table in SELECT stmt, millisecond will be used if no table is specified. The output of `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` will be truncated to millisecond precision. If a nano precision table is specified, no truncation will be applied. Like `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
+
 
 ### Time and Date Functions
 

docs/zh/12-taos-sql/10-function.md

@@ -483,6 +483,93 @@ return_timestamp: {
 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。
 - return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。
 
+#### TO_CHAR
+
+```sql
+TO_CHAR(ts, format_str_literal)
+```
+
+**功能说明**: 将timestamp类型按照指定格式转换为字符串
+
+**返回结果数据类型**: VARCHAR
+
+**应用字段**: TIMESTAMP
+
+**嵌套子查询支持**: 适用于内层查询和外层查询
+
+**适用于**: 表和超级表
+
+**支持的格式**
+
+| **格式** | **说明**| **例子** |
+| --- | --- | --- |
+|AM,am,PM,pm| 无点分隔的上午下午 | 07:00:00am|
+|A.M.,a.m.,P.M.,p.m.| 有点分隔的上午下午| 07:00:00a.m.|
+|YYYY,yyyy|年, 4个及以上数字| 2023-10-10|
+|YYY,yyy| 年, 最后3位数字| 023-10-10|
+|YY,yy| 年, 最后2位数字| 23-10-10|
+|Y,y|年, 最后一位数字| 3-10-10|
+|MONTH|月, 全大写| 2023-JANUARY-01|
+|Month|月, 首字母大写| 2023-January-01|
+|month|月, 全小写| 2023-january-01|
+|MON| 月, 缩写, 全大写(三个字符)| JAN, SEP|
+|Mon| 月, 缩写, 首字母大写| Jan, Sep|
+|mon|月, 缩写, 全小写| jan, sep|
+|MM,mm|月, 数字 01-12|2023-01-01|
+|DD,dd|月日, 01-31||
+|DAY|周日, 全大写|MONDAY|
+|Day|周日, 首字符大写|Monday|
+|day|周日, 全小写|monday|
+|DY|周日, 缩写, 全大写|MON|
+|Dy|周日, 缩写, 首字符大写|Mon|
+|dy|周日, 缩写, 全小写|mon|
+|DDD|年日, 001-366||
+|D,d|周日, 数字, 1-7, Sunday(1) to Saturday(7)||
+|HH24,hh24|小时, 00-23|2023-01-30 23:59:59|
+|hh12,HH12, hh, HH| 小时, 01-12|2023-01-30 12:59:59PM|
+|MI,mi|分钟, 00-59||
+|SS,ss|秒, 00-59||
+|MS,ms|毫秒, 000-999||
+|US,us|微秒, 000000-999999||
+|NS,ns|纳秒, 000000000-999999999||
+|TZH,tzh|时区小时|2023-01-30 11:59:59PM +08|
+
+**使用说明**:
+- `Month`, `Day`等的输出格式是左对齐的, 右侧添加空格, 如`2023-OCTOBER  -01`, `2023-SEPTEMBER-01`, 9月是月份中英文字母数最长的, 因此9月没有空格. 星期类似.
+- 使用`ms`, `us`, `ns`时, 以上三种格式的输出只在精度上不同, 比如ts为 `1697182085123`,  `ms` 的输出为 `123`, `us` 的输出为 `123000`, `ns` 的输出为 `123000000`.
+- 时间格式中无法匹配规则的内容会直接输出. 如果想要在格式串中指定某些能够匹配规则的部分不做转换, 可以使用双引号, 如`to_char(ts, 'yyyy-mm-dd "is formated by yyyy-mm-dd"')`. 如果想要输出双引号, 那么在双引号之前加一个反斜杠, 如 `to_char(ts, '\"yyyy-mm-dd\"')` 将会输出 `"2023-10-10"`.
+- 那些输出是数字的格式, 如`YYYY`, `DD`, 大写与小写意义相同, 即`yyyy` 和 `YYYY` 可以互换.
+- 推荐在时间格式中带时区信息，如果不带则默认输出的时区为服务端或客户端所配置的时区.
+- 输入时间戳的精度由所查询表的精度确定, 若未指定表, 则精度为毫秒.
+
+#### TO_TIMESTAMP
+
+```sql
+TO_TIMESTAMP(ts_str_literal, format_str_literal)
+```
+
+**功能说明**: 将字符串按照指定格式转化为时间戳.
+
+**返回结果数据类型**: TIMESTAMP
+
+**应用字段**: VARCHAR
+
+**嵌套子查询支持**: 适用于内层查询和外层查询
+
+**适用于**: 表和超级表
+
+**支持的格式**: 与`to_char`相同
+
+**使用说明**:
+- 若`ms`, `us`, `ns`同时指定, 那么结果时间戳包含上述三个字段的和. 如 `to_timestamp('2023-10-10 10:10:10.123.000456.000000789', 'yyyy-mm-dd hh:mi:ss.ms.us.ns')` 输出为 `2023-10-10 10:10:10.123456789`对应的时间戳.
+- `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`.
+- 如果同一字段被指定了多次, 那么前面的指定将会被覆盖.  如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`.
+- 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。
+- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分.
+- 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12.
+- `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp('  23 年 - 1 月 - 01 日  ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换.
+- 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微妙和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
+
 
 ### 时间和日期函数
 

include/client/taos.h

@@ -338,8 +338,11 @@ typedef struct tmq_raw_data {
 DLL_EXPORT int32_t     tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
 DLL_EXPORT int32_t     tmq_write_raw(TAOS *taos, tmq_raw_data raw);
 DLL_EXPORT int         taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
+DLL_EXPORT int         taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid);
 DLL_EXPORT int         taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
                                                         TAOS_FIELD *fields, int numFields);
+DLL_EXPORT int         taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
+                                                        TAOS_FIELD *fields, int numFields, int64_t reqid);
 DLL_EXPORT void        tmq_free_raw(tmq_raw_data raw);
 
 // Returning null means error. Returned result need to be freed by tmq_free_json_meta

include/common/systable.h

@@ -49,6 +49,7 @@ extern "C" {
 #define TSDB_INS_TABLE_STREAMS           "ins_streams"
 #define TSDB_INS_TABLE_STREAM_TASKS      "ins_stream_tasks"
 #define TSDB_INS_TABLE_USER_PRIVILEGES   "ins_user_privileges"
+#define TSDB_INS_TABLE_VIEWS             "ins_views"
 
 #define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS        "perf_smas"

include/common/taosdef.h

@@ -42,7 +42,8 @@ typedef enum {
   TSDB_TEMP_TABLE = 4,    // temp table created by nest query
   TSDB_SYSTEM_TABLE = 5,
   TSDB_TSMA_TABLE = 6,  // time-range-wise sma
-  TSDB_TABLE_MAX = 7
+  TSDB_VIEW_TABLE = 7,
+  TSDB_TABLE_MAX = 8
 } ETableType;
 
 typedef enum {

include/common/tglobal.h

@@ -195,7 +195,8 @@ extern int64_t tsWalFsyncDataSizeLimit;
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;
-extern int32_t tsStreamCheckpointTickInterval;
+extern int32_t tsStreamCheckpointInterval;
+extern float   tsSinkDataRate;
 extern int32_t tsStreamNodeCheckInterval;
 extern int32_t tsTtlUnit;
 extern int32_t tsTtlPushIntervalSec;
@@ -204,9 +205,6 @@ extern int32_t tsTrimVDbIntervalSec;
 extern int32_t tsGrantHBInterval;
 extern int32_t tsUptimeInterval;
 
-extern int32_t tsRpcRetryLimit;
-extern int32_t tsRpcRetryInterval;
-
 extern bool    tsDisableStream;
 extern int64_t tsStreamBufferSize;
 extern bool    tsFilterScalarMode;

include/common/tgrant.h

@@ -51,11 +51,7 @@ typedef enum {
 } EGrantType;
 
 int32_t grantCheck(EGrantType grant);
-#ifndef TD_GRANT_OPTIMIZE
-int32_t grantAlterActiveCode(const char* old, const char* newer, char* out, int8_t type);
-#else
 int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
-#endif
 
 #ifndef GRANTS_CFG
 #ifdef TD_ENTERPRISE

include/common/tmsg.h

@@ -107,6 +107,8 @@ enum {
   HEARTBEAT_KEY_DBINFO,
   HEARTBEAT_KEY_STBINFO,
   HEARTBEAT_KEY_TMQ,
+  HEARTBEAT_KEY_DYN_VIEW,
+  HEARTBEAT_KEY_VIEWINFO,
 };
 
 typedef enum _mgmt_table {
@@ -141,6 +143,7 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_APPS,
   TSDB_MGMT_TABLE_STREAM_TASKS,
   TSDB_MGMT_TABLE_PRIVILEGES,
+  TSDB_MGMT_TABLE_VIEWS,
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
@@ -168,26 +171,12 @@ typedef enum _mgmt_table {
 
 #define TSDB_ALTER_USER_PASSWD                 0x1
 #define TSDB_ALTER_USER_SUPERUSER              0x2
-#define TSDB_ALTER_USER_ADD_READ_DB            0x3
+#define TSDB_ALTER_USER_ENABLE                 0x3
-#define TSDB_ALTER_USER_REMOVE_READ_DB         0x4
+#define TSDB_ALTER_USER_SYSINFO                0x4
-#define TSDB_ALTER_USER_ADD_WRITE_DB           0x5
+#define TSDB_ALTER_USER_ADD_PRIVILEGES         0x5
-#define TSDB_ALTER_USER_REMOVE_WRITE_DB        0x6
+#define TSDB_ALTER_USER_DEL_PRIVILEGES         0x6
-#define TSDB_ALTER_USER_ADD_ALL_DB             0x7
+#define TSDB_ALTER_USER_ADD_WHITE_LIST         0x7
-#define TSDB_ALTER_USER_REMOVE_ALL_DB          0x8
+#define TSDB_ALTER_USER_DROP_WHITE_LIST        0x8
-#define TSDB_ALTER_USER_ENABLE                 0x9
-#define TSDB_ALTER_USER_SYSINFO                0xA
-#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC    0xB
-#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
-#define TSDB_ALTER_USER_ADD_READ_TABLE         0xD
-#define TSDB_ALTER_USER_REMOVE_READ_TABLE      0xE
-#define TSDB_ALTER_USER_ADD_WRITE_TABLE        0xF
-#define TSDB_ALTER_USER_REMOVE_WRITE_TABLE     0x10
-#define TSDB_ALTER_USER_ADD_ALL_TABLE          0x11
-#define TSDB_ALTER_USER_REMOVE_ALL_TABLE       0x12
-#define TSDB_ALTER_USER_ADD_WHITE_LIST         0x13
-#define TSDB_ALTER_USER_DROP_WHITE_LIST        0x14
-
-#define TSDB_ALTER_USER_PRIVILEGES 0x2
 
 #define TSDB_KILL_MSG_LEN 30
 
@@ -251,6 +240,7 @@ typedef enum ENodeType {
   QUERY_NODE_CASE_WHEN,
   QUERY_NODE_EVENT_WINDOW,
   QUERY_NODE_HINT,
+  QUERY_NODE_VIEW,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR = 100,
@@ -333,6 +323,8 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
+  QUERY_NODE_SHOW_VIEWS_STMT,
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
   QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
   QUERY_NODE_SHOW_CREATE_TABLE_STMT,
   QUERY_NODE_SHOW_CREATE_STABLE_STMT,
@@ -355,6 +347,8 @@ typedef enum ENodeType {
   QUERY_NODE_RESTORE_VNODE_STMT,
   QUERY_NODE_PAUSE_STREAM_STMT,
   QUERY_NODE_RESUME_STREAM_STMT,  
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@@ -451,7 +445,7 @@ typedef struct SRetention {
   int8_t  keepUnit;
 } SRetention;
 
-#define RETENTION_VALID(r) (((r)->freq > 0) && ((r)->keep > 0))
+#define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0))
 
 #pragma pack(push, 1)
 
@@ -943,6 +937,7 @@ typedef struct {
   int8_t  superUser;
   int8_t  sysInfo;
   int8_t  enable;
+  int8_t  isView;
   char    user[TSDB_USER_LEN];
   char    pass[TSDB_USET_PASSWORD_LEN];
   char    objname[TSDB_DB_FNAME_LEN];  // db or topic
@@ -951,6 +946,7 @@ typedef struct {
   int32_t tagCondLen;
   int32_t     numIpRanges;
   SIpV4Range* pIpRanges;
+  int64_t     privileges;
   int32_t sqlLen;
   char*   sql;
 } SAlterUserReq;
@@ -979,6 +975,10 @@ typedef struct {
   SHashObj* writeDbs;
   SHashObj* readTbs;
   SHashObj* writeTbs;
+  SHashObj* alterTbs;
+  SHashObj* readViews;
+  SHashObj* writeViews;
+  SHashObj* alterViews;  
   SHashObj* useDbs;
   int64_t whiteListVer;
 } SGetUserAuthRsp;
@@ -1568,6 +1568,7 @@ typedef struct {
 typedef struct {
   int32_t id;
   int8_t  isMnode;
+  int8_t  offlineReason;
   SEp     ep;
   char    active[TSDB_ACTIVE_KEY_LEN];
   char    connActive[TSDB_CONN_ACTIVE_KEY_LEN];
@@ -1808,6 +1809,15 @@ int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp);
 int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp);
 void    tFreeSSTbHbRsp(SSTbHbRsp* pRsp);
 
+typedef struct {
+  SArray* pViewRsp;  // Array of SViewMetaRsp*;
+} SViewHbRsp;
+
+int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
+int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
+void    tFreeSViewHbRsp(SViewHbRsp* pRsp);
+
+
 typedef struct {
   int32_t numOfTables;
   int32_t numOfVgroup;
@@ -3880,6 +3890,58 @@ typedef struct {
   };
 } SPackedData;
 
+typedef struct {
+  char     fullname[TSDB_VIEW_FNAME_LEN];
+  char     name[TSDB_VIEW_NAME_LEN];
+  char     dbFName[TSDB_DB_FNAME_LEN];
+  char*    querySql;
+  char*    sql;
+  int8_t   orReplace;
+  int8_t   precision;
+  int32_t  numOfCols;
+  SSchema* pSchema;
+} SCMCreateViewReq;
+
+int32_t tSerializeSCMCreateViewReq(void* buf, int32_t bufLen, const SCMCreateViewReq* pReq);
+int32_t tDeserializeSCMCreateViewReq(void* buf, int32_t bufLen, SCMCreateViewReq* pReq);
+void    tFreeSCMCreateViewReq(SCMCreateViewReq* pReq);
+
+typedef struct {
+  char   fullname[TSDB_VIEW_FNAME_LEN];
+  char   name[TSDB_VIEW_NAME_LEN];
+  char   dbFName[TSDB_DB_FNAME_LEN];
+  char*  sql;
+  int8_t igNotExists;
+} SCMDropViewReq;
+
+int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq);
+int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq);
+void    tFreeSCMDropViewReq(SCMDropViewReq* pReq);
+
+typedef struct {
+  char   fullname[TSDB_VIEW_FNAME_LEN];
+} SViewMetaReq;
+int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq);
+int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq);
+
+typedef struct {
+  char     name[TSDB_VIEW_NAME_LEN];
+  char     dbFName[TSDB_DB_FNAME_LEN];
+  char*    user;
+  uint64_t dbId;
+  uint64_t viewId;
+  char*    querySql;
+  int8_t   precision;
+  int8_t   type;
+  int32_t  version;
+  int32_t  numOfCols;
+  SSchema* pSchema;
+} SViewMetaRsp;
+int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp);
+int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
+void    tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
+
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -189,6 +189,9 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)

include/common/ttime.h

@@ -90,6 +90,34 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
 
 void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
 
+struct STm {
+  struct tm tm;
+  int64_t   fsec;  // in NANOSECOND
+};
+
+int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm);
+int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision);
+
+/// @brief convert a timestamp to a formatted string
+/// @param format the timestamp format, must null terminated
+/// @param [in,out] formats the formats array pointer generated. Shouldn't be NULL.
+/// If (*formats == NULL), [format] will be used and [formats] will be updated to the new generated
+/// formats array; If not NULL, [formats] will be used instead of [format] to skip parse formats again.
+/// @param out output buffer, should be initialized by memset
+/// @notes remember to free the generated formats
+void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen);
+/// @brief convert a formatted timestamp string to a timestamp
+/// @param format must null terminated
+/// @param [in, out] formats, see taosTs2Char
+/// @param tsStr must null terminated
+/// @retval 0 for success, otherwise error occured
+/// @notes remember to free the generated formats even when error occured
+int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,
+                    int32_t errMsgLen);
+
+void    TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen);
+int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr);
+
 #ifdef __cplusplus
 }
 #endif

include/common/ttokendef.h

@@ -187,179 +187,183 @@
 #define TK_SUBSCRIPTIONS                  168
 #define TK_VNODES                         169
 #define TK_ALIVE                          170
-#define TK_NORMAL                         171
+#define TK_VIEWS                          171
-#define TK_CHILD                          172
+#define TK_VIEW                           172
-#define TK_LIKE                           173
+#define TK_NORMAL                         173
-#define TK_TBNAME                         174
+#define TK_CHILD                          174
-#define TK_QTAGS                          175
+#define TK_LIKE                           175
-#define TK_AS                             176
+#define TK_TBNAME                         176
-#define TK_SYSTEM                         177
+#define TK_QTAGS                          177
-#define TK_INDEX                          178
+#define TK_AS                             178
-#define TK_FUNCTION                       179
+#define TK_SYSTEM                         179
-#define TK_INTERVAL                       180
+#define TK_INDEX                          180
-#define TK_COUNT                          181
+#define TK_FUNCTION                       181
-#define TK_LAST_ROW                       182
+#define TK_INTERVAL                       182
-#define TK_META                           183
+#define TK_COUNT                          183
-#define TK_ONLY                           184
+#define TK_LAST_ROW                       184
-#define TK_TOPIC                          185
+#define TK_META                           185
-#define TK_CONSUMER                       186
+#define TK_ONLY                           186
-#define TK_GROUP                          187
+#define TK_TOPIC                          187
-#define TK_DESC                           188
+#define TK_CONSUMER                       188
-#define TK_DESCRIBE                       189
+#define TK_GROUP                          189
-#define TK_RESET                          190
+#define TK_DESC                           190
-#define TK_QUERY                          191
+#define TK_DESCRIBE                       191
-#define TK_CACHE                          192
+#define TK_RESET                          192
-#define TK_EXPLAIN                        193
+#define TK_QUERY                          193
-#define TK_ANALYZE                        194
+#define TK_CACHE                          194
-#define TK_VERBOSE                        195
+#define TK_EXPLAIN                        195
-#define TK_NK_BOOL                        196
+#define TK_ANALYZE                        196
-#define TK_RATIO                          197
+#define TK_VERBOSE                        197
-#define TK_NK_FLOAT                       198
+#define TK_NK_BOOL                        198
-#define TK_OUTPUTTYPE                     199
+#define TK_RATIO                          199
-#define TK_AGGREGATE                      200
+#define TK_NK_FLOAT                       200
-#define TK_BUFSIZE                        201
+#define TK_OUTPUTTYPE                     201
-#define TK_LANGUAGE                       202
+#define TK_AGGREGATE                      202
-#define TK_REPLACE                        203
+#define TK_BUFSIZE                        203
-#define TK_STREAM                         204
+#define TK_LANGUAGE                       204
-#define TK_INTO                           205
+#define TK_REPLACE                        205
-#define TK_PAUSE                          206
+#define TK_STREAM                         206
-#define TK_RESUME                         207
+#define TK_INTO                           207
-#define TK_TRIGGER                        208
+#define TK_PAUSE                          208
-#define TK_AT_ONCE                        209
+#define TK_RESUME                         209
-#define TK_WINDOW_CLOSE                   210
+#define TK_TRIGGER                        210
-#define TK_IGNORE                         211
+#define TK_AT_ONCE                        211
-#define TK_EXPIRED                        212
+#define TK_WINDOW_CLOSE                   212
-#define TK_FILL_HISTORY                   213
+#define TK_IGNORE                         213
-#define TK_UPDATE                         214
+#define TK_EXPIRED                        214
-#define TK_SUBTABLE                       215
+#define TK_FILL_HISTORY                   215
-#define TK_UNTREATED                      216
+#define TK_UPDATE                         216
-#define TK_KILL                           217
+#define TK_SUBTABLE                       217
-#define TK_CONNECTION                     218
+#define TK_UNTREATED                      218
-#define TK_TRANSACTION                    219
+#define TK_KILL                           219
-#define TK_BALANCE                        220
+#define TK_CONNECTION                     220
-#define TK_VGROUP                         221
+#define TK_TRANSACTION                    221
-#define TK_LEADER                         222
+#define TK_BALANCE                        222
-#define TK_MERGE                          223
+#define TK_VGROUP                         223
-#define TK_REDISTRIBUTE                   224
+#define TK_LEADER                         224
-#define TK_SPLIT                          225
+#define TK_MERGE                          225
-#define TK_DELETE                         226
+#define TK_REDISTRIBUTE                   226
-#define TK_INSERT                         227
+#define TK_SPLIT                          227
-#define TK_NULL                           228
+#define TK_DELETE                         228
-#define TK_NK_QUESTION                    229
+#define TK_INSERT                         229
-#define TK_NK_ARROW                       230
+#define TK_NULL                           230
-#define TK_ROWTS                          231
+#define TK_NK_QUESTION                    231
-#define TK_QSTART                         232
+#define TK_NK_ALIAS                       232
-#define TK_QEND                           233
+#define TK_NK_ARROW                       233
-#define TK_QDURATION                      234
+#define TK_ROWTS                          234
-#define TK_WSTART                         235
+#define TK_QSTART                         235
-#define TK_WEND                           236
+#define TK_QEND                           236
-#define TK_WDURATION                      237
+#define TK_QDURATION                      237
-#define TK_IROWTS                         238
+#define TK_WSTART                         238
-#define TK_ISFILLED                       239
+#define TK_WEND                           239
-#define TK_CAST                           240
+#define TK_WDURATION                      240
-#define TK_NOW                            241
+#define TK_IROWTS                         241
-#define TK_TODAY                          242
+#define TK_ISFILLED                       242
-#define TK_TIMEZONE                       243
+#define TK_CAST                           243
-#define TK_CLIENT_VERSION                 244
+#define TK_NOW                            244
-#define TK_SERVER_VERSION                 245
+#define TK_TODAY                          245
-#define TK_SERVER_STATUS                  246
+#define TK_TIMEZONE                       246
-#define TK_CURRENT_USER                   247
+#define TK_CLIENT_VERSION                 247
-#define TK_CASE                           248
+#define TK_SERVER_VERSION                 248
-#define TK_WHEN                           249
+#define TK_SERVER_STATUS                  249
-#define TK_THEN                           250
+#define TK_CURRENT_USER                   250
-#define TK_ELSE                           251
+#define TK_CASE                           251
-#define TK_BETWEEN                        252
+#define TK_WHEN                           252
-#define TK_IS                             253
+#define TK_THEN                           253
-#define TK_NK_LT                          254
+#define TK_ELSE                           254
-#define TK_NK_GT                          255
+#define TK_BETWEEN                        255
-#define TK_NK_LE                          256
+#define TK_IS                             256
-#define TK_NK_GE                          257
+#define TK_NK_LT                          257
-#define TK_NK_NE                          258
+#define TK_NK_GT                          258
-#define TK_MATCH                          259
+#define TK_NK_LE                          259
-#define TK_NMATCH                         260
+#define TK_NK_GE                          260
-#define TK_CONTAINS                       261
+#define TK_NK_NE                          261
-#define TK_IN                             262
+#define TK_MATCH                          262
-#define TK_JOIN                           263
+#define TK_NMATCH                         263
-#define TK_INNER                          264
+#define TK_CONTAINS                       264
-#define TK_SELECT                         265
+#define TK_IN                             265
-#define TK_NK_HINT                        266
+#define TK_JOIN                           266
-#define TK_DISTINCT                       267
+#define TK_INNER                          267
-#define TK_WHERE                          268
+#define TK_SELECT                         268
-#define TK_PARTITION                      269
+#define TK_NK_HINT                        269
-#define TK_BY                             270
+#define TK_DISTINCT                       270
-#define TK_SESSION                        271
+#define TK_WHERE                          271
-#define TK_STATE_WINDOW                   272
+#define TK_PARTITION                      272
-#define TK_EVENT_WINDOW                   273
+#define TK_BY                             273
-#define TK_SLIDING                        274
+#define TK_SESSION                        274
-#define TK_FILL                           275
+#define TK_STATE_WINDOW                   275
-#define TK_VALUE                          276
+#define TK_EVENT_WINDOW                   276
-#define TK_VALUE_F                        277
+#define TK_SLIDING                        277
-#define TK_NONE                           278
+#define TK_FILL                           278
-#define TK_PREV                           279
+#define TK_VALUE                          279
-#define TK_NULL_F                         280
+#define TK_VALUE_F                        280
-#define TK_LINEAR                         281
+#define TK_NONE                           281
-#define TK_NEXT                           282
+#define TK_PREV                           282
-#define TK_HAVING                         283
+#define TK_NULL_F                         283
-#define TK_RANGE                          284
+#define TK_LINEAR                         284
-#define TK_EVERY                          285
+#define TK_NEXT                           285
-#define TK_ORDER                          286
+#define TK_HAVING                         286
-#define TK_SLIMIT                         287
+#define TK_RANGE                          287
-#define TK_SOFFSET                        288
+#define TK_EVERY                          288
-#define TK_LIMIT                          289
+#define TK_ORDER                          289
-#define TK_OFFSET                         290
+#define TK_SLIMIT                         290
-#define TK_ASC                            291
+#define TK_SOFFSET                        291
-#define TK_NULLS                          292
+#define TK_LIMIT                          292
-#define TK_ABORT                          293
+#define TK_OFFSET                         293
-#define TK_AFTER                          294
+#define TK_ASC                            294
-#define TK_ATTACH                         295
+#define TK_NULLS                          295
-#define TK_BEFORE                         296
+#define TK_ABORT                          296
-#define TK_BEGIN                          297
+#define TK_AFTER                          297
-#define TK_BITAND                         298
+#define TK_ATTACH                         298
-#define TK_BITNOT                         299
+#define TK_BEFORE                         299
-#define TK_BITOR                          300
+#define TK_BEGIN                          300
-#define TK_BLOCKS                         301
+#define TK_BITAND                         301
-#define TK_CHANGE                         302
+#define TK_BITNOT                         302
-#define TK_COMMA                          303
+#define TK_BITOR                          303
-#define TK_CONCAT                         304
+#define TK_BLOCKS                         304
-#define TK_CONFLICT                       305
+#define TK_CHANGE                         305
-#define TK_COPY                           306
+#define TK_COMMA                          306
-#define TK_DEFERRED                       307
+#define TK_CONCAT                         307
-#define TK_DELIMITERS                     308
+#define TK_CONFLICT                       308
-#define TK_DETACH                         309
+#define TK_COPY                           309
-#define TK_DIVIDE                         310
+#define TK_DEFERRED                       310
-#define TK_DOT                            311
+#define TK_DELIMITERS                     311
-#define TK_EACH                           312
+#define TK_DETACH                         312
-#define TK_FAIL                           313
+#define TK_DIVIDE                         313
-#define TK_FILE                           314
+#define TK_DOT                            314
-#define TK_FOR                            315
+#define TK_EACH                           315
-#define TK_GLOB                           316
+#define TK_FAIL                           316
-#define TK_ID                             317
+#define TK_FILE                           317
-#define TK_IMMEDIATE                      318
+#define TK_FOR                            318
-#define TK_IMPORT                         319
+#define TK_GLOB                           319
-#define TK_INITIALLY                      320
+#define TK_ID                             320
-#define TK_INSTEAD                        321
+#define TK_IMMEDIATE                      321
-#define TK_ISNULL                         322
+#define TK_IMPORT                         322
-#define TK_KEY                            323
+#define TK_INITIALLY                      323
-#define TK_MODULES                        324
+#define TK_INSTEAD                        324
-#define TK_NK_BITNOT                      325
+#define TK_ISNULL                         325
-#define TK_NK_SEMI                        326
+#define TK_KEY                            326
-#define TK_NOTNULL                        327
+#define TK_MODULES                        327
-#define TK_OF                             328
+#define TK_NK_BITNOT                      328
-#define TK_PLUS                           329
+#define TK_NK_SEMI                        329
-#define TK_PRIVILEGE                      330
+#define TK_NOTNULL                        330
-#define TK_RAISE                          331
+#define TK_OF                             331
-#define TK_RESTRICT                       332
+#define TK_PLUS                           332
-#define TK_ROW                            333
+#define TK_PRIVILEGE                      333
-#define TK_SEMI                           334
+#define TK_RAISE                          334
-#define TK_STAR                           335
+#define TK_RESTRICT                       335
-#define TK_STATEMENT                      336
+#define TK_ROW                            336
-#define TK_STRICT                         337
+#define TK_SEMI                           337
-#define TK_STRING                         338
+#define TK_STAR                           338
-#define TK_TIMES                          339
+#define TK_STATEMENT                      339
-#define TK_VALUES                         340
+#define TK_STRICT                         340
-#define TK_VARIABLE                       341
+#define TK_STRING                         341
-#define TK_VIEW                           342
+#define TK_TIMES                          342
-#define TK_WAL                            343
+#define TK_VALUES                         343
+#define TK_VARIABLE                       344
+#define TK_WAL                            345
+
+
 
 
 

include/libs/catalog/catalog.h

@@ -37,13 +37,16 @@ enum {
   CTG_DBG_DB_NUM = 1,
   CTG_DBG_META_NUM,
   CTG_DBG_STB_NUM,
+  CTG_DBG_VIEW_NUM,
   CTG_DBG_DB_RENT_NUM,
   CTG_DBG_STB_RENT_NUM,
+  CTG_DBG_VIEW_RENT_NUM,
 };
 
 typedef enum {
   AUTH_TYPE_READ = 1,
   AUTH_TYPE_WRITE,
+  AUTH_TYPE_ALTER,
   AUTH_TYPE_OTHER,
   AUTH_TYPE_READ_OR_WRITE,
 } AUTH_TYPE;
@@ -51,12 +54,19 @@ typedef enum {
 typedef struct SUserAuthInfo {
   char      user[TSDB_USER_LEN];
   SName     tbName;
+  bool      isView;
   AUTH_TYPE type;
 } SUserAuthInfo;
 
+typedef enum {
+  AUTH_RES_BASIC = 0,
+  AUTH_RES_VIEW,
+  AUTH_RES_MAX_VALUE
+} AUTH_RES_TYPE;
+
 typedef struct SUserAuthRes {
-  bool   pass;
+  bool   pass[AUTH_RES_MAX_VALUE];
-  SNode* pCond;
+  SNode* pCond[AUTH_RES_MAX_VALUE];
 } SUserAuthRes;
 
 typedef struct SDbInfo {
@@ -83,6 +93,7 @@ typedef struct SCatalogReq {
   SArray* pTableIndex;    // element is SNAME
   SArray* pTableCfg;      // element is SNAME
   SArray* pTableTag;      // element is SNAME
+  SArray* pView;          // element is STablesReq
   bool    qNodeRequired;  // valid qnode
   bool    dNodeRequired;  // valid dnode
   bool    svrVerRequired;
@@ -96,6 +107,7 @@ typedef struct SMetaRes {
 } SMetaRes;
 
 typedef struct SMetaData {
+  bool      ctgFree;      // need to freed by catalog module
   SArray*   pDbVgroup;    // pRes = SArray<SVgroupInfo>*
   SArray*   pDbCfg;       // pRes = SDbCfgInfo*
   SArray*   pDbInfo;      // pRes = SDbInfo*
@@ -109,21 +121,24 @@ typedef struct SMetaData {
   SArray*   pTableCfg;    // pRes = STableCfg*
   SArray*   pTableTag;    // pRes = SArray<STagVal>*
   SArray*   pDnodeList;   // pRes = SArray<SEpSet>*
+  SArray*   pView;        // pRes = SViewMeta*
   SMetaRes* pSvrVer;      // pRes = char*
 } SMetaData;
 
 typedef struct SCatalogCfg {
   uint32_t maxTblCacheNum;
+  uint32_t maxViewCacheNum;
   uint32_t maxDBCacheNum;
   uint32_t maxUserCacheNum;
   uint32_t dbRentSec;
   uint32_t stbRentSec;
+  uint32_t viewRentSec;
 } SCatalogCfg;
 
 typedef struct SSTableVersion {
   char     dbFName[TSDB_DB_FNAME_LEN];
   char     stbName[TSDB_TABLE_NAME_LEN];
-  uint64_t dbId;
+  int64_t  dbId;
   uint64_t suid;
   int32_t  sversion;
   int32_t  tversion;
@@ -139,6 +154,20 @@ typedef struct SDbCacheInfo {
   int64_t stateTs;
 } SDbCacheInfo;
 
+typedef struct SDynViewVersion {
+  int64_t  svrBootTs;
+  uint64_t dynViewVer;
+} SDynViewVersion;
+
+typedef struct SViewVersion {
+  char     dbFName[TSDB_DB_FNAME_LEN];
+  char     viewName[TSDB_VIEW_NAME_LEN];
+  int64_t  dbId;
+  uint64_t viewId;
+  int32_t  version;
+} SViewVersion;
+
+
 typedef struct STbSVersion {
   char*   tbFName;
   int32_t sver;
@@ -307,6 +336,8 @@ int32_t catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray*
 
 int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, uint32_t* num);
 
+int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion);
+
 int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num);
 
 int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num);
@@ -343,6 +374,16 @@ SMetaData* catalogCloneMetaData(SMetaData* pData);
 
 void catalogFreeMetaData(SMetaData* pData);
 
+int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* viewName, uint64_t viewId);
+
+int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer);
+
+int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg);
+
+int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg);
+
+int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pViewName, STableMeta** pTableMeta);
+
 int32_t ctgdEnableDebug(char* option, bool enable);
 
 int32_t ctgdHandleDbgCommand(char* command);

include/libs/function/functionMgt.h

@@ -94,6 +94,8 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_TO_ISO8601,
   FUNCTION_TYPE_TO_UNIXTIMESTAMP,
   FUNCTION_TYPE_TO_JSON,
+  FUNCTION_TYPE_TO_TIMESTAMP,
+  FUNCTION_TYPE_TO_CHAR,
 
   // date and time function
   FUNCTION_TYPE_NOW = 2500,

include/libs/nodes/cmdnodes.h

@@ -26,7 +26,7 @@ extern "C" {
 #define DESCRIBE_RESULT_COLS      4
 #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
 #define DESCRIBE_RESULT_TYPE_LEN  (20 + VARSTR_HEADER_SIZE)
-#define DESCRIBE_RESULT_NOTE_LEN  (8 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_NOTE_LEN  (16 + VARSTR_HEADER_SIZE)
 
 #define SHOW_CREATE_DB_RESULT_COLS       2
 #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
@@ -36,6 +36,11 @@ extern "C" {
 #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN * 3)
 
+#define SHOW_CREATE_VIEW_RESULT_COLS       2
+#define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE)
+#define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE)
+
+
 #define SHOW_LOCAL_VARIABLES_RESULT_COLS       3
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
@@ -51,6 +56,7 @@ extern "C" {
 #define PRIVILEGE_TYPE_READ      BIT_FLAG_MASK(1)
 #define PRIVILEGE_TYPE_WRITE     BIT_FLAG_MASK(2)
 #define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3)
+#define PRIVILEGE_TYPE_ALTER     BIT_FLAG_MASK(4)
 
 typedef struct SDatabaseOptions {
   ENodeType   type;
@@ -297,6 +303,13 @@ typedef struct SShowCreateTableStmt {
   void*     pTableCfg;  // STableCfg
 } SShowCreateTableStmt;
 
+typedef struct SShowCreateViewStmt {
+  ENodeType type;
+  char      dbName[TSDB_DB_NAME_LEN];
+  char      viewName[TSDB_VIEW_NAME_LEN];
+  void*     pViewMeta;
+} SShowCreateViewStmt;
+
 typedef struct SShowTableDistributedStmt {
   ENodeType type;
   char      dbName[TSDB_DB_NAME_LEN];
@@ -490,6 +503,23 @@ typedef struct SDropFunctionStmt {
   bool      ignoreNotExists;
 } SDropFunctionStmt;
 
+typedef struct SCreateViewStmt {
+  ENodeType           type;
+  char                dbName[TSDB_DB_NAME_LEN];
+  char                viewName[TSDB_VIEW_NAME_LEN];
+  char*               pQuerySql;
+  bool                orReplace;
+  SNode*              pQuery;
+  SCMCreateViewReq    createReq;
+} SCreateViewStmt;
+
+typedef struct SDropViewStmt {
+  ENodeType  type;
+  char       dbName[TSDB_DB_NAME_LEN];
+  char       viewName[TSDB_VIEW_NAME_LEN];
+  bool       ignoreNotExists;
+} SDropViewStmt;
+
 typedef struct SGrantStmt {
   ENodeType type;
   char      userName[TSDB_USER_LEN];

include/libs/nodes/querynodes.h

@@ -30,6 +30,11 @@ extern "C" {
 #define VGROUPS_INFO_SIZE(pInfo) \
   (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
 
+typedef struct SAssociationNode {
+  SNode** pPlace;
+  SNode*  pAssociationNode;
+} SAssociationNode;
+
 typedef struct SRawExprNode {
   ENodeType nodeType;
   char*     p;
@@ -182,6 +187,16 @@ typedef struct STempTableNode {
   SNode*     pSubquery;
 } STempTableNode;
 
+typedef struct SViewNode {
+  STableNode         table;  // QUERY_NODE_REAL_TABLE
+  struct STableMeta* pMeta;
+  SVgroupsInfo*      pVgroupList;
+  char               qualDbName[TSDB_DB_NAME_LEN];  // SHOW qualDbName.TABLES
+  double             ratio;
+  SArray*            pSmaIndexes;
+  int8_t             cacheLastMode;
+} SViewNode;
+
 typedef enum EJoinType { 
   JOIN_TYPE_INNER = 1,
   JOIN_TYPE_LEFT,
@@ -413,7 +428,8 @@ typedef struct SVgDataBlocks {
 
 typedef void (*FFreeTableBlockHash)(SHashObj*);
 typedef void (*FFreeVgourpBlockArray)(SArray*);
-
+struct SStbRowsDataContext;
+typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*);
 typedef struct SVnodeModifyOpStmt {
   ENodeType             nodeType;
   ENodeType             sqlNodeType;
@@ -428,11 +444,11 @@ typedef struct SVnodeModifyOpStmt {
   struct STableMeta*    pTableMeta;
   SNode*                pTagCond;
   SArray*               pTableTag;
-  SHashObj*             pVgroupsHashObj;
+  SHashObj*             pVgroupsHashObj;     // SHashObj<vgId, SVgInfo>
   SHashObj*             pTableBlockHashObj;  // SHashObj<tuid, STableDataCxt*>
-  SHashObj*             pSubTableHashObj;
+  SHashObj*             pSubTableHashObj;    // SHashObj<table_name, STableMeta*>
-  SHashObj*             pTableNameHashObj;
+  SHashObj*             pTableNameHashObj;   // set of table names for refreshing meta, sync mode
-  SHashObj*             pDbFNameHashObj;
+  SHashObj*             pDbFNameHashObj;     // set of db names for refreshing meta, sync mode
   SArray*               pVgDataBlocks;  // SArray<SVgroupDataCxt*>
   SVCreateTbReq*        pCreateTblReq;
   TdFilePtr             fp;
@@ -440,6 +456,10 @@ typedef struct SVnodeModifyOpStmt {
   FFreeVgourpBlockArray freeArrayFunc;
   bool                  usingTableProcessing;
   bool                  fileProcessing;
+
+  bool                  stbSyntax;
+  struct SStbRowsDataContext*  pStbRowsCxt;
+  FFreeStbRowsDataContext     freeStbRowsCxtFunc;
 } SVnodeModifyOpStmt;
 
 typedef struct SExplainOptions {

include/libs/parser/parser.h

@@ -22,9 +22,7 @@ extern "C" {
 
 #include "query.h"
 #include "querynodes.h"
-
+#include "catalog.h"
-struct SCatalogReq;
-struct SMetaData;
 
 typedef struct SStmtCallback {
   TAOS_STMT* pStmt;
@@ -33,6 +31,33 @@ typedef struct SStmtCallback {
   int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
 } SStmtCallback;
 
+typedef enum {
+  PARSE_SQL_RES_QUERY = 1,
+  PARSE_SQL_RES_SCHEMA,
+} SParseResType;
+
+typedef struct SParseSchemaRes {
+  int8_t        precision;
+  int32_t       numOfCols;
+  SSchema*      pSchema;
+} SParseSchemaRes;
+
+typedef struct SParseQueryRes {
+  SNode*              pQuery;
+  SCatalogReq*        pCatalogReq;
+  SMetaData           meta;
+} SParseQueryRes;
+
+typedef struct SParseSqlRes {
+  SParseResType resType;
+  union {
+    SParseSchemaRes schemaRes;
+    SParseQueryRes  queryRes;
+  };
+} SParseSqlRes;
+
+typedef int32_t (*parseSqlFn)(void*, const char*, const char*, bool, const char*, SParseSqlRes*);
+
 typedef struct SParseCsvCxt {
   TdFilePtr   fp;           // last parsed file
   int32_t     tableNo;      // last parsed table
@@ -55,6 +80,8 @@ typedef struct SParseContext {
   struct SCatalog* pCatalog;
   SStmtCallback*   pStmtCb;
   const char*      pUser;
+  const char*      pEffectiveUser;
+  bool             parseOnly;
   bool             isSuperUser;
   bool             enableSysInfo;
   bool             async;
@@ -64,7 +91,10 @@ typedef struct SParseContext {
   SArray*          pTableMetaPos;    // sql table pos => catalog data pos
   SArray*          pTableVgroupPos;  // sql table pos => catalog data pos
   int64_t          allocatorId;
+  parseSqlFn       parseSqlFp;
+  void*            parseSqlParam;
   int8_t           biMode;
+  SArray*          pSubMetaList;
 } SParseContext;
 
 int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@@ -125,6 +155,8 @@ int     rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCr
 int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
 SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
 SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
+void    destoryCatalogReq(SCatalogReq *pCatalogReq);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/qcom/query.h

@@ -119,6 +119,17 @@ typedef struct STableMeta {
 } STableMeta;
 #pragma pack(pop)
 
+typedef struct SViewMeta {
+  uint64_t viewId;
+  char*    user;
+  char*    querySql;
+  int8_t   precision;
+  int8_t   type;
+  int32_t  version;
+  int32_t  numOfCols;
+  SSchema* pSchema;
+} SViewMeta;
+
 typedef struct SDBVgInfo {
   int32_t   vgVersion;
   int16_t   hashPrefix;
@@ -148,6 +159,15 @@ typedef struct STableMetaOutput {
   STableMeta* tbMeta;
 } STableMetaOutput;
 
+typedef struct SViewMetaOutput {
+  char     name[TSDB_VIEW_NAME_LEN];
+  char     dbFName[TSDB_DB_FNAME_LEN];
+  char*    querySql;
+  int8_t   precision;
+  int32_t  numOfCols;
+  SSchema* pSchema;
+} SViewMetaOutput;
+
 typedef struct SDataBuf {
   int32_t  msgType;
   void*    pData;
@@ -300,9 +320,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
   (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
    SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
 
+#define IS_VIEW_REQUEST(_type) ((_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
+
 #define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
   ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
-   (_type) == TDMT_MND_DROP_STB)
+   (_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
 
 #define NEED_SCHEDULER_REDIRECT_ERROR(_code)                                              \
   (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \

include/libs/scalar/scalar.h

@@ -80,6 +80,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/libs/stream/tstream.h

@@ -38,16 +38,25 @@ extern "C" {
 #define TASK_DOWNSTREAM_READY       0x0
 #define TASK_DOWNSTREAM_NOT_READY   0x1
 #define TASK_DOWNSTREAM_NOT_LEADER  0x2
-#define TASK_SELF_NEW_STAGE         0x3
+#define TASK_UPSTREAM_NEW_STAGE     0x3
 
 #define NODE_ROLE_UNINIT     0x1
 #define NODE_ROLE_LEADER     0x2
 #define NODE_ROLE_FOLLOWER   0x3
 
+#define HAS_RELATED_FILLHISTORY_TASK(_t)  ((_t)->hTaskInfo.id.taskId != 0)
+#define CLEAR_RELATED_FILLHISTORY_TASK(_t) \
+  do {                                     \
+    (_t)->hTaskInfo.id.taskId = 0;         \
+    (_t)->hTaskInfo.id.streamId = 0;       \
+  } while (0)
+
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
+typedef struct SStreamTaskSM SStreamTaskSM;
 
 #define SSTREAM_TASK_VER 2
+
 enum {
   STREAM_STATUS__NORMAL = 0,
   STREAM_STATUS__STOP,
@@ -58,7 +67,7 @@ enum {
 };
 
 typedef enum ETaskStatus {
-  TASK_STATUS__NORMAL = 0,
+  TASK_STATUS__READY = 0,
   TASK_STATUS__DROPPING,
   TASK_STATUS__UNINIT,  // not used, an placeholder
   TASK_STATUS__STOP,
@@ -66,6 +75,7 @@ typedef enum ETaskStatus {
   TASK_STATUS__HALT,          // pause, but not be manipulated by user command
   TASK_STATUS__PAUSE,         // pause
   TASK_STATUS__CK,            // stream task is in checkpoint status, no data are allowed to put into inputQ anymore
+  TASK_STATUS__STREAM_SCAN_HISTORY,
 } ETaskStatus;
 
 enum {
@@ -118,6 +128,22 @@ enum {
   STREAM_META_OK_TO_STOP = 2,
 };
 
+typedef enum EStreamTaskEvent {
+  TASK_EVENT_INIT = 0x1,
+  TASK_EVENT_INIT_SCANHIST = 0x2,
+  TASK_EVENT_INIT_STREAM_SCANHIST = 0x3,
+  TASK_EVENT_SCANHIST_DONE = 0x4,
+  TASK_EVENT_STOP = 0x5,
+  TASK_EVENT_GEN_CHECKPOINT = 0x6,
+  TASK_EVENT_CHECKPOINT_DONE = 0x7,
+  TASK_EVENT_PAUSE = 0x8,
+  TASK_EVENT_RESUME = 0x9,
+  TASK_EVENT_HALT = 0xA,
+  TASK_EVENT_DROPPING = 0xB,
+  TASK_EVENT_SCAN_TSDB = 0xC,
+  TASK_EVENT_SCAN_WAL = 0xD,
+} EStreamTaskEvent;
+
 typedef struct {
   int8_t type;
 } SStreamQueueItem;
@@ -155,11 +181,6 @@ typedef struct {
   SSDataBlock* pBlock;
 } SStreamRefDataBlock;
 
-typedef struct {
-  int8_t       type;
-  SSDataBlock* pBlock;
-} SStreamTrigger;
-
 typedef struct SStreamQueueNode SStreamQueueNode;
 
 struct SStreamQueueNode {
@@ -215,6 +236,11 @@ typedef struct {
   SUseDbRsp dbInfo;
 } STaskDispatcherShuffle;
 
+typedef struct {
+  int32_t nodeId;
+  SEpSet  epset;
+} SDownstreamTaskEpset;
+
 typedef struct {
   int64_t         stbUid;
   char            stbFullName[TSDB_TABLE_FNAME_LEN];
@@ -265,6 +291,7 @@ typedef struct SCheckpointInfo {
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
+  SStreamTaskSM* pSM;
   int8_t         taskStatus;
   int8_t         downstreamReady;  // downstream tasks are all ready now, if this flag is set
   int8_t         schedStatus;
@@ -305,15 +332,10 @@ typedef struct SDispatchMsgInfo {
   void*   pTimer; // used to dispatch data after a given time duration
 } SDispatchMsgInfo;
 
-typedef struct STaskOutputQueue {
+typedef struct STaskQueue {
   int8_t        status;
   SStreamQueue* queue;
-} STaskOutputQueue;
+} STaskQueue;
-
-typedef struct STaskInputInfo {
-  int8_t        status;
-  SStreamQueue* queue;
-} STaskInputInfo;
 
 typedef struct STaskSchedInfo {
   int8_t  status;
@@ -349,6 +371,7 @@ typedef struct SHistoryTaskInfo {
   int32_t tickCount;
   int32_t retryTimes;
   int32_t waitInterval;
+  int64_t haltVer;      // offset in wal when halt the stream task
 } SHistoryTaskInfo;
 
 typedef struct STaskOutputInfo {
@@ -361,6 +384,7 @@ typedef struct STaskOutputInfo {
   };
   int8_t        type;
   STokenBucket* pTokenBucket;
+  SArray*       pDownstreamUpdateList;
 } STaskOutputInfo;
 
 typedef struct SUpstreamInfo {
@@ -372,8 +396,8 @@ struct SStreamTask {
   int64_t             ver;
   SStreamTaskId       id;
   SSTaskBasicInfo     info;
-  STaskOutputQueue    outputq;
+  STaskQueue          outputq;
-  STaskInputInfo      inputInfo;
+  STaskQueue          inputq;
   STaskSchedInfo      schedInfo;
   STaskOutputInfo     outputInfo;
   SDispatchMsgInfo    msgInfo;
@@ -408,11 +432,16 @@ struct SStreamTask {
 typedef struct STaskStartInfo {
   int64_t   startTs;
   int64_t   readyTs;
-  int32_t   startedAfterNodeUpdate;
+  int32_t   startAllTasksFlag;
   SHashObj* pReadyTaskSet;           // tasks that are all ready for running stream processing
   int32_t   elapsedTime;
 } STaskStartInfo;
 
+typedef struct STaskUpdateInfo {
+  SHashObj* pTasks;
+  int32_t   transId;
+} STaskUpdateInfo;
+
 // meta
 typedef struct SStreamMeta {
   char*           path;
@@ -435,7 +464,7 @@ typedef struct SStreamMeta {
   SHashObj*       pTaskBackendUnique;
   TdThreadMutex   backendMutex;
   SMetaHbInfo*    pHbInfo;
-  SHashObj*     pUpdateTaskSet;
+  STaskUpdateInfo updateInfo;
   int32_t         numOfStreamTasks;  // this value should be increased when a new task is added into the meta
   int32_t         numOfPausedTasks;
   int32_t         chkptNotReadyTasks;
@@ -617,7 +646,8 @@ typedef struct STaskStatusEntry {
 typedef struct SStreamHbMsg {
   int32_t vgId;
   int32_t numOfTasks;
-  SArray* pTaskStatus;  // SArray<SStreamTaskStatusEntry>
+  SArray* pTaskStatus;  // SArray<STaskStatusEntry>
+  SArray* pUpdateNodes; // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
 } SStreamHbMsg;
 
 int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
@@ -641,6 +671,7 @@ typedef struct SNodeUpdateInfo {
 } SNodeUpdateInfo;
 
 typedef struct SStreamTaskNodeUpdateMsg {
+  int32_t transId;   // to identify the msg
   int64_t streamId;
   int32_t taskId;
   SArray* pNodeList;  // SArray<SNodeUpdateInfo>
@@ -686,24 +717,36 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t
 void    streamTaskInputFail(SStreamTask* pTask);
 int32_t streamExecTask(SStreamTask* pTask);
 int32_t streamSchedExec(SStreamTask* pTask);
-bool    streamTaskShouldStop(const SStreamStatus* pStatus);
+bool    streamTaskShouldStop(const SStreamTask* pStatus);
-bool    streamTaskShouldPause(const SStreamStatus* pStatus);
+bool    streamTaskShouldPause(const SStreamTask* pStatus);
 bool    streamTaskIsIdle(const SStreamTask* pTask);
+bool    streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
+
+char*       createStreamTaskIdStr(int64_t streamId, int32_t taskId);
+ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr);
+const char* streamTaskGetStatusStr(ETaskStatus status);
+void        streamTaskResetStatus(SStreamTask* pTask);
+void        streamTaskSetStatusReady(SStreamTask* pTask);
 
 void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
 
-char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
-
 // recover and fill history
 void    streamTaskCheckDownstream(SStreamTask* pTask);
-int32_t streamTaskStartScanHistory(SStreamTask* pTask);
+int32_t onNormalTaskReady(SStreamTask* pTask);
+int32_t onScanhistoryTaskReady(SStreamTask* pTask);
+
 int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
 void    streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
 bool    streamTaskAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
-int8_t  streamTaskSetSchedStatusInActive(SStreamTask* pTask);
+int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask);
+
+int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
+void    streamTaskRestoreStatus(SStreamTask* pTask);
 
 int32_t streamTaskStop(SStreamTask* pTask);
 int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
@@ -714,17 +757,11 @@ int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask);
 int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
 bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
-int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize);
 
 // common
 int32_t     streamRestoreParam(SStreamTask* pTask);
-int32_t     streamSetStatusNormal(SStreamTask* pTask);
-int32_t     streamSetStatusUnint(SStreamTask* pTask);
-const char* streamGetTaskStatusStr(int32_t status);
 void        streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
-void        streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta);
+void        streamTaskResume(SStreamTask* pTask);
-void        streamTaskResumeFromHalt(SStreamTask* pTask);
-void        streamTaskDisablePause(SStreamTask* pTask);
 void        streamTaskEnablePause(SStreamTask* pTask);
 int32_t     streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
 void        streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
@@ -759,7 +796,6 @@ int32_t      streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
 int32_t      streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
 int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
-int32_t      streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaReopen(SStreamMeta* pMeta);
@@ -768,12 +804,17 @@ int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 void         streamMetaStartHb(SStreamMeta* pMeta);
 void         streamMetaInitForSnode(SStreamMeta* pMeta);
+bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
+int32_t      streamMetaUpdateTaskReadyInfo(SStreamTask* pTask);
+void         streamMetaRLock(SStreamMeta* pMeta);
+void         streamMetaRUnLock(SStreamMeta* pMeta);
+void         streamMetaWLock(SStreamMeta* pMeta);
+void         streamMetaWUnLock(SStreamMeta* pMeta);
 
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask);
-
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,

include/libs/sync/sync.h

@@ -46,6 +46,7 @@ extern "C" {
 #define SYNC_HEARTBEAT_SLOW_MS       1500
 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
 #define SYNC_SNAP_RESEND_MS          1000 * 60
+#define SYNC_SNAP_TIMEOUT_MS         1000 * 600
 
 #define SYNC_VND_COMMIT_MIN_MS 3000
 

include/util/taoserror.h

@@ -332,6 +332,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_REPLICA           TAOS_DEF_ERROR_CODE(0, 0x03B7)
 #define TSDB_CODE_MND_DNODE_IN_CREATING         TAOS_DEF_ERROR_CODE(0, 0x03B8)
 #define TSDB_CODE_MND_DNODE_IN_DROPPING         TAOS_DEF_ERROR_CODE(0, 0x03B9)
+#define TSDB_CODE_MND_NO_ENOUGH_VNODES          TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-stable-part2
 #define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC  TAOS_DEF_ERROR_CODE(0, 0x03C0)
@@ -383,15 +384,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_STREAMS          TAOS_DEF_ERROR_CODE(0, 0x03F6)
 #define TSDB_CODE_MND_INVALID_TARGET_TABLE      TAOS_DEF_ERROR_CODE(0, 0x03F7)
 
-// mnode-sma
-#define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0480)
-#define TSDB_CODE_MND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0481)
-#define TSDB_CODE_MND_INVALID_SMA_OPTION        TAOS_DEF_ERROR_CODE(0, 0x0482)
 
-// mnode-tag-indxe
-
-#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST   TAOS_DEF_ERROR_CODE(0, 0x0483)
-#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0484)
 
 // dnode
 // #define TSDB_CODE_DND_MSG_NOT_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x
@@ -418,6 +411,22 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MNODE_NO_NEED_RESTORE         TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
 #define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE   TAOS_DEF_ERROR_CODE(0, 0x0416)
 
+// mnode-sma
+#define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0480)
+#define TSDB_CODE_MND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0481)
+#define TSDB_CODE_MND_INVALID_SMA_OPTION        TAOS_DEF_ERROR_CODE(0, 0x0482)
+
+// mnode-tag-indxe
+
+#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST   TAOS_DEF_ERROR_CODE(0, 0x0483)
+#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0484)
+
+
+// mnode-view
+#define TSDB_CODE_MND_VIEW_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x04A0)
+#define TSDB_CODE_MND_VIEW_NOT_EXIST            TAOS_DEF_ERROR_CODE(0, 0x04A1)
+
+
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
 // #define TSDB_CODE_VND_MSG_NOT_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x
@@ -657,6 +666,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 #define TSDB_CODE_PAR_INVALID_DB_OPTION         TAOS_DEF_ERROR_CODE(0, 0x2619)
 #define TSDB_CODE_PAR_INVALID_TABLE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x261A)
+#define TSDB_CODE_PAR_INTER_VALUE_TOO_BIG       TAOS_DEF_ERROR_CODE(0, 0x261B)
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 #define TSDB_CODE_PAR_AGG_FUNC_NESTING          TAOS_DEF_ERROR_CODE(0, 0x2627)
 #define TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE    TAOS_DEF_ERROR_CODE(0, 0x2628)
@@ -724,6 +734,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED      TAOS_DEF_ERROR_CODE(0, 0x2669)
 #define TSDB_CODE_PAR_INVALID_VARBINARY         TAOS_DEF_ERROR_CODE(0, 0x266A)
 #define TSDB_CODE_PAR_INVALID_IP_RANGE          TAOS_DEF_ERROR_CODE(0, 0x266B)
+#define TSDB_CODE_PAR_INVALID_VIEW_QUERY        TAOS_DEF_ERROR_CODE(0, 0x266C)
+#define TSDB_CODE_PAR_COL_QUERY_MISMATCH        TAOS_DEF_ERROR_CODE(0, 0x266D)
+#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE  TAOS_DEF_ERROR_CODE(0, 0x266E)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner
@@ -739,6 +752,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
 #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION      TAOS_DEF_ERROR_CODE(0, 0x2804)
 #define TSDB_CODE_FUNC_DUP_TIMESTAMP            TAOS_DEF_ERROR_CODE(0, 0x2805)
+#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806)
+#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
 
 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
@@ -805,6 +820,8 @@ int32_t* taosGetErrno();
 // stream
 #define TSDB_CODE_STREAM_TASK_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x4100)
 #define TSDB_CODE_STREAM_EXEC_CANCELLED          TAOS_DEF_ERROR_CODE(0, 0x4102)
+#define TSDB_CODE_STREAM_INVALID_STATETRANS      TAOS_DEF_ERROR_CODE(0, 0x4103)
+#define TSDB_CODE_STREAM_TASK_IVLD_STATUS        TAOS_DEF_ERROR_CODE(0, 0x4104)
 
 // TDLite
 #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x5100)

include/util/tconfig.h

@@ -65,7 +65,6 @@ typedef struct SConfigItem {
   union {
     bool    bval;
     float   fval;
-    double  dval;
     int32_t i32;
     int64_t i64;
     char   *str;
@@ -104,7 +103,6 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop
 int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
 int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
-int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
 int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
 int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);

include/util/tdef.h

@@ -109,6 +109,15 @@ extern const int32_t TYPE_BYTES[21];
 
 #define TSDB_INS_USER_STABLES_DBNAME_COLID 2
 
+static const int64_t TICK_PER_SECOND[] = {
+  1000LL, // MILLISECOND
+  1000000LL, // MICROSECOND
+  1000000000LL, // NANOSECOND
+  0LL, // HOUR
+  0LL, // MINUTE
+  1LL // SECOND
+};
+
 #define TSDB_TICK_PER_SECOND(precision)               \
   ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI \
                  ? 1000LL                             \
@@ -231,6 +240,9 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_SQL_SHOW_LEN    1024
 #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u)  // sql length should be less than 1mb
 
+#define TSDB_VIEW_NAME_LEN       193
+#define TSDB_VIEW_FNAME_LEN      (TSDB_DB_FNAME_LEN + TSDB_VIEW_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
+
 #define TSDB_APP_NAME_LEN   TSDB_UNI_LEN
 #define TSDB_TB_COMMENT_LEN 1025
 
@@ -294,6 +306,8 @@ typedef enum ELogicConditionType {
 #define TSDB_SYNC_APPLYQ_SIZE_LIMIT    512
 #define TSDB_SYNC_NEGOTIATION_WIN      512
 
+#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048
+
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta
 
@@ -316,7 +330,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
 #define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
 #define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
-#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
+#define TSDB_MAX_DURATION_PER_FILE      (90 * 1440)
 #define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
 #define TSDB_MIN_KEEP                   (1 * 1440)          // data in db to be reserved. unit minute
 #define TSDB_MAX_KEEP                   (365000 * 1440)     // data in db to be reserved.

include/util/tqueue.h

@@ -101,6 +101,9 @@ struct STaosQall {
   STaosQnode *current;
   STaosQnode *start;
   int32_t     numOfItems;
+  int64_t     memOfItems;
+  int32_t     unAccessedNumOfItems;
+  int64_t     unAccessMemOfItems;
 };
 
 STaosQueue *taosOpenQueue();
@@ -123,6 +126,9 @@ int32_t    taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
 int32_t    taosGetQitem(STaosQall *qall, void **ppItem);
 void       taosResetQitems(STaosQall *qall);
 int32_t    taosQallItemSize(STaosQall *qall);
+int64_t    taosQallMemSize(STaosQall *qll);
+int64_t    taosQallUnAccessedItemSize(STaosQall *qall);
+int64_t    taosQallUnAccessedMemSize(STaosQall *qall);
 
 STaosQset *taosOpenQset();
 void       taosCloseQset(STaosQset *qset);
@@ -135,8 +141,6 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo)
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
 
-extern int64_t tsRpcQueueMemoryAllowed;
-
 #ifdef __cplusplus
 }
 #endif

source/client/CMakeLists.txt

@@ -1,5 +1,9 @@
 aux_source_directory(src CLIENT_SRC)
 
+IF (TD_ENTERPRISE)
+  LIST(APPEND CLIENT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/clientView.c)
+ENDIF ()
+
 if(TD_WINDOWS)
     add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in)
 else()

source/client/inc/clientInt.h

@@ -269,6 +269,7 @@ typedef struct SRequestObj {
   bool                 syncQuery;     // todo refactor: async query object
   bool                 stableQuery;   // todo refactor
   bool                 validateOnly;  // todo refactor
+  bool                 parseOnly;
   bool                 killed;
   bool                 inRetry;
   bool                 isSubReq;
@@ -279,6 +280,8 @@ typedef struct SRequestObj {
   void*                pPostPlan;
   SReqRelInfo          relation;
   void*                pWrapper;
+  SMetaData            parseMeta;
+  char*                effectiveUser;
 } SRequestObj;
 
 typedef struct SSyncQueryParam {
@@ -305,6 +308,8 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp,
 void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
                                  int64_t reqid);
 void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param);
+int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes);
+void syncQueryFn(void* param, void* res, int32_t code);
 
 int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols);
 
@@ -403,7 +408,7 @@ void    launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResu
 int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
 int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
 void    doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
-int32_t removeMeta(STscObj* pTscObj, SArray* tbList);
+int32_t removeMeta(STscObj* pTscObj, SArray* tbList, bool isView);
 int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
 int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
 bool    qnodeRequired(SRequestObj* pRequest);
@@ -415,6 +420,11 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
 int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
 void    returnToUser(SRequestObj* pRequest);
 void    stopAllQueries(SRequestObj *pRequest);
+void    freeQueryParam(SSyncQueryParam* param);
+
+#ifdef TD_ENTERPRISE
+int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
+#endif
 
 #ifdef __cplusplus
 }

source/client/src/clientEnv.c

@@ -447,6 +447,7 @@ void doDestroyRequest(void *p) {
   qDestroyQuery(pRequest->pQuery);
   nodesDestroyAllocator(pRequest->allocatorRefId);
 
+  taosMemoryFreeClear(pRequest->effectiveUser);
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFree(pRequest);
   tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);

source/client/src/clientHb.c

@@ -53,7 +53,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
   int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray);
   for (int32_t i = 0; i < numOfBatchs; ++i) {
     SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i);
-    tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version);
+    tscDebug("hb to update user auth, user:%s, version:%d", rsp->user, rsp->version);
 
     catalogUpdateUserAuthInfo(pCatalog, rsp);
   }
@@ -205,6 +205,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
         rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
 
       if (rsp->useDbRsp->vgVersion < 0) {
+        tscDebug("hb to remove db, db:%s", rsp->useDbRsp->db);
         code = catalogRemoveDB(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid);
       } else {
         SDBVgInfo *vgInfo = NULL;
@@ -213,6 +214,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
           goto _return;
         }
 
+        tscDebug("hb to update db vgInfo, db:%s", rsp->useDbRsp->db);
+
         catalogUpdateDBVgInfo(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid, vgInfo);
 
         if (IS_SYS_DBNAME(rsp->useDbRsp->db)) {
@@ -253,10 +256,10 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
     STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i);
 
     if (rsp->numOfColumns < 0) {
-      tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
+      tscDebug("hb to remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
       catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid);
     } else {
-      tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
+      tscDebug("hb to update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
       if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
         tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId);
         tFreeSSTbHbRsp(&hbRsp);
@@ -281,6 +284,108 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
   return TSDB_CODE_SUCCESS;
 }
 
+
+static int32_t hbProcessDynViewRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
+  return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion*)value);
+}
+
+static void hbFreeSViewMetaInRsp(void* p) {
+  if (NULL == p || NULL == *(void**)p) {
+    return;
+  }
+  SViewMetaRsp *pRsp = *(SViewMetaRsp**)p;
+  tFreeSViewMetaRsp(pRsp);
+  taosMemoryFreeClear(pRsp);
+}
+
+static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
+  int32_t code = 0;
+
+  SViewHbRsp hbRsp = {0};
+  if (tDeserializeSViewHbRsp(value, valueLen, &hbRsp) != 0) {
+    taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp);
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp);
+  for (int32_t i = 0; i < numOfMeta; ++i) {
+    SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i);
+
+    if (rsp->numOfCols < 0) {
+      tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name);
+      catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId);
+      tFreeSViewMetaRsp(rsp);
+      taosMemoryFreeClear(rsp);
+    } else {
+      tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name);
+      catalogUpdateViewMeta(pCatalog, rsp);
+    }
+  }
+
+  taosArrayDestroy(hbRsp.pViewRsp);
+  return TSDB_CODE_SUCCESS;
+}
+
+
+static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
+  for (int32_t i = 0; i < kvNum; ++i) {
+    SKv *kv = taosArrayGet(pKvs, i);
+    switch (kv->key) {
+      case HEARTBEAT_KEY_USER_AUTHINFO: {
+        if (kv->valueLen <= 0 || NULL == kv->value) {
+          tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value);
+          break;
+        }
+
+        hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr);
+        break;
+      }
+      case HEARTBEAT_KEY_DBINFO: {
+        if (kv->valueLen <= 0 || NULL == kv->value) {
+          tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value);
+          break;
+        }
+
+        hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog);
+        break;
+      }
+      case HEARTBEAT_KEY_STBINFO: {
+        if (kv->valueLen <= 0 || NULL == kv->value) {
+          tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value);
+          break;
+        }
+
+        hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog);
+        break;
+      }
+#ifdef TD_ENTERPRISE
+      case HEARTBEAT_KEY_DYN_VIEW: {
+        if (kv->valueLen <= 0 || NULL == kv->value) {
+          tscError("invalid dyn view info, len:%d, value:%p", kv->valueLen, kv->value);
+          break;
+        }
+
+        hbProcessDynViewRsp(kv->value, kv->valueLen, pCatalog);
+        break;
+      }
+      case HEARTBEAT_KEY_VIEWINFO: {
+        if (kv->valueLen <= 0 || NULL == kv->value) {
+          tscError("invalid view info, len:%d, value:%p", kv->valueLen, kv->value);
+          break;
+        }
+
+        hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog);
+        break;
+      }
+#endif
+      default:
+        tscError("invalid hb key type:%d", kv->key);
+        break;
+    }
+  }
+}
+
 static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
   SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == pReq) {
@@ -338,63 +443,13 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
 
   tscDebug("hb got %d rsp kv", kvNum);
 
-  for (int32_t i = 0; i < kvNum; ++i) {
+  if (kvNum > 0) {
-    SKv *kv = taosArrayGet(pRsp->info, i);
-    switch (kv->key) {
-      case HEARTBEAT_KEY_USER_AUTHINFO: {
-        if (kv->valueLen <= 0 || NULL == kv->value) {
-          tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value);
-          break;
-        }
-
     struct SCatalog *pCatalog = NULL;
-
     int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
     if (code != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
-          break;
+    } else {
-        }
+      hbProcessQueryRspKvs(kvNum, pRsp->info, pCatalog, pAppHbMgr);
-
-        hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr);
-        break;
-      }
-      case HEARTBEAT_KEY_DBINFO: {
-        if (kv->valueLen <= 0 || NULL == kv->value) {
-          tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value);
-          break;
-        }
-
-        struct SCatalog *pCatalog = NULL;
-
-        int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
-        if (code != TSDB_CODE_SUCCESS) {
-          tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
-          break;
-        }
-
-        hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog);
-        break;
-      }
-      case HEARTBEAT_KEY_STBINFO: {
-        if (kv->valueLen <= 0 || NULL == kv->value) {
-          tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value);
-          break;
-        }
-
-        struct SCatalog *pCatalog = NULL;
-
-        int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
-        if (code != TSDB_CODE_SUCCESS) {
-          tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
-          break;
-        }
-
-        hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog);
-        break;
-      }
-      default:
-        tscError("invalid hb key type:%d", kv->key);
-        break;
     }
   }
   
@@ -740,8 +795,8 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
   for (int32_t i = 0; i < stbNum; ++i) {
     SSTableVersion *stb = &stbs[i];
     stb->suid = htobe64(stb->suid);
-    stb->sversion = htons(stb->sversion);
+    stb->sversion = htonl(stb->sversion);
-    stb->tversion = htons(stb->tversion);
+    stb->tversion = htonl(stb->tversion);
     stb->smaVer = htonl(stb->smaVer);
   }
 
@@ -762,6 +817,56 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) {
+  SViewVersion    *views = NULL;
+  uint32_t         viewNum = 0;
+  int32_t          code = 0;
+  SDynViewVersion *pDynViewVer = NULL;
+
+  code = catalogGetExpiredViews(pCatalog, &views, &viewNum, &pDynViewVer);
+  if (TSDB_CODE_SUCCESS != code) {
+    taosMemoryFree(views);
+    taosMemoryFree(pDynViewVer);
+    return code;
+  }
+
+  if (viewNum <= 0) {
+    taosMemoryFree(views);
+    taosMemoryFree(pDynViewVer);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  for (int32_t i = 0; i < viewNum; ++i) {
+    SViewVersion *view = &views[i];
+    view->dbId = htobe64(view->dbId);
+    view->viewId = htobe64(view->viewId);
+    view->version = htonl(view->version);
+  }
+
+  tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum);
+
+  if (NULL == req->info) {
+    req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
+  }
+
+  SKv kv = {
+      .key = HEARTBEAT_KEY_DYN_VIEW,
+      .valueLen = sizeof(SDynViewVersion),
+      .value = pDynViewVer,
+  };
+
+  taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
+
+  kv.key = HEARTBEAT_KEY_VIEWINFO;
+  kv.valueLen = sizeof(SViewVersion) * viewNum;
+  kv.value = views;
+
+  taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) {
   SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
   if (NULL != pApp) {
@@ -781,19 +886,17 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
   SHbParam *hbParam = (SHbParam *)param;
   SCatalog *pCatalog = NULL;
 
+  hbGetQueryBasicInfo(connKey, req);
+
   if (hbParam->reqCnt == 0) {
     code = catalogGetHandle(hbParam->clusterId, &pCatalog);
     if (code != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code));
       return code;
     }
-  }
 
     hbGetAppInfo(hbParam->clusterId, req);
 
-  hbGetQueryBasicInfo(connKey, req);
-
-  if (hbParam->reqCnt == 0) {
     if (!taosHashGet(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(hbParam->clusterId))) {
       code = hbGetExpiredUserInfo(connKey, pCatalog, req);
       if (TSDB_CODE_SUCCESS != code) {
@@ -819,6 +922,15 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
     if (TSDB_CODE_SUCCESS != code) {
       return code;
     }
+
+#ifdef TD_ENTERPRISE
+    code = hbGetExpiredViewInfo(connKey, pCatalog, req);
+    if (TSDB_CODE_SUCCESS != code) {
+      return code;
+    }
+#endif    
+  } else {
+    req->app.appId = 0;
   }
 
   ++hbParam->reqCnt;  // success to get catalog info

source/client/src/clientImpl.c

@@ -1009,7 +1009,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
 
   tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) {
-    removeMeta(pTscObj, pRequest->targetTableList);
+    removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
   }
 
   pRequest->metric.execCostUs = taosGetTimestampUs() - pRequest->metric.execStart;
@@ -1097,7 +1097,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
   }
 
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) {
-    removeMeta(pRequest->pTscObj, pRequest->targetTableList);
+    removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
   }
 
   handleQueryExecRsp(pRequest);
@@ -1116,9 +1116,14 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
 
 static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta,
                                  SSqlCallbackWrapper* pWrapper) {
+  int32_t code = TSDB_CODE_SUCCESS;
   pRequest->type = pQuery->msgType;
+  SArray* pMnodeList = NULL;
+  SQueryPlan* pDag = NULL;  
+  int64_t st = taosGetTimestampUs();
 
-  SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
+  if (!pRequest->parseOnly) {
+    pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
 
     SPlanContext cxt = {.queryId = pRequest->requestId,
                         .acctId = pRequest->pTscObj->acctId,
@@ -1131,10 +1136,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
                         .sysInfo = pRequest->pTscObj->sysInfo,
                         .allocatorId = pRequest->allocatorRefId};
 
-  SQueryPlan* pDag = NULL;
+    code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
-
-  int64_t st = taosGetTimestampUs();
-  int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
     if (code) {
       tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
                pRequest->requestId);
@@ -1142,6 +1144,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
       pRequest->body.subplanNum = pDag->numOfSubplans;
       TSWAP(pRequest->pPostPlan, pDag->pPostPlan);
     }
+  }
 
   pRequest->metric.execStart = taosGetTimestampUs();
   pRequest->metric.planCostUs = pRequest->metric.execStart - st;
@@ -1173,6 +1176,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
     code = schedulerExecJob(&req, &pRequest->body.queryJob);
     taosArrayDestroy(pNodeList);
   } else {
+    qDestroyQueryPlan(pDag);
     tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
              pRequest->requestId);
     destorySqlCallbackWrapper(pWrapper);
@@ -1193,6 +1197,11 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
 void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) {
   int32_t code = 0;
 
+  if (pRequest->parseOnly) {
+    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    return;
+  }
+
   pRequest->body.execMode = pQuery->execMode;
   if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) {
     destorySqlCallbackWrapper(pWrapper);
@@ -1226,6 +1235,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
       pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
       break;
     default:
+      tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
       pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
       break;
   }
@@ -1272,7 +1282,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   return code;
 }
 
-int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
+int32_t removeMeta(STscObj* pTscObj, SArray* tbList, bool isView) {
   SCatalog* pCatalog = NULL;
   int32_t   tbNum = taosArrayGetSize(tbList);
   int32_t   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
@@ -1280,10 +1290,19 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
     return code;
   }
 
+  if (isView) {
+    for (int32_t i = 0; i < tbNum; ++i) {
+      SName* pViewName = taosArrayGet(tbList, i);
+      char dbFName[TSDB_DB_FNAME_LEN];
+      tNameGetFullDbName(pViewName, dbFName);
+      catalogRemoveViewMeta(pCatalog, dbFName, 0, pViewName->tname, 0);
+    }
+  } else {
     for (int32_t i = 0; i < tbNum; ++i) {
       SName* pTbName = taosArrayGet(tbList, i);
       catalogRemoveTableMeta(pCatalog, pTbName);
     }
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -2604,3 +2623,13 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
 
   schedulerFetchRows(pRequest->body.queryJob, &req);
 }
+
+int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
+#ifndef TD_ENTERPRISE
+  return TSDB_CODE_SUCCESS;
+#else
+  return clientParseSqlImpl(param, dbName, sql, parseOnly, effectiveUser, pRes);
+#endif
+}
+
+

source/client/src/clientMain.c

@@ -32,7 +32,7 @@
 #define TSC_VAR_RELEASED    0
 
 static int32_t sentinel = TSC_VAR_NOT_RELEASE;
-static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt);
+static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper);
 
 int taos_options(TSDB_OPTION option, const void *arg, ...) {
   static int32_t lock = 0;
@@ -879,39 +879,12 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
   return code;
 }
 
-static void destoryTablesReq(void *p) {
-  STablesReq *pRes = (STablesReq *)p;
-  taosArrayDestroy(pRes->pTables);
-}
-
-static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
-  if (NULL == pCatalogReq) {
-    return;
-  }
-  taosArrayDestroy(pCatalogReq->pDbVgroup);
-  taosArrayDestroy(pCatalogReq->pDbCfg);
-  taosArrayDestroy(pCatalogReq->pDbInfo);
-  if (pCatalogReq->cloned) {
-    taosArrayDestroy(pCatalogReq->pTableMeta);
-    taosArrayDestroy(pCatalogReq->pTableHash);
-  } else {
-    taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq);
-    taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
-  }
-  taosArrayDestroy(pCatalogReq->pUdf);
-  taosArrayDestroy(pCatalogReq->pIndex);
-  taosArrayDestroy(pCatalogReq->pUser);
-  taosArrayDestroy(pCatalogReq->pTableIndex);
-  taosArrayDestroy(pCatalogReq->pTableCfg);
-  taosArrayDestroy(pCatalogReq->pTableTag);
-  taosMemoryFree(pCatalogReq);
-}
-
 void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
   if (NULL == pWrapper) {
     return;
   }
   destoryCatalogReq(pWrapper->pCatalogReq);
+  taosMemoryFree(pWrapper->pCatalogReq);
   qDestroyParseContext(pWrapper->pParseCtx);
   taosMemoryFree(pWrapper);
 }
@@ -933,6 +906,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
 
   int64_t analyseStart = taosGetTimestampUs();
   pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart;
+  pWrapper->pParseCtx->parseOnly = pRequest->parseOnly;
 
   if (TSDB_CODE_SUCCESS == code) {
     code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
@@ -940,6 +914,11 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
 
   pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart;
 
+  if (pRequest->parseOnly) {
+    memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta));
+    memset(pResultMeta, 0, sizeof(*pResultMeta));
+  }
+  
   handleQueryAnslyseRes(pWrapper, pResultMeta, code);
 }
 
@@ -960,6 +939,7 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
     pTarget->pTableIndex = taosArrayDup(pSrc->pTableIndex, NULL);
     pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL);
     pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL);
+    pTarget->pView = taosArrayDup(pSrc->pView, NULL);
     pTarget->qNodeRequired = pSrc->qNodeRequired;
     pTarget->dNodeRequired = pSrc->dNodeRequired;
     pTarget->svrVerRequired = pSrc->svrVerRequired;
@@ -1147,7 +1127,7 @@ void taos_query_a_with_reqid(TAOS *taos, const char *sql, __taos_async_fn_t fp,
   taosAsyncQueryImplWithReqid(connId, sql, fp, param, false, reqid);
 }
 
-int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
+int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper) {
   const STscObj *pTscObj = pRequest->pTscObj;
 
   *pCxt = taosMemoryCalloc(1, sizeof(SParseContext));
@@ -1167,12 +1147,15 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
                            .pTransporter = pTscObj->pAppInfo->pTransporter,
                            .pStmtCb = NULL,
                            .pUser = pTscObj->user,
+                           .pEffectiveUser = pRequest->effectiveUser,
                            .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                            .enableSysInfo = pTscObj->sysInfo,
                            .async = true,
                            .svrVer = pTscObj->sVer,
                            .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
-                           .allocatorId = pRequest->allocatorRefId};
+                           .allocatorId = pRequest->allocatorRefId,
+                           .parseSqlFp = clientParseSql,
+                           .parseSqlParam = pWrapper};
   int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode);
   (*pCxt)->biMode = biMode;
   return TSDB_CODE_SUCCESS;
@@ -1191,7 +1174,7 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    code = createParseContext(pRequest, &pWrapper->pParseCtx);
+    code = createParseContext(pRequest, &pWrapper->pParseCtx, pWrapper);
   }
 
   if (TSDB_CODE_SUCCESS == code) {
@@ -1570,7 +1553,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   tsem_wait(&pParam->sem);
 
 _return:
-  taosArrayDestroyEx(catalogReq.pTableMeta, destoryTablesReq);
+  destoryCatalogReq(&catalogReq);
   destroyRequest(pRequest);
   return code;
 }

source/client/src/clientMsgHandler.c

@@ -35,7 +35,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
   setErrno(pRequest, code);
 
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
-    removeMeta(pRequest->pTscObj, pRequest->targetTableList);
+    removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
   }
 
   taosMemoryFree(pMsg->pEpSet);

source/client/src/clientSml.c

@@ -104,9 +104,9 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   SUserAuthRes authRes = {0};
 
   code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
-  nodesDestroyNode(authRes.pCond);
+  nodesDestroyNode(authRes.pCond[AUTH_RES_BASIC]);
 
-  return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
+  return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
 inline bool smlDoubleToInt64OverFlow(double num) {

source/client/src/clientTmq.c

@@ -1451,7 +1451,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
   bool set = false;
 
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
-  if (topicNumGet <= 0 && epoch <= tmq->epoch) {
+  if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) {
     tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d",
             tmq->consumerId, tmq->epoch, epoch, topicNumGet);
     return false;

source/common/src/systable.c

@@ -24,6 +24,7 @@
 #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
 #define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
 #define SYSTABLE_SCH_COL_NAME_LEN   ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
 
 // clang-format off
 static const SSysDbTableSchema dnodesSchema[] = {
@@ -315,8 +316,23 @@ static const SSysDbTableSchema userUserPrivilegesSchema[] = {
     {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 };
 
+static const SSysDbTableSchema userViewsSchema[] = {
+    {.name = "view_name", .bytes = SYSTABLE_SCH_VIEW_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "effective_user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+    {.name = "type", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "query_sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "parameters", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "default_values", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+//    {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+};
+
+
 static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
     {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@@ -343,6 +359,7 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
     {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
     {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
+    {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tglobal.c

@@ -104,13 +104,21 @@ uint16_t tsAuditPort = 6043;
 bool     tsEnableAuditCreateTable = true;
 
 // telem
+#ifdef TD_ENTERPRISE
+bool tsEnableTelem = false;
+#else
 bool    tsEnableTelem = true;
+#endif
 int32_t  tsTelemInterval = 43200;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com";
 uint16_t tsTelemPort = 80;
 char    *tsTelemUri = "/report";
 
+#ifdef TD_ENTERPRISE
+bool tsEnableCrashReport = false;
+#else
 bool    tsEnableCrashReport = true;
+#endif
 char *tsClientCrashReportUri = "/ccrashreport";
 char *tsSvrCrashReportUri = "/dcrashreport";
 
@@ -120,7 +128,7 @@ char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts";
 char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
 char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = "";  // user defined child table name can be specified in tag value.
 char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = "";
-                                                                   // If set to empty system will generate table name using MD5 hash.
+// If set to empty system will generate table name using MD5 hash.
 // true means that the name and order of cols in each line are the same(only for influx protocol)
 // bool    tsSmlDataFormat = false;
 // int32_t tsSmlBatchSize = 10000;
@@ -246,7 +254,8 @@ int32_t tsTtlBatchDropNum = 10000;   // number of tables dropped per batch
 // internal
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
-int32_t tsStreamCheckpointTickInterval = 300;
+int32_t tsStreamCheckpointInterval = 60;
+float   tsSinkDataRate = 2.0;
 int32_t tsStreamNodeCheckInterval = 30;
 int32_t tsTtlUnit = 86400;
 int32_t tsTtlPushIntervalSec = 10;
@@ -271,10 +280,10 @@ int8_t tsS3Enabled = false;
 int8_t tsS3Https = true;
 char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
 
-int32_t tsS3BlockSize = 4096;     // number of tsdb pages
+int32_t tsS3BlockSize = -1;        // number of tsdb pages (4096)
 int32_t tsS3BlockCacheSize = 16;   // number of blocks
-
+int32_t tsS3PageCacheSize = 4096;  // number of pages
-int32_t tsCheckpointInterval = 300;
+int32_t tsS3UploadDelaySec = 60 * 60;
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -453,7 +462,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1;
-  if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0)
+    return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1;
@@ -504,11 +514,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
-  if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1;
 
   if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1;
@@ -665,18 +675,19 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
     return -1;
 
-  if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddFloat(pCfg, "FPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
 
   if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@@ -686,8 +697,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, 2048, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
   // min free disk space used to check if the disk is full [50MB, 1GB]
   if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024,
@@ -946,7 +960,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
     return -1;
   }
 
-  tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN);
+  tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str,
+           TSDB_TABLE_NAME_LEN);
   tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
   tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
   tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN);
@@ -1038,7 +1053,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
   tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
 
-  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval;
+  tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "simdEnable")->bval;
   tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;
 
   tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@@ -1097,16 +1112,18 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
 
-  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns));
+  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
-  tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval;
+  tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
-  tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval;
+  tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
-  tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32;
+  tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
-  tsCurRange = cfgGetItem(pCfg, "CurRange")->i32;
+  tsCurRange = cfgGetItem(pCfg, "curRange")->i32;
-  tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval;
+  tsIfAdtFse = cfgGetItem(pCfg, "ifAdtFse")->bval;
-  tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor));
+  tstrncpy(tsCompressor, cfgGetItem(pCfg, "compressor")->str, sizeof(tsCompressor));
 
   tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
   tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
+  tsStreamCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i32;
+  tsSinkDataRate = cfgGetItem(pCfg, "streamSinkDataRate")->fval;
 
   tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
   tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;
@@ -1116,6 +1133,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
   tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
+  tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
+  tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
 
   GRANT_CFG_GET;
   return 0;
@@ -1413,7 +1432,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
       } else if (strcasecmp("smlChildTableName", name) == 0) {
         tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
       } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) {
-        tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN);
+        tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str,
+                 TSDB_TABLE_NAME_LEN);
       } else if (strcasecmp("smlTagName", name) == 0) {
         tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
         //      } else if (strcasecmp("smlDataFormat", name) == 0) {
@@ -1708,6 +1728,20 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
     return;
   }
 
+  if (strcasecmp(option, "s3PageCacheSize") == 0) {
+    int32_t newS3PageCacheSize = atoi(value);
+    uInfo("s3PageCacheSize set from %d to %d", tsS3PageCacheSize, newS3PageCacheSize);
+    tsS3PageCacheSize = newS3PageCacheSize;
+    return;
+  }
+
+  if (strcasecmp(option, "s3UploadDelaySec") == 0) {
+    int32_t newS3UploadDelaysec = atoi(value);
+    uInfo("s3UploadDelaySec set from %d to %d", tsS3UploadDelaySec, newS3UploadDelaysec);
+    tsS3UploadDelaySec = newS3UploadDelaysec;
+    return;
+  }
+
   if (strcasecmp(option, "ttlPushInterval") == 0) {
     int32_t newTtlPushInterval = atoi(value);
     uInfo("ttlPushInterval set from %d to %d", tsTtlPushIntervalSec, newTtlPushInterval);

source/common/src/tmsg.c

@@ -1677,6 +1677,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
   if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->sysInfo) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->enable) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->isView) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
@@ -1691,6 +1692,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
     if (tEncodeU32(&encoder, pReq->pIpRanges[i].ip) < 0) return -1;
     if (tEncodeU32(&encoder, pReq->pIpRanges[i].mask) < 0) return -1;
   }
+  if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1;
   ENCODESQL();
   tEndEncode(&encoder);
 
@@ -1708,6 +1710,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
   if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->sysInfo) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->isView) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
@@ -1728,6 +1731,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
     if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].ip)) < 0) return -1;
     if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].mask)) < 0) return -1;
   }
+  if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1;
   DECODESQL();
   tEndDecode(&decoder);
 
@@ -1802,10 +1806,18 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
 
   int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs);
   int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs);
-  int32_t numOfUseTbs = taosHashGetSize(pRsp->useDbs);
+  int32_t numOfAlterTbs = taosHashGetSize(pRsp->alterTbs);
+  int32_t numOfReadViews = taosHashGetSize(pRsp->readViews);
+  int32_t numOfWriteViews = taosHashGetSize(pRsp->writeViews);
+  int32_t numOfAlterViews = taosHashGetSize(pRsp->alterViews);
+  int32_t numOfUseDbs = taosHashGetSize(pRsp->useDbs);
   if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1;
   if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1;
-  if (tEncodeI32(pEncoder, numOfUseTbs) < 0) return -1;
+  if (tEncodeI32(pEncoder, numOfAlterTbs) < 0) return -1;
+  if (tEncodeI32(pEncoder, numOfReadViews) < 0) return -1;
+  if (tEncodeI32(pEncoder, numOfWriteViews) < 0) return -1;
+  if (tEncodeI32(pEncoder, numOfAlterViews) < 0) return -1;
+  if (tEncodeI32(pEncoder, numOfUseDbs) < 0) return -1;
 
   char *tb = taosHashIterate(pRsp->readTbs, NULL);
   while (tb != NULL) {
@@ -1837,6 +1849,66 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
     tb = taosHashIterate(pRsp->writeTbs, tb);
   }
 
+  tb = taosHashIterate(pRsp->alterTbs, NULL);
+  while (tb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(tb, &keyLen);
+    if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, key) < 0) return -1;
+
+    size_t valueLen = 0;
+    valueLen = strlen(tb);
+    if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, tb) < 0) return -1;
+
+    tb = taosHashIterate(pRsp->alterTbs, tb);
+  }
+
+  tb = taosHashIterate(pRsp->readViews, NULL);
+  while (tb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(tb, &keyLen);
+    if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, key) < 0) return -1;
+
+    size_t valueLen = 0;
+    valueLen = strlen(tb);
+    if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, tb) < 0) return -1;
+
+    tb = taosHashIterate(pRsp->readViews, tb);
+  }
+
+  tb = taosHashIterate(pRsp->writeViews, NULL);
+  while (tb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(tb, &keyLen);
+    if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, key) < 0) return -1;
+
+    size_t valueLen = 0;
+    valueLen = strlen(tb);
+    if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, tb) < 0) return -1;
+
+    tb = taosHashIterate(pRsp->writeViews, tb);
+  }
+
+  tb = taosHashIterate(pRsp->alterViews, NULL);
+  while (tb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(tb, &keyLen);
+    if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, key) < 0) return -1;
+
+    size_t valueLen = 0;
+    valueLen = strlen(tb);
+    if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
+    if (tEncodeCStr(pEncoder, tb) < 0) return -1;
+
+    tb = taosHashIterate(pRsp->alterViews, tb);
+  }
+
   int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
   while (useDb != NULL) {
     size_t keyLen = 0;
@@ -1876,9 +1948,14 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
   pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  pRsp->alterTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  pRsp->readViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  pRsp->writeViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  pRsp->alterViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL ||
-      pRsp->writeTbs == NULL || pRsp->useDbs == NULL) {
+      pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL ||
+      pRsp->writeViews == NULL || pRsp->alterViews == NULL ||pRsp->useDbs == NULL) {
     goto _err;
   }
 
@@ -1900,29 +1977,37 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
     int32_t len = strlen(db);
-    taosHashPut(pRsp->createdDbs, db, len, db, len + 1);
+    taosHashPut(pRsp->createdDbs, db, len + 1, db, len + 1);
   }
 
   for (int32_t i = 0; i < numOfReadDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
     int32_t len = strlen(db);
-    taosHashPut(pRsp->readDbs, db, len, db, len + 1);
+    taosHashPut(pRsp->readDbs, db, len + 1, db, len + 1);
   }
 
   for (int32_t i = 0; i < numOfWriteDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
     if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
     int32_t len = strlen(db);
-    taosHashPut(pRsp->writeDbs, db, len, db, len + 1);
+    taosHashPut(pRsp->writeDbs, db, len + 1, db, len + 1);
   }
 
   if (!tDecodeIsEnd(pDecoder)) {
     int32_t numOfReadTbs = 0;
     int32_t numOfWriteTbs = 0;
+    int32_t numOfAlterTbs = 0;
+    int32_t numOfReadViews = 0;
+    int32_t numOfWriteViews = 0;
+    int32_t numOfAlterViews = 0;
     int32_t numOfUseDbs = 0;
     if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) goto _err;
     if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) goto _err;
+    if (tDecodeI32(pDecoder, &numOfAlterTbs) < 0) goto _err;
+    if (tDecodeI32(pDecoder, &numOfReadViews) < 0) goto _err;
+    if (tDecodeI32(pDecoder, &numOfWriteViews) < 0) goto _err;
+    if (tDecodeI32(pDecoder, &numOfAlterViews) < 0) goto _err;
     if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) goto _err;
 
     for (int32_t i = 0; i < numOfReadTbs; ++i) {
@@ -1938,7 +2023,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
       value = taosMemoryCalloc(valuelen + 1, sizeof(char));
       if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
 
-      taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1);
+      taosHashPut(pRsp->readTbs, key, keyLen, value, valuelen + 1);
 
       taosMemoryFreeClear(key);
       taosMemoryFreeClear(value);
@@ -1957,7 +2042,83 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
       value = taosMemoryCalloc(valuelen + 1, sizeof(char));
       if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
 
-      taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1);
+      taosHashPut(pRsp->writeTbs, key, keyLen, value, valuelen + 1);
+
+      taosMemoryFreeClear(key);
+      taosMemoryFreeClear(value);
+    }
+
+    for (int32_t i = 0; i < numOfAlterTbs; ++i) {
+      int32_t keyLen = 0;
+      if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
+
+      key = taosMemoryCalloc(keyLen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
+
+      int32_t valuelen = 0;
+      if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
+
+      value = taosMemoryCalloc(valuelen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
+
+      taosHashPut(pRsp->alterTbs, key, keyLen, value, valuelen + 1);
+
+      taosMemoryFreeClear(key);
+      taosMemoryFreeClear(value);
+    }
+
+    for (int32_t i = 0; i < numOfReadViews; ++i) {
+      int32_t keyLen = 0;
+      if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
+
+      key = taosMemoryCalloc(keyLen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
+
+      int32_t valuelen = 0;
+      if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
+
+      value = taosMemoryCalloc(valuelen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
+
+      taosHashPut(pRsp->readViews, key, keyLen, value, valuelen + 1);
+
+      taosMemoryFreeClear(key);
+      taosMemoryFreeClear(value);
+    }
+
+    for (int32_t i = 0; i < numOfWriteViews; ++i) {
+      int32_t keyLen = 0;
+      if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
+
+      key = taosMemoryCalloc(keyLen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
+
+      int32_t valuelen = 0;
+      if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
+
+      value = taosMemoryCalloc(valuelen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
+
+      taosHashPut(pRsp->writeViews, key, keyLen, value, valuelen + 1);
+
+      taosMemoryFreeClear(key);
+      taosMemoryFreeClear(value);
+    }
+
+    for (int32_t i = 0; i < numOfAlterViews; ++i) {
+      int32_t keyLen = 0;
+      if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
+
+      key = taosMemoryCalloc(keyLen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
+
+      int32_t valuelen = 0;
+      if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
+
+      value = taosMemoryCalloc(valuelen + 1, sizeof(char));
+      if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
+
+      taosHashPut(pRsp->alterViews, key, keyLen, value, valuelen + 1);
 
       taosMemoryFreeClear(key);
       taosMemoryFreeClear(value);
@@ -1973,7 +2134,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
       int32_t ref = 0;
       if (tDecodeI32(pDecoder, &ref) < 0) goto _err;
 
-      taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
+      taosHashPut(pRsp->useDbs, key, keyLen, &ref, sizeof(ref));
       taosMemoryFreeClear(key);
     }
     // since 3.0.7.0
@@ -1993,8 +2154,12 @@ _err:
   taosHashCleanup(pRsp->createdDbs);
   taosHashCleanup(pRsp->readDbs);
   taosHashCleanup(pRsp->writeDbs);
-  taosHashCleanup(pRsp->writeTbs);
   taosHashCleanup(pRsp->readTbs);
+  taosHashCleanup(pRsp->writeTbs);
+  taosHashCleanup(pRsp->alterTbs);
+  taosHashCleanup(pRsp->readViews);
+  taosHashCleanup(pRsp->writeViews);
+  taosHashCleanup(pRsp->alterViews);
   taosHashCleanup(pRsp->useDbs);
 
   taosMemoryFreeClear(key);
@@ -2020,8 +2185,12 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
   taosHashCleanup(pRsp->createdDbs);
   taosHashCleanup(pRsp->readDbs);
   taosHashCleanup(pRsp->writeDbs);
-  taosHashCleanup(pRsp->writeTbs);
   taosHashCleanup(pRsp->readTbs);
+  taosHashCleanup(pRsp->writeTbs);
+  taosHashCleanup(pRsp->alterTbs);
+  taosHashCleanup(pRsp->readViews);
+  taosHashCleanup(pRsp->writeViews);
+  taosHashCleanup(pRsp->alterViews);
   taosHashCleanup(pRsp->useDbs);
 }
 
@@ -8405,7 +8574,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
       } else {
         tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE);
       }
-      taosMemoryFree(pTbData->pCreateTbReq);
+      taosMemoryFreeClear(pTbData->pCreateTbReq);
     }
 
     if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
@@ -8634,3 +8803,286 @@ void tDeleteMqSubTopicEp(SMqSubTopicEp *pSubTopicEp) {
   pSubTopicEp->schema.nCols = 0;
   taosArrayDestroy(pSubTopicEp->vgs);
 }
+
+int32_t tSerializeSCMCreateViewReq(void *buf, int32_t bufLen, const SCMCreateViewReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->querySql) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfCols) < 0) return -1;
+  for (int32_t i = 0; i < pReq->numOfCols; ++i) {
+    SSchema *pSchema = &pReq->pSchema[i];
+    if (tEncodeSSchema(&encoder, pSchema) < 0) return -1;
+  }
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1;
+  if (tDecodeCStrAlloc(&decoder, &pReq->querySql) < 0) return -1;
+  if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfCols) < 0) return -1;
+
+  if (pReq->numOfCols > 0) {
+    pReq->pSchema = taosMemoryCalloc(pReq->numOfCols, sizeof(SSchema));
+    if (pReq->pSchema == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    for (int32_t i = 0; i < pReq->numOfCols; ++i) {
+      SSchema* pSchema = pReq->pSchema + i;
+      if (tDecodeSSchema(&decoder, pSchema) < 0) return -1;
+    }
+  }
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void    tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) {
+  if (NULL == pReq) {
+    return;
+  }
+
+  taosMemoryFreeClear(pReq->querySql);
+  taosMemoryFreeClear(pReq->sql);  
+  taosMemoryFreeClear(pReq->pSchema);
+}
+
+int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1;
+  if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+void    tFreeSCMDropViewReq(SCMDropViewReq* pReq) {
+  if (NULL == pReq) {
+    return;
+  }
+  
+  taosMemoryFree(pReq->sql);
+}
+
+int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+static int32_t tEncodeSViewMetaRsp(SEncoder *pEncoder, const SViewMetaRsp *pRsp) {
+  if (tEncodeCStr(pEncoder, pRsp->name) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1;
+  if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
+  if (tEncodeU64(pEncoder, pRsp->viewId) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pRsp->querySql) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->type) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->numOfCols) < 0) return -1;
+  for (int32_t i = 0; i < pRsp->numOfCols; ++i) {
+    SSchema *pSchema = &pRsp->pSchema[i];
+    if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
+  }
+
+  return 0;
+}
+
+
+int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeSViewMetaRsp(&encoder, pRsp) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) {
+  if (tDecodeCStrTo(pDecoder, pRsp->name) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
+  if (tDecodeCStrAlloc(pDecoder, &pRsp->user) < 0) return -1;
+  if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
+  if (tDecodeU64(pDecoder, &pRsp->viewId) < 0) return -1;
+  if (tDecodeCStrAlloc(pDecoder, &pRsp->querySql) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->type) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->numOfCols) < 0) return -1;
+  if (pRsp->numOfCols > 0) {
+    pRsp->pSchema = taosMemoryCalloc(pRsp->numOfCols, sizeof(SSchema));
+    if (pRsp->pSchema == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    for (int32_t i = 0; i < pRsp->numOfCols; ++i) {
+      SSchema* pSchema = pRsp->pSchema + i;
+      if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1;
+    }
+  }
+
+  return 0;
+}
+
+int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeSViewMetaRsp(&decoder, pRsp) < 0) return -1;
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void    tFreeSViewMetaRsp(SViewMetaRsp* pRsp) {
+  if (NULL == pRsp) {
+    return;
+  }
+
+  taosMemoryFree(pRsp->user);
+  taosMemoryFree(pRsp->querySql);
+  taosMemoryFree(pRsp->pSchema);
+}
+
+int32_t tSerializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
+  if (tEncodeI32(&encoder, numOfMeta) < 0) return -1;
+  for (int32_t i = 0; i < numOfMeta; ++i) {
+    SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
+    if (tEncodeSViewMetaRsp(&encoder, pMetaRsp) < 0) return -1;
+  }
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  int32_t numOfMeta = 0;
+  if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1;
+  pRsp->pViewRsp = taosArrayInit(numOfMeta, POINTER_BYTES);
+  if (pRsp->pViewRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  for (int32_t i = 0; i < numOfMeta; ++i) {
+    SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp));
+    if (NULL == metaRsp) return -1;
+    if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1;
+    taosArrayPush(pRsp->pViewRsp, &metaRsp);
+  }
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
+  int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
+  for (int32_t i = 0; i < numOfMeta; ++i) {
+    SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
+    tFreeSViewMetaRsp(pMetaRsp);
+    taosMemoryFree(pMetaRsp);
+  }
+
+  taosArrayDestroy(pRsp->pViewRsp);
+}
+
+
+
+

source/common/src/ttime.c

@@ -25,7 +25,6 @@
 
 #include "tlog.h"
 
-
 // ==== mktime() kernel code =================//
 static int64_t m_deltaUtc = 0;
 
@@ -682,7 +681,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
   }
 
   // The following code handles the y/n time duration
-  int64_t numOfMonth = (unit == 'y')? duration*12:duration;
+  int64_t numOfMonth = (unit == 'y') ? duration * 12 : duration;
   int64_t fraction = t % TSDB_TICK_PER_SECOND(precision);
 
   struct tm tm;
@@ -768,7 +767,6 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
   int32_t precision = pInterval->precision;
 
   if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) {
-
     start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
     struct tm tm;
     time_t    tt = (time_t)start;
@@ -799,7 +797,7 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
         int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
 
         if (newe < ts) {  // move towards the greater endpoint
-          while(newe < ts && news < ts) {
+          while (newe < ts && news < ts) {
             news += pInterval->sliding;
             newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1;
           }
@@ -978,3 +976,947 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
 
   tstrncpy(buf, ts, bufLen);
 }
+
+int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
+  tm->fsec = ts % TICK_PER_SECOND[precision] * (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]);
+  time_t t = ts / TICK_PER_SECOND[precision];
+  taosLocalTime(&t, &tm->tm, NULL);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t taosTm2Ts(struct STm* tm, int64_t* ts, int32_t precision) {
+  *ts = taosMktime(&tm->tm);
+  *ts *= TICK_PER_SECOND[precision];
+  *ts += tm->fsec / (TICK_PER_SECOND[TSDB_TIME_PRECISION_NANO] / TICK_PER_SECOND[precision]);
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef struct {
+  const char* name;
+  int         len;
+  int         id;
+  bool        isDigit;
+} TSFormatKeyWord;
+
+typedef enum {
+  // TSFKW_AD,   // BC AD
+  // TSFKW_A_D,  // A.D. B.C.
+  TSFKW_AM,   // AM, PM
+  TSFKW_A_M,  // A.M., P.M.
+  // TSFKW_BC,   // BC AD
+  // TSFKW_B_C,  // B.C. A.D.
+  TSFKW_DAY,  // MONDAY, TUESDAY ...
+  TSFKW_DDD,  // Day of year 001-366
+  TSFKW_DD,   // Day of month 01-31
+  TSFKW_Day,  // Sunday, Monday
+  TSFKW_DY,   // MON, TUE
+  TSFKW_Dy,   // Mon, Tue
+  TSFKW_D,    // 1-7 -> Sunday(1) -> Saturday(7)
+  TSFKW_HH24,
+  TSFKW_HH12,
+  TSFKW_HH,
+  TSFKW_MI,  // minute
+  TSFKW_MM,
+  TSFKW_MONTH,  // JANUARY, FEBRUARY
+  TSFKW_MON,
+  TSFKW_Month,
+  TSFKW_Mon,
+  TSFKW_MS,
+  TSFKW_NS,
+  //TSFKW_OF,
+  TSFKW_PM,
+  TSFKW_P_M,
+  TSFKW_SS,
+  TSFKW_TZH,
+  // TSFKW_TZM,
+  // TSFKW_TZ,
+  TSFKW_US,
+  TSFKW_YYYY,
+  TSFKW_YYY,
+  TSFKW_YY,
+  TSFKW_Y,
+  // TSFKW_a_d,
+  // TSFKW_ad,
+  TSFKW_am,
+  TSFKW_a_m,
+  // TSFKW_b_c,
+  // TSFKW_bc,
+  TSFKW_day,
+  TSFKW_ddd,
+  TSFKW_dd,
+  TSFKW_dy,   // mon, tue
+  TSFKW_d,
+  TSFKW_hh24,
+  TSFKW_hh12,
+  TSFKW_hh,
+  TSFKW_mi,
+  TSFKW_mm,
+  TSFKW_month,
+  TSFKW_mon,
+  TSFKW_ms,
+  TSFKW_ns,
+  TSFKW_pm,
+  TSFKW_p_m,
+  TSFKW_ss,
+  TSFKW_tzh,
+  // TSFKW_tzm,
+  // TSFKW_tz,
+  TSFKW_us,
+  TSFKW_yyyy,
+  TSFKW_yyy,
+  TSFKW_yy,
+  TSFKW_y,
+  TSFKW_last_
+} TSFormatKeywordId;
+
+// clang-format off
+static const TSFormatKeyWord formatKeyWords[] = {
+  //{"AD", 2, TSFKW_AD, false},
+  //{"A.D.", 4, TSFKW_A_D},
+  {"AM", 2, TSFKW_AM, false},
+  {"A.M.", 4, TSFKW_A_M, false},
+  //{"BC", 2, TSFKW_BC, false},
+  //{"B.C.", 4, TSFKW_B_C, false},
+  {"DAY", 3, TSFKW_DAY, false},
+  {"DDD", 3, TSFKW_DDD, true},
+  {"DD", 2, TSFKW_DD, true},
+  {"Day", 3, TSFKW_Day, false},
+  {"DY", 2, TSFKW_DY, false},
+  {"Dy", 2, TSFKW_Dy, false},
+  {"D", 1, TSFKW_D, true},
+  {"HH24", 4, TSFKW_HH24, true},
+  {"HH12", 4, TSFKW_HH12, true},
+  {"HH", 2, TSFKW_HH, true},
+  {"MI", 2, TSFKW_MI, true},
+  {"MM", 2, TSFKW_MM, true},
+  {"MONTH", 5, TSFKW_MONTH, false},
+  {"MON", 3, TSFKW_MON, false},
+  {"Month", 5, TSFKW_Month, false},
+  {"Mon", 3, TSFKW_Mon, false},
+  {"MS", 2, TSFKW_MS, true},
+  {"NS", 2, TSFKW_NS, true},
+  //{"OF", 2, TSFKW_OF, false},
+  {"PM", 2, TSFKW_PM, false},
+  {"P.M.", 4, TSFKW_P_M, false},
+  {"SS", 2, TSFKW_SS, true},
+  {"TZH", 3, TSFKW_TZH, false},
+  //{"TZM", 3, TSFKW_TZM},
+  //{"TZ", 2, TSFKW_TZ},
+  {"US", 2, TSFKW_US, true},
+  {"YYYY", 4, TSFKW_YYYY, true},
+  {"YYY", 3, TSFKW_YYY, true},
+  {"YY", 2, TSFKW_YY, true},
+  {"Y", 1, TSFKW_Y, true},
+  //{"a.d.", 4, TSFKW_a_d, false},
+  //{"ad", 2, TSFKW_ad, false},
+  {"am", 2, TSFKW_am, false},
+  {"a.m.", 4, TSFKW_a_m, false},
+  //{"b.c.", 4, TSFKW_b_c, false},
+  //{"bc", 2, TSFKW_bc, false},
+  {"day", 3, TSFKW_day, false},
+  {"ddd", 3, TSFKW_DDD, true},
+  {"dd", 2, TSFKW_DD, true},
+  {"dy", 2, TSFKW_dy, false},
+  {"d", 1, TSFKW_D, true},
+  {"hh24", 4, TSFKW_HH24, true},
+  {"hh12", 4, TSFKW_HH12, true},
+  {"hh", 2, TSFKW_HH, true},
+  {"mi", 2, TSFKW_MI, true},
+  {"mm", 2, TSFKW_MM, true},
+  {"month", 5, TSFKW_month, false},
+  {"mon", 3, TSFKW_mon, false},
+  {"ms", 2, TSFKW_MS, true},
+  {"ns", 2, TSFKW_NS, true},
+  //{"of", 2, TSFKW_OF, false},
+  {"pm", 2, TSFKW_pm, false},
+  {"p.m.", 4, TSFKW_p_m, false},
+  {"ss", 2, TSFKW_SS, true},
+  {"tzh", 3, TSFKW_TZH, false},
+  //{"tzm", 3, TSFKW_TZM},
+  //{"tz", 2, TSFKW_tz},
+  {"us", 2, TSFKW_US, true},
+  {"yyyy", 4, TSFKW_YYYY, true},
+  {"yyy", 3, TSFKW_YYY, true},
+  {"yy", 2, TSFKW_YY, true},
+  {"y", 1, TSFKW_Y, true},
+  {NULL, 0, 0}
+};
+// clang-format on
+
+#define TS_FROMAT_KEYWORD_INDEX_SIZE ('z' - 'A' + 1)
+static const int TSFormatKeywordIndex[TS_FROMAT_KEYWORD_INDEX_SIZE] = {
+    /*A*/ TSFKW_AM,     -1, -1,
+    /*D*/ TSFKW_DAY,    -1, -1, -1,
+    /*H*/ TSFKW_HH24,   -1, -1, -1, -1,
+    /*M*/ TSFKW_MI,
+    /*N*/ TSFKW_NS,     -1,
+    /*P*/ TSFKW_PM,     -1, -1,
+    /*S*/ TSFKW_SS,
+    /*T*/ TSFKW_TZH,
+    /*U*/ TSFKW_US,     -1, -1, -1,
+    /*Y*/ TSFKW_YYYY,   -1,
+    /*[ \ ] ^ _ `*/ -1, -1, -1, -1, -1, -1,
+    /*a*/ TSFKW_am,     -1, -1,
+    /*d*/ TSFKW_day,    -1, -1, -1,
+    /*h*/ TSFKW_hh24,   -1, -1, -1, -1,
+    /*m*/ TSFKW_mi,
+    /*n*/ TSFKW_ns,     -1,
+    /*p*/ TSFKW_pm,     -1, -1,
+    /*s*/ TSFKW_ss,
+    /*t*/ TSFKW_tzh,
+    /*u*/ TSFKW_us,     -1, -1, -1,
+    /*y*/ TSFKW_yyyy,   -1};
+
+typedef struct {
+  uint8_t                type;
+  const char*            c;
+  int32_t                len;
+  const TSFormatKeyWord* key;
+} TSFormatNode;
+
+static const char* const weekDays[] = {"Sunday",   "Monday", "Tuesday",  "Wednesday",
+                                       "Thursday", "Friday", "Saturday", "NULL"};
+static const char* const shortWeekDays[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "NULL"};
+static const char* const fullMonths[] = {"January", "February",  "March",   "April",    "May",      "June", "July",
+                                         "August",  "September", "October", "November", "December", NULL};
+static const char* const months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul",
+                                     "Aug", "Sep", "Oct", "Nov", "Dec", NULL};
+#define A_M_STR "A.M."
+#define a_m_str "a.m."
+#define AM_STR  "AM"
+#define am_str  "am"
+#define P_M_STR "P.M."
+#define p_m_str "p.m."
+#define PM_STR  "PM"
+#define pm_str  "pm"
+static const char* const apms[] = {AM_STR, PM_STR, am_str, pm_str, NULL};
+static const char* const long_apms[] = {A_M_STR, P_M_STR, a_m_str, p_m_str, NULL};
+
+#define TS_FORMAT_NODE_TYPE_KEYWORD   1
+#define TS_FORMAT_NODE_TYPE_SEPARATOR 2
+#define TS_FORMAT_NODE_TYPE_CHAR      3
+
+static const TSFormatKeyWord* keywordSearch(const char* str) {
+  if (*str < 'A' || *str > 'z' || (*str > 'Z' && *str < 'a')) return NULL;
+  int32_t idx = TSFormatKeywordIndex[str[0] - 'A'];
+  if (idx < 0) return NULL;
+  const TSFormatKeyWord* key = &formatKeyWords[idx++];
+  while (key->name && str[0] == key->name[0]) {
+    if (0 == strncmp(key->name, str, key->len)) {
+      return key;
+    }
+    key = &formatKeyWords[idx++];
+  }
+  return NULL;
+}
+
+static bool isSeperatorChar(char c) {
+  return (c > 0x20 && c < 0x7F && !(c >= 'A' && c <= 'Z') && !(c >= 'a' && c <= 'z') && !(c >= '0' && c <= '9'));
+}
+
+static void parseTsFormat(const char* formatStr, SArray* formats) {
+  TSFormatNode* lastOtherFormat = NULL;
+  while (*formatStr) {
+    const TSFormatKeyWord* key = keywordSearch(formatStr);
+    if (key) {
+      TSFormatNode format = {.key = key, .type = TS_FORMAT_NODE_TYPE_KEYWORD};
+      taosArrayPush(formats, &format);
+      formatStr += key->len;
+      lastOtherFormat = NULL;
+    } else {
+      if (*formatStr == '"') {
+        lastOtherFormat = NULL;
+        // for double quoted string
+        formatStr++;
+        TSFormatNode* last = NULL;
+        while (*formatStr) {
+          if (*formatStr == '"') {
+            formatStr++;
+            break;
+          }
+          if (*formatStr == '\\' && *(formatStr + 1)) {
+            formatStr++;
+            last = NULL; // stop expanding last format, create new format
+          }
+          if (last) {
+            // expand
+            assert(last->type == TS_FORMAT_NODE_TYPE_CHAR);
+            last->len++;
+            formatStr++;
+          } else {
+            // create new
+            TSFormatNode format = {.type = TS_FORMAT_NODE_TYPE_CHAR, .key = NULL};
+            format.c = formatStr;
+            format.len = 1;
+            taosArrayPush(formats, &format);
+            formatStr++;
+            last = taosArrayGetLast(formats);
+          }
+        }
+      } else {
+        // for other strings
+        if (*formatStr == '\\' && *(formatStr + 1)) {
+          formatStr++;
+          lastOtherFormat = NULL; // stop expanding
+        } else {
+          if (lastOtherFormat && !isSeperatorChar(*formatStr)) {
+            // expanding
+          } else {
+            // create new
+            lastOtherFormat = NULL;
+          }
+        }
+        if (lastOtherFormat) {
+          assert(lastOtherFormat->type == TS_FORMAT_NODE_TYPE_CHAR);
+          lastOtherFormat->len++;
+          formatStr++;
+        } else {
+          TSFormatNode format = {
+            .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR,
+            .key = NULL};
+          format.c = formatStr;
+          format.len = 1;
+          taosArrayPush(formats, &format);
+          formatStr++;
+          if (format.type == TS_FORMAT_NODE_TYPE_CHAR) lastOtherFormat = taosArrayGetLast(formats);
+        }
+      }
+    }
+  }
+}
+
+static void tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
+  int32_t size = taosArrayGetSize(formats);
+  const char* start = s;
+  for (int32_t i = 0; i < size; ++i) {
+    TSFormatNode* format = taosArrayGet(formats, i);
+    if (format->type != TS_FORMAT_NODE_TYPE_KEYWORD) {
+      if (s - start + format->len + 1 > outLen) break;
+      strncpy(s, format->c, format->len);
+      s += format->len;
+      continue;
+    }
+    if (s - start + 16 > outLen) break;
+
+    switch (format->key->id) {
+      case TSFKW_AM:
+      case TSFKW_PM:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "PM" : "AM");
+        s += 2;
+        break;
+      case TSFKW_A_M:
+      case TSFKW_P_M:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "P.M." : "A.M.");
+        s += 4;
+        break;
+      case TSFKW_am:
+      case TSFKW_pm:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "pm" : "am");
+        s += 2;
+        break;
+      case TSFKW_a_m:
+      case TSFKW_p_m:
+        sprintf(s, tm->tm.tm_hour % 24 >= 12 ? "p.m." : "a.m.");
+        s += 4;
+        break;
+      case TSFKW_DDD:
+        sprintf(s, "%03d", tm->tm.tm_yday + 1);
+        s += strlen(s);
+        break;
+      case TSFKW_DD:
+        sprintf(s, "%02d", tm->tm.tm_mday);
+        s += 2;
+        break;
+      case TSFKW_D:
+        sprintf(s, "%d", tm->tm.tm_wday + 1);
+        s += 1;
+        break;
+      case TSFKW_DAY: {
+        // MONDAY, TUESDAY...
+        const char* wd = weekDays[tm->tm.tm_wday];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Day:
+        // Monday, TuesDay...
+        sprintf(s, "%-9s", weekDays[tm->tm.tm_wday]);
+        s += strlen(s);
+        break;
+      case TSFKW_day: {
+        const char* wd = weekDays[tm->tm.tm_wday];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_DY: {
+        // MON, TUE
+        const char* wd = shortWeekDays[tm->tm.tm_wday];
+        char        buf[8] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = toupper(wd[i]);
+        sprintf(s, "%3s", buf);
+        s += 3;
+        break;
+      }
+      case TSFKW_Dy:
+        // Mon, Tue
+        sprintf(s, "%3s", shortWeekDays[tm->tm.tm_wday]);
+        s += 3;
+        break;
+      case TSFKW_dy: {
+        // mon, tue
+        const char* wd = shortWeekDays[tm->tm.tm_wday];
+        char        buf[8] = {0};
+        for (int32_t i = 0; i < strlen(wd); ++i) buf[i] = tolower(wd[i]);
+        sprintf(s, "%3s", buf);
+        s += 3;
+        break;
+      }
+      case TSFKW_HH24:
+        sprintf(s, "%02d", tm->tm.tm_hour);
+        s += 2;
+        break;
+      case TSFKW_HH:
+      case TSFKW_HH12:
+        // 0 or 12 o'clock in 24H coresponds to 12 o'clock (AM/PM) in 12H
+        sprintf(s, "%02d", tm->tm.tm_hour % 12 == 0 ? 12 : tm->tm.tm_hour % 12);
+        s += 2;
+        break;
+      case TSFKW_MI:
+        sprintf(s, "%02d", tm->tm.tm_min);
+        s += 2;
+        break;
+      case TSFKW_MM:
+        sprintf(s, "%02d", tm->tm.tm_mon + 1);
+        s += 2;
+        break;
+      case TSFKW_MONTH: {
+        const char* mon = fullMonths[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_MON: {
+        const char* mon = months[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = toupper(mon[i]);
+        sprintf(s, "%s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Month:
+        sprintf(s, "%-9s", fullMonths[tm->tm.tm_mon]);
+        s += strlen(s);
+        break;
+      case TSFKW_month: {
+        const char* mon = fullMonths[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]);
+        sprintf(s, "%-9s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_Mon:
+        sprintf(s, "%s", months[tm->tm.tm_mon]);
+        s += strlen(s);
+        break;
+      case TSFKW_mon: {
+        const char* mon = months[tm->tm.tm_mon];
+        char        buf[10] = {0};
+        for (int32_t i = 0; i < strlen(mon); ++i) buf[i] = tolower(mon[i]);
+        sprintf(s, "%s", buf);
+        s += strlen(s);
+        break;
+      }
+      case TSFKW_SS:
+        sprintf(s, "%02d", tm->tm.tm_sec);
+        s += 2;
+        break;
+      case TSFKW_MS:
+        sprintf(s, "%03" PRId64, tm->fsec / 1000000L);
+        s += 3;
+        break;
+      case TSFKW_US:
+        sprintf(s, "%06" PRId64, tm->fsec / 1000L);
+        s += 6;
+        break;
+      case TSFKW_NS:
+        sprintf(s, "%09" PRId64, tm->fsec);
+        s += 9;
+        break;
+      case TSFKW_TZH:
+        sprintf(s, "%s%02d", tsTimezone < 0 ? "-" : "+", tsTimezone);
+        s += strlen(s);
+        break;
+      case TSFKW_YYYY:
+        sprintf(s, "%04d", tm->tm.tm_year + 1900);
+        s += strlen(s);
+        break;
+      case TSFKW_YYY:
+        sprintf(s, "%03d", (tm->tm.tm_year + 1900) % 1000);
+        s += strlen(s);
+        break;
+      case TSFKW_YY:
+        sprintf(s, "%02d", (tm->tm.tm_year + 1900) % 100);
+        s += strlen(s);
+        break;
+      case TSFKW_Y:
+        sprintf(s, "%01d", (tm->tm.tm_year + 1900) % 10);
+        s += strlen(s);
+        break;
+      default:
+        break;
+    }
+  }
+}
+
+/// @brief find s in arr case insensitively
+/// @retval the index in arr if found, -1 if not found
+static int32_t strArrayCaseSearch(const char* const* arr, const char* s) {
+  if (!*s) return -1;
+  const char* const* fmt = arr;
+  for (; *fmt; ++fmt) {
+    const char *l, *r;
+    for (l = fmt[0], r = s;; l++, r++) {
+      if (*l == '\0') return fmt - arr;
+      if (*r == '\0' || tolower(*l) != tolower(*r)) break;
+    }
+  }
+  return -1;
+}
+
+static const char* tsFormatStr2Int32(int32_t* dest, const char* str, int32_t len, bool needMoreDigit) {
+  char*       last;
+  int64_t     res;
+  const char* s = str;
+  if ('\0' == str[0]) return NULL;
+  if (len <= 0) {
+    res = taosStr2Int64(s, &last, 10);
+    s = last;
+  } else {
+    char buf[16] = {0};
+    strncpy(buf, s, len);
+    int32_t copiedLen = strlen(buf);
+    if (copiedLen < len) {
+      if (!needMoreDigit) {
+        // digits not enough, that's ok, cause we do not need more digits
+        // '2023-1' 'YYYY-MM'
+        // '202a' 'YYYY' -> 202
+        res = taosStr2Int64(s, &last, 10);
+        s += copiedLen;
+      } else {
+        // bytes not enough, and there are other digit formats to match
+        // '2023-1' 'YYYY-MMDD'
+        return NULL;
+      }
+    } else {
+      if (needMoreDigit) {
+        res = taosStr2Int64(buf, &last, 10);
+        // bytes enough, but digits not enough, like '202a12' 'YYYYMM', YYYY needs four digits
+        if (last - buf < len) return NULL;
+        s += last - buf;
+      } else {
+        res = taosStr2Int64(s, &last, 10);
+        s = last;
+      }
+    }
+  }
+  if (s == str) {
+    // no integers found
+    return NULL;
+  }
+  if (errno == ERANGE || res > INT32_MAX || res < INT32_MIN) {
+    // out of range
+    return NULL;
+  }
+  *dest = res;
+  return s;
+}
+
+static int32_t adjustYearTo2020(int32_t year) {
+  if (year < 70) return year + 2000;    // 2000 - 2069
+  if (year < 100) return year + 1900;   // 1970 - 1999
+  if (year < 520) return year + 2000;   // 2100 - 2519
+  if (year < 1000) return year + 1000;  // 1520 - 1999
+  return year;
+}
+
+static bool checkTm(const struct tm* tm) {
+  if (tm->tm_mon < 0 || tm->tm_mon > 11) return false;
+  if (tm->tm_wday < 0 || tm->tm_wday > 6) return false;
+  if (tm->tm_yday < 0 || tm->tm_yday > 365) return false;
+  if (tm->tm_mday < 0 || tm->tm_mday > 31) return false;
+  if (tm->tm_hour < 0 || tm->tm_hour > 23) return false;
+  if (tm->tm_min < 0 || tm->tm_min > 59) return false;
+  if (tm->tm_sec < 0 || tm->tm_sec > 60) return false;
+  return true;
+}
+
+static bool needMoreDigits(SArray* formats, int32_t curIdx) {
+  if (curIdx == taosArrayGetSize(formats) - 1) return false;
+  TSFormatNode* pNextNode = taosArrayGet(formats, curIdx + 1);
+  if (pNextNode->type == TS_FORMAT_NODE_TYPE_SEPARATOR) {
+    return false;
+  } else if (pNextNode->type == TS_FORMAT_NODE_TYPE_KEYWORD) {
+    return pNextNode->key->isDigit;
+  } else {
+    return isdigit(pNextNode->c[0]);
+  }
+}
+
+/// @brief convert a formatted time str to timestamp
+/// @param[in] s the formatted timestamp str
+/// @param[in] formats array of TSFormatNode, output of parseTsFormat
+/// @param[out] ts output timestamp
+/// @param precision the timestamp precision to convert to, sec/milli/micro/nano
+/// @param[out] sErrPos if not NULL, when err occured, points to the failed position of s, only set when ret is -1
+/// @param[out] fErrIdx if not NULL, when err occured, the idx of the failed format idx, only set when ret is -1
+/// @retval 0 for success
+/// @retval -1 for format and s mismatch error
+/// @retval -2 if datetime err, like 2023-13-32 25:61:69
+static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos,
+                       int32_t* fErrIdx) {
+  int32_t size = taosArrayGetSize(formats);
+  int32_t pm = 0;      // default am
+  int32_t hour12 = 0;  // default HH24
+  int32_t year = 0, mon = 0, yd = 0, md = 1, wd = 0;
+  int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0;
+  int32_t tzSign = 1, tz = tsTimezone;
+  int32_t err = 0;
+
+  for (int32_t i = 0; i < size && *s != '\0'; ++i) {
+    while (isspace(*s) && *s != '\0') {
+      s++;
+    }
+    if (!s) break;
+    TSFormatNode* node = taosArrayGet(formats, i);
+    if (node->type == TS_FORMAT_NODE_TYPE_SEPARATOR) {
+      // separator matches any character
+      if (isSeperatorChar(s[0])) s += node->len;
+      continue;
+    }
+    if (node->type == TS_FORMAT_NODE_TYPE_CHAR) {
+      int32_t pos = 0;
+      // skip leading spaces
+      while (isspace(node->c[pos]) && node->len > 0) pos++;
+      while (pos < node->len && *s != '\0') {
+        if (!isspace(node->c[pos++])) {
+          while (isspace(*s) && *s != '\0') s++;
+          if (*s != '\0') s++;  // forward together
+        }
+      }
+      continue;
+    }
+    assert(node->type == TS_FORMAT_NODE_TYPE_KEYWORD);
+    switch (node->key->id) {
+      case TSFKW_A_M:
+      case TSFKW_P_M:
+      case TSFKW_a_m:
+      case TSFKW_p_m: {
+        int32_t idx = strArrayCaseSearch(long_apms, s);
+        if (idx >= 0) {
+          s += 4;
+          pm = idx % 2;
+          hour12 = 1;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_AM:
+      case TSFKW_PM:
+      case TSFKW_am:
+      case TSFKW_pm: {
+        int32_t idx = strArrayCaseSearch(apms, s);
+        if (idx >= 0) {
+          s += 2;
+          pm = idx % 2;
+          hour12 = 1;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_HH:
+      case TSFKW_HH12: {
+        const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos || hour > 12 || hour <= 0) {
+          err = -1;
+        } else {
+          hour12 = 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_HH24: {
+        const char* newPos = tsFormatStr2Int32(&hour, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          hour12 = 0;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_MI: {
+        const char* newPos = tsFormatStr2Int32(&min, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_SS: {
+        const char* newPos = tsFormatStr2Int32(&sec, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else
+          s = newPos;
+      } break;
+      case TSFKW_MS: {
+        const char* newPos = tsFormatStr2Int32(&ms, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          ms *= len == 1 ? 100 : len == 2 ? 10 : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_US: {
+        const char* newPos = tsFormatStr2Int32(&us, s, 6, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          us *= len == 1 ? 100000 : len == 2 ? 10000 : len == 3 ? 1000 : len == 4 ? 100 : len == 5 ? 10 : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_NS: {
+        const char* newPos = tsFormatStr2Int32(&ns, s, 9, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          int32_t len = newPos - s;
+          ns *= len == 1   ? 100000000
+                : len == 2 ? 10000000
+                : len == 3 ? 1000000
+                : len == 4 ? 100000
+                : len == 5 ? 10000
+                : len == 6 ? 1000
+                : len == 7 ? 100
+                : len == 8 ? 10
+                           : 1;
+          s = newPos;
+        }
+      } break;
+      case TSFKW_TZH: {
+        tzSign = *s == '-' ? -1 : 1;
+        const char* newPos = tsFormatStr2Int32(&tz, s, -1, needMoreDigits(formats, i));
+        if (NULL == newPos)
+          err = -1;
+        else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_MONTH:
+      case TSFKW_Month:
+      case TSFKW_month: {
+        int32_t idx = strArrayCaseSearch(fullMonths, s);
+        if (idx >= 0) {
+          s += strlen(fullMonths[idx]);
+          mon = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_MON:
+      case TSFKW_Mon:
+      case TSFKW_mon: {
+        int32_t idx = strArrayCaseSearch(months, s);
+        if (idx >= 0) {
+          s += strlen(months[idx]);
+          mon = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_MM: {
+        const char* newPos = tsFormatStr2Int32(&mon, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+          mon -= 1;
+        }
+      } break;
+      case TSFKW_DAY:
+      case TSFKW_Day:
+      case TSFKW_day: {
+        int32_t idx = strArrayCaseSearch(weekDays, s);
+        if (idx >= 0) {
+          s += strlen(weekDays[idx]);
+          wd = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_DY:
+      case TSFKW_Dy:
+      case TSFKW_dy: {
+        int32_t idx = strArrayCaseSearch(shortWeekDays, s);
+        if (idx >= 0) {
+          s += strlen(shortWeekDays[idx]);
+          wd = idx;
+        } else {
+          err = -1;
+        }
+      } break;
+      case TSFKW_DDD: {
+        const char* newPos = tsFormatStr2Int32(&yd, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_DD: {
+        const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_D: {
+        const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YYYY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 4, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YYY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 3, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      case TSFKW_YY: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 2, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      case TSFKW_Y: {
+        const char* newPos = tsFormatStr2Int32(&year, s, 1, needMoreDigits(formats, i));
+        if (NULL == newPos) {
+          err = -1;
+        } else {
+          year = adjustYearTo2020(year);
+          s = newPos;
+        }
+      } break;
+      default:
+        break;
+    }
+    if (err) {
+      if (sErrPos) *sErrPos = s;
+      if (fErrIdx) *fErrIdx = i;
+      return err;
+    }
+  }
+  struct STm tm = {0};
+  tm.tm.tm_year = year - 1900;
+  tm.tm.tm_mon = mon;
+  tm.tm.tm_yday = yd;
+  tm.tm.tm_mday = md;
+  tm.tm.tm_wday = wd;
+  if (hour12) {
+    if (pm && hour < 12)
+      tm.tm.tm_hour = hour + 12;
+    else if (!pm && hour == 12)
+      tm.tm.tm_hour = 0;
+    else
+      tm.tm.tm_hour = hour;
+  } else {
+    tm.tm.tm_hour = hour;
+  }
+  tm.tm.tm_min = min;
+  tm.tm.tm_sec = sec;
+  if (!checkTm(&tm.tm)) return -2;
+  if (tz < -12 || tz > 12) return -2;
+  tm.fsec = ms * 1000000 + us * 1000 + ns;
+  int32_t ret = taosTm2Ts(&tm, ts, precision);
+  *ts += 60 * 60 * (tsTimezone - tz) * TICK_PER_SECOND[precision];
+  return ret;
+}
+
+void taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t precision, char* out, int32_t outLen) {
+  if (!*formats) {
+    *formats = taosArrayInit(8, sizeof(TSFormatNode));
+    parseTsFormat(format, *formats);
+  }
+  struct STm tm;
+  taosTs2Tm(ts, precision, &tm);
+  tm2char(*formats, &tm, out, outLen);
+}
+
+int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,
+                    int32_t errMsgLen) {
+  const char* sErrPos;
+  int32_t     fErrIdx;
+  if (!*formats) {
+    *formats = taosArrayInit(4, sizeof(TSFormatNode));
+    parseTsFormat(format, *formats);
+  }
+  int32_t code = char2ts(tsStr, *formats, ts, precision, &sErrPos, &fErrIdx);
+  if (code == -1) {
+    TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx));
+    snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos,
+             fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : "");
+  } else if (code == -2) {
+    snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format);
+  }
+  return code;
+}
+
+void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen) {
+  SArray* formats = taosArrayInit(4, sizeof(TSFormatNode));
+  parseTsFormat(format, formats);
+  struct STm tm;
+  taosTs2Tm(ts, precision, &tm);
+  tm2char(formats, &tm, out, outLen);
+  taosArrayDestroy(formats);
+}
+
+int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr) {
+  const char* sErrPos;
+  int32_t     fErrIdx;
+  SArray*     formats = taosArrayInit(4, sizeof(TSFormatNode));
+  parseTsFormat(format, formats);
+  int32_t code = char2ts(tsStr, formats, ts, precision, &sErrPos, &fErrIdx);
+  if (code == -1) {
+    printf("failed position: %s\n", sErrPos);
+    printf("failed format: %s\n", ((TSFormatNode*)taosArrayGet(formats, fErrIdx))->key->name);
+  }
+  taosArrayDestroy(formats);
+  return code;
+}

source/common/test/commonTests.cpp

@@ -13,6 +13,7 @@
 #include "tdatablock.h"
 #include "tdef.h"
 #include "tvariant.h"
+#include "ttime.h"
 
 namespace {
 //
@@ -260,4 +261,234 @@ TEST(testCase, var_dataBlock_split_test) {
   }
 }
 
+void check_tm(const STm* tm, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) {
+  ASSERT_EQ(tm->tm.tm_year, y);
+  ASSERT_EQ(tm->tm.tm_mon, mon);
+  ASSERT_EQ(tm->tm.tm_mday, d);
+  ASSERT_EQ(tm->tm.tm_hour, h);
+  ASSERT_EQ(tm->tm.tm_min, m);
+  ASSERT_EQ(tm->tm.tm_sec, s);
+  ASSERT_EQ(tm->fsec, fsec);
+}
+
+void test_timestamp_tm_conversion(int64_t ts, int32_t precision, int32_t y, int32_t mon, int32_t d, int32_t h, int32_t m, int32_t s, int64_t fsec) {
+  int64_t ts_tmp;
+  char buf[128] = {0};
+  struct STm tm;
+  taosFormatUtcTime(buf, 128, ts, precision);
+  printf("formated ts of %ld, precision: %d is: %s\n", ts, precision, buf);
+  taosTs2Tm(ts, precision, &tm);
+  check_tm(&tm, y, mon, d, h, m, s, fsec);
+  taosTm2Ts(&tm, &ts_tmp, precision);
+  ASSERT_EQ(ts, ts_tmp);
+}
+
+TEST(timeTest, timestamp2tm) {
+  const char* ts_str_ns = "2023-10-12T11:29:00.775726171+0800";
+  const char* ts_str_us = "2023-10-12T11:29:00.775726+0800";
+  const char* ts_str_ms = "2023-10-12T11:29:00.775+0800";
+  int64_t     ts, tmp_ts = 0;
+  struct STm  tm;
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ns, &ts, strlen(ts_str_ns), TSDB_TIME_PRECISION_NANO, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_NANO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775726171L);
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_us, &ts, strlen(ts_str_us), TSDB_TIME_PRECISION_MICRO, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MICRO, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775726000L);
+
+  ASSERT_EQ(TSDB_CODE_SUCCESS, taosParseTime(ts_str_ms, &ts, strlen(ts_str_ms), TSDB_TIME_PRECISION_MILLI, 0));
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 2023 - 1900, 9 /* mon start from 0*/, 12, 11, 29, 0,
+                               775000000L);
+
+  ts = -5364687943000;  // milliseconds since epoch, Wednesday, January 1, 1800 1:00:00 AM GMT+08:06
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1800 - 1900, 0 /* mon start from 0*/, 1, 1, 0, 0,
+                               000000000L);
+
+  ts = 0;
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, 1970 - 1900, 0 /* mon start from 0*/, 1, 8, 0, 0,
+                               000000000L);
+
+  ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06
+  test_timestamp_tm_conversion(ts, TSDB_TIME_PRECISION_MILLI, -1 - 1900, 0 /* mon start from 0*/, 1,
+                               0 /* hour start from 0*/, 0, 0, 000000000L);
+}
+
+void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* expected) {
+  char buf[256] = {0};
+  TEST_ts2char(format, ts, precison, buf, 256);
+  printf("ts: %ld format: %s res: [%s], expected: [%s]\n", ts, format, buf, expected);
+  ASSERT_STREQ(expected, buf);
+}
+
+TEST(timeTest, ts2char) {
+  osDefaultInit();
+  if (tsTimezone != TdEastZone8) GTEST_SKIP();
+  int64_t ts;
+  const char* format = "YYYY-MM-DD";
+  ts = 0;
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_MILLI, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_MICRO, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_NANO, "1970-01-01");
+  test_ts2char(ts, format, TSDB_TIME_PRECISION_SECONDS, "1970-01-01");
+
+  ts = 1697163517;
+  test_ts2char(ts, "YYYY-MM-DD", TSDB_TIME_PRECISION_SECONDS, "2023-10-13");
+  ts = 1697163517000;
+  test_ts2char(ts, "YYYY-MM-DD-Day-DAY", TSDB_TIME_PRECISION_MILLI, "2023-10-13-Friday   -FRIDAY   ");
+#ifndef WINDOWS
+  // double quoted: year, month, day are not parsed
+  test_ts2char(ts,
+               "YYYY-YYY-YY-Y-yyyy-yyy-yy-y-\"年\"-MONTH-MON-Month-Mon-month-mon-\"月\"-DDD-DD-D-ddd-dd-d-DAY-Day-"
+               "day-\"日\"",
+               TSDB_TIME_PRECISION_MILLI,
+               "2023-023-23-3-2023-023-23-3-年-OCTOBER  -OCT-October  -Oct-october  "
+               "-oct-月-286-13-6-286-13-6-FRIDAY   -Friday   -friday   -日");
+#endif
+  ts = 1697182085123L; // Friday, October 13, 2023 3:28:05.123 PM GMT+08:00
+  test_ts2char(ts, "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm");
+
+  // double quotes normal output
+  test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am\\\"", TSDB_TIME_PRECISION_MILLI,
+               "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm\"");
+  test_ts2char(ts, "\\\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "\"15:15:03:03:03:03:28:28:05:05:123:123:123000:123000:123000000:123000000:PM:PM:pm:pm");
+  // double quoted strings recognized as literal string, parsing skipped
+  test_ts2char(ts, "\"HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am", TSDB_TIME_PRECISION_MILLI,
+               "HH24:hh24:HH12:hh12:HH:hh:MI:mi:SS:ss:MS:ms:US:us:NS:ns:PM:AM:pm:am");
+  test_ts2char(ts, "yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "2023-10-13 15:28:05.123000000pmaaa");
+  test_ts2char(ts, "aaa--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "aaa--2023-10-13 15:28:05.123000000pmaaa");
+  test_ts2char(ts, "add--yyyy-mm-dd hh24:mi:ss.nsamaaa", TSDB_TIME_PRECISION_MILLI, "a13--2023-10-13 15:28:05.123000000pmaaa");
+
+  ts = 1693946405000;
+  test_ts2char(ts, "Day, Month dd, YYYY hh24:mi:ss AM TZH:tzh", TSDB_TIME_PRECISION_MILLI, "Wednesday, September 06, 2023 04:40:05 AM +08:+08");
+
+  ts = -62198784343000; // milliseconds before epoch, Friday, January 1, -0001 12:00:00 AM GMT+08:06
+  test_ts2char(ts, "Day, Month dd, YYYY hh12:mi:ss AM", TSDB_TIME_PRECISION_MILLI, "Friday   , January   01, -001 12:00:00 AM");
+}
+
+TEST(timeTest, char2ts) {
+  osDefaultInit();
+  if (tsTimezone != TdEastZone8) GTEST_SKIP();
+  int64_t ts;
+  int32_t code =
+      TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM");
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(ts, 1696867200000LL);
+
+  // 2009-1-1 00:00:00
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY-Y", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(1230739200000LL, ts);
+  // 2023-1-1
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY-YY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(ts, 1672502400000LL);
+
+  // 2123-1-1, the second year(123) is used, which converted to 2123
+  ASSERT_EQ(0, TEST_char2ts("YYYY-YYY", &ts, TSDB_TIME_PRECISION_MILLI, "2023-123-23-9"));
+  ASSERT_EQ(ts, 4828176000000LL);
+  // 2023-1-1 12:10:10am
+  ASSERT_EQ(0, TEST_char2ts("yyyy-mm-dd HH12:MI:SSAM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-1-1 12:10:10am"));
+  ASSERT_EQ(ts, 1672503010000LL);
+
+  // 2023-1-1 21:10:10.123
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH12:MI:ss.msa.m.", &ts, TSDB_TIME_PRECISION_MILLI, "23-1-01 9:10:10.123p.m."));
+  ASSERT_EQ(ts, 1672578610123LL);
+
+  // 2023-1-1 21:10:10.123456789
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 9:10:10.123.000456.000000789p.m."));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2023-1-1 21:10:10.120450780
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:SS.ms.us.ns", &ts, TSDB_TIME_PRECISION_NANO,
+                            "   23   - 1 - 01   \t  21:10:10 .  12 .  \t 00045 . 00000078  \t"));
+  ASSERT_EQ(ts, 1672578610120450780LL);
+
+#ifndef WINDOWS
+  // 2023-1-1 21:10:10.120450780
+  ASSERT_EQ(0, TEST_char2ts("yy \"年\"-MM 月-dd  \"日 子\" HH24:MI:ss.ms.us.ns TZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "   23  年 - 1 月 - 01 日 子  \t  21:10:10 .  12 .  \t 00045 . 00000078  \t+08"));
+  ASSERT_EQ(ts, 1672578610120450780LL);
+#endif
+
+  // 2023-1-1 19:10:10.123456789+06 -> 2023-1-1 21:10:10.123456789+08
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH:MI:ss.ms.us.nsa.m.TZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 7:10:10.123.000456.000000789p.m.6"));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2023-1-1 12:10:10.123456789-01 -> 2023-1-1 21:10:10.123456789+08
+  ASSERT_EQ(0, TEST_char2ts("yy-MM-dd HH24:MI:ss.ms.us.nsTZH", &ts, TSDB_TIME_PRECISION_NANO,
+                            "23-1-01 12:10:10.123.000456.000000789-1"));
+  ASSERT_EQ(ts, 1672578610123456789LL);
+
+  // 2100-01-01 11:10:10.124456+08
+  ASSERT_EQ(
+      0, TEST_char2ts("yyyy-MM-dd HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO, "2100-01-01 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  // 2100-01-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/MONTH/dd DAY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 friday 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 FRIDAY 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+  ASSERT_EQ(0, TEST_char2ts("yyyy/Month/dd Dy HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 Fri 11:10:10.124456+08:00"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyy/month/dd day HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/january/01 Friday 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4102456210124456LL);
+
+  // 2100-02-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/Feb/01 Mon 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4105134610124456LL);
+
+  // 2100-02-01 11:10:10.124456+08 Firday
+  ASSERT_EQ(0, TEST_char2ts("yyyy/mon/dd DY DDD-DD-D HH24:MI:ss.usTZH", &ts, TSDB_TIME_PRECISION_MICRO,
+                            "2100/Feb/01 Mon 100-1-01 11:10:10.124456+08"));
+  ASSERT_EQ(ts, 4105134610124456LL);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21000101"));
+
+  // What is Fe?
+  ASSERT_EQ(-1, TEST_char2ts("yyyy/mon/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/Fe/01"));
+  // '/' cannot convert to MM
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100/2/1"));
+  // nothing to be converted to dd
+  ASSERT_EQ(0, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "210012"));
+  ASSERT_EQ(ts, 4131273600000000LL); // 2100-12-1
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMMdd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001"));
+  ASSERT_EQ(-1, TEST_char2ts("yyyyMM-dd ", &ts, TSDB_TIME_PRECISION_MICRO, "23a1-1"));
+
+  // 2100-1-2
+  ASSERT_EQ(0, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "21001/2"));
+  ASSERT_EQ(ts, 4102502400000000LL);
+
+  // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00
+  ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970"));
+  ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60);
+
+  ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2"));
+  ASSERT_EQ(ts, 4102502400000000LL);
+
+  ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210013/2"));
+  ASSERT_EQ(-2, TEST_char2ts("yyyyMM/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210011/32"));
+  ASSERT_EQ(-1, TEST_char2ts("HH12:MI:SS", &ts, TSDB_TIME_PRECISION_MICRO, "21:12:12"));
+  ASSERT_EQ(-1, TEST_char2ts("yyyy/MM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "2100111111111/11/2"));
+  ASSERT_EQ(-2, TEST_char2ts("yyyy/MM1/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "23/11/2-13"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年1/1+0"));
+  ASSERT_EQ(ts, 0);
+  ASSERT_EQ(-1, TEST_char2ts("yyyy年a MM/dd", &ts, TSDB_TIME_PRECISION_MICRO, "2023年1/2"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年   1/1+0"));
+  ASSERT_EQ(ts, 0);
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0"));
+  ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a     "));
+}
+
 #pragma GCC diagnostic pop

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -163,7 +163,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
-
+  if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@@ -189,6 +189,9 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_RESTORE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -137,7 +137,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
     SRetention *pRetention = &pCfg->tsdbCfg.retentions[i];
     memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
     if (i == 0) {
-      if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
+      if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
     }
   }
 

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -97,7 +97,7 @@ int32_t       dmMarkWrapper(SMgmtWrapper *pWrapper);
 void          dmReleaseWrapper(SMgmtWrapper *pWrapper);
 int32_t       dmInitVars(SDnode *pDnode);
 void          dmClearVars(SDnode *pDnode);
-int32_t       dmInitModule(SDnode *pDnode);
+int32_t       dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
 bool          dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
 SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
 void          dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
@@ -119,7 +119,7 @@ int32_t dmInitStatusClient(SDnode *pDnode);
 void    dmCleanupClient(SDnode *pDnode);
 void    dmCleanupStatusClient(SDnode *pDnode);
 SMsgCb  dmGetMsgcb(SDnode *pDnode);
-int32_t dmInitMsgHandle(SDnode *pDnode);
+int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
 int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 
 // dmMonitor.c

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -66,7 +66,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
     goto _OVER;
   }
 
-  if (dmInitModule(pDnode) != 0) {
+  if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
     goto _OVER;
   }
 
@@ -107,6 +107,75 @@ void dmCleanupDnode(SDnode *pDnode) {
   dDebug("dnode is closed, ptr:%p", pDnode);
 }
 
+int32_t dmInitVars(SDnode *pDnode) {
+  SDnodeData *pData = &pDnode->data;
+  pData->dnodeId = 0;
+  pData->clusterId = 0;
+  pData->dnodeVer = 0;
+  pData->engineVer = 0;
+  pData->updateTime = 0;
+  pData->rebootTime = taosGetTimestampMs();
+  pData->dropped = 0;
+  pData->stopped = 0;
+
+  pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+  if (pData->dnodeHash == NULL) {
+    dError("failed to init dnode hash");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (dmReadEps(pData) != 0) {
+    dError("failed to read file since %s", terrstr());
+    return -1;
+  }
+
+  if (pData->dropped) {
+    dError("dnode will not start since its already dropped");
+    return -1;
+  }
+
+  taosThreadRwlockInit(&pData->lock, NULL);
+  taosThreadMutexInit(&pDnode->mutex, NULL);
+  return 0;
+}
+
+void dmClearVars(SDnode *pDnode) {
+  for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    taosMemoryFreeClear(pWrapper->path);
+    taosThreadRwlockDestroy(&pWrapper->lock);
+  }
+  if (pDnode->lockfile != NULL) {
+    taosUnLockFile(pDnode->lockfile);
+    taosCloseFile(&pDnode->lockfile);
+    pDnode->lockfile = NULL;
+  }
+
+  SDnodeData *pData = &pDnode->data;
+  taosThreadRwlockWrlock(&pData->lock);
+  if (pData->oldDnodeEps != NULL) {
+    if (dmWriteEps(pData) == 0) {
+      dmRemoveDnodePairs(pData);
+    }
+    taosArrayDestroy(pData->oldDnodeEps);
+    pData->oldDnodeEps = NULL;
+  }
+  if (pData->dnodeEps != NULL) {
+    taosArrayDestroy(pData->dnodeEps);
+    pData->dnodeEps = NULL;
+  }
+  if (pData->dnodeHash != NULL) {
+    taosHashCleanup(pData->dnodeHash);
+    pData->dnodeHash = NULL;
+  }
+  taosThreadRwlockUnlock(&pData->lock);
+
+  taosThreadRwlockDestroy(&pData->lock);
+  taosThreadMutexDestroy(&pDnode->mutex);
+  memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
+}
+
 void dmSetStatus(SDnode *pDnode, EDndRunStatus status) {
   if (pDnode->status != status) {
     dDebug("dnode status set from %s to %s", dmStatStr(pDnode->status), dmStatStr(status));

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -251,11 +251,11 @@ _OVER:
   dmReleaseWrapper(pWrapper);
 }
 
-int32_t dmInitMsgHandle(SDnode *pDnode) {
+int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
   SDnodeTrans *pTrans = &pDnode->trans;
 
   for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    SMgmtWrapper *pWrapper = wrappers + ntype;
     SArray       *pArray = (*pWrapper->func.getHandlesFp)();
     if (pArray == NULL) return -1;
 

source/dnode/mnode/impl/CMakeLists.txt

@@ -7,6 +7,7 @@ IF (TD_ENTERPRISE)
   LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c)
   LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c)
   LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDnode.c)
+  LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/mndView.c)
 ENDIF ()
 
 add_library(mnode STATIC ${MNODE_SRC})

source/dnode/mnode/impl/inc/mndDef.h

@@ -74,6 +74,8 @@ typedef enum {
   MND_OPER_SUBSCRIBE,
   MND_OPER_CREATE_TOPIC,
   MND_OPER_DROP_TOPIC,
+  MND_OPER_CREATE_VIEW,
+  MND_OPER_DROP_VIEW,
 } EOperType;
 
 typedef enum {
@@ -297,6 +299,10 @@ typedef struct {
   SHashObj* topics;
   SHashObj* readTbs;
   SHashObj* writeTbs;
+  SHashObj* alterTbs;
+  SHashObj* readViews;
+  SHashObj* writeViews;
+  SHashObj* alterViews;
   SHashObj* useDbs;
   SRWLatch  lock;
 } SUserObj;
@@ -704,6 +710,34 @@ void    tFreeStreamObj(SStreamObj* pObj);
 //   SArray* childInfo;  // SArray<SStreamChildEpInfo>
 // } SStreamCheckpointObj;
 
+#define VIEW_TYPE_UPDATABLE    (1 << 0)
+#define VIEW_TYPE_MATERIALIZED (1 << 1)
+
+typedef struct {
+  char     fullname[TSDB_VIEW_FNAME_LEN];
+  char     name[TSDB_VIEW_NAME_LEN];
+  char     dbFName[TSDB_DB_FNAME_LEN];
+  char     user[TSDB_USER_LEN];
+  char*    querySql;
+  char*    parameters;
+  void**   defaultValues;
+  char*    targetTable;
+  uint64_t viewId;
+  uint64_t dbId;
+  int64_t  createdTime;
+  int32_t  version;
+  int8_t   precision;
+  int8_t   type;
+  int32_t  numOfCols;
+  SSchema* pSchema;
+  SRWLatch lock;  
+} SViewObj;
+
+int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj);
+int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver);
+void    tFreeSViewObj(SViewObj* pObj);
+
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndPrivilege.h

@@ -28,6 +28,7 @@ void    mndCleanupPrivilege(SMnode *pMnode);
 int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
 int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
 int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
+int32_t mndCheckViewPrivilege(SMnode *pMnode, const char *user, EOperType operType, const char *pViewFName);
 int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
 int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
 int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);

source/dnode/mnode/impl/inc/mndUser.h

@@ -41,6 +41,7 @@ int32_t   mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int3
                                   int32_t *pRspLen);
 int32_t   mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db);
 int32_t   mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb);
+int32_t   mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view);
 int32_t   mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic);
 
 int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew);

source/dnode/mnode/impl/inc/mndView.h

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MND_VIEW_H_
+#define _TD_MND_VIEW_H_
+
+#include "mndInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t mndInitView(SMnode *pMnode);
+void    mndCleanupView(SMnode *pMnode);
+
+int32_t mndProcessCreateViewReq(SRpcMsg *pReq);
+int32_t mndProcessDropViewReq(SRpcMsg *pReq);
+int32_t mndProcessGetViewMetaReq(SRpcMsg *pReq);
+
+
+#ifdef TD_ENTERPRISE
+
+void initDynViewVersion(void);
+
+SViewObj *mndAcquireView(SMnode *pMnode, char *viewName);
+void      mndReleaseView(SMnode *pMnode, SViewObj *pView);
+
+SSdbRaw *mndViewActionEncode(SViewObj *pView);
+SSdbRow *mndViewActionDecode(SSdbRaw *pRaw);
+int32_t mndViewActionInsert(SSdb *pSdb, SViewObj *pView);
+int32_t mndViewActionDelete(SSdb *pSdb, SViewObj *pView);
+int32_t mndViewActionUpdate(SSdb *pSdb, SViewObj *pOldView, SViewObj *pNewView);
+
+int32_t mndProcessCreateViewReqImpl(SCMCreateViewReq* pCreateView, SRpcMsg *pReq);
+int32_t mndProcessDropViewReqImpl(SCMDropViewReq* pDropView, SRpcMsg *pReq);
+int32_t mndProcessViewMetaReqImpl(SViewMetaReq* pMetaReq, SRpcMsg *pReq);
+int32_t mndRetrieveViewImpl(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
+int32_t mndDropViewByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
+int32_t mndValidateDynViewVersion(SMnode *pMnode, SDynViewVersion* pReqVer, bool *needCheck, SDynViewVersion** ppRspVer);
+int32_t mndValidateViewInfo(SMnode *pMnode, SViewVersion *pViewVersions, int32_t numOfViews, void **ppRsp,
+                           int32_t *pRspLen);
+
+#endif
+
+int32_t mndRetrieveView(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
+void mndCancelGetNextView(SMnode *pMnode, void *pIter);
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MND_VIEW_H_*/

source/dnode/mnode/impl/src/mndDb.c

@@ -28,6 +28,7 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
+#include "mndView.h"
 #include "systable.h"
 #include "tjson.h"
 #include "thttp.h"
@@ -1262,6 +1263,9 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
+#ifdef TD_ENTERPRISE
+  if (mndDropViewByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
+#endif  
   if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndDnode.c

@@ -397,6 +397,7 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
     SDnodeInfo dInfo;
     dInfo.id = pDnode->id;
     dInfo.ep.port = pDnode->port;
+    dInfo.offlineReason = pDnode->offlineReason;
     tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
     tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
     tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
@@ -781,11 +782,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
 
     SDnodeObj tmpDnode = *pDnode;
     if (action == DND_ACTIVE_CODE) {
-#ifndef TD_GRANT_OPTIMIZE
-      if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
-#else
       if (grantAlterActiveCode(pDnode->id, pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
-#endif
         if (TSDB_CODE_DUP_KEY != terrno) {
           mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
                  pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
@@ -801,11 +798,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
         goto _OVER;
       }
     } else if (action == DND_CONN_ACTIVE_CODE) {
-#ifndef TD_GRANT_OPTIMIZE
-      if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
-#else
       if (grantAlterActiveCode(pDnode->id, pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
-#endif
         if (TSDB_CODE_DUP_KEY != terrno) {
           mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
                  pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
@@ -1025,7 +1018,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
   char obj[200] = {0};
   sprintf(obj, "%s:%d", createReq.fqdn, createReq.port);
 
-  auditRecord(pReq, pMnode->clusterId, "createDnode", obj, "", createReq.sql, createReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "createDnode", "", obj, createReq.sql, createReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -1174,7 +1167,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
   char obj1[30] = {0};
   sprintf(obj1, "%d", dropReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "dropDnode", obj1, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropDnode", "", obj1, dropReq.sql, dropReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -1235,6 +1228,70 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
 
     strcpy(dcfgReq.config, "monitor");
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+  } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
+    int32_t optLen = strlen("s3blocksize");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag > 1024 * 1024) {
+      mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId,
+             flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3blocksize");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+  } else if (strncasecmp(cfgReq.config, "s3blockcachesize", 16) == 0) {
+    int32_t optLen = strlen("s3blockcachesize");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag < 4 || flag > 1024 * 1024) {
+      mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]",
+             cfgReq.dnodeId, flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3blockcachesize");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+  } else if (strncasecmp(cfgReq.config, "s3pagecachesize", 16) == 0) {
+    int32_t optLen = strlen("s3pagecachesize");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag < 4 || flag > 1024 * 1024 * 1024) {
+      mError("dnode:%d, failed to config s3PageCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId,
+             flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3pagecachesize");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+  } else if (strncasecmp(cfgReq.config, "s3uploaddelaysec", 16) == 0) {
+    int32_t optLen = strlen("s3uploaddelaysec");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag < 600 || flag > 60 * 60 * 24 * 30) {
+      mError("dnode:%d, failed to config s3UploadDelaySec since value:%d. Valid range: [600, 60 * 60 * 24 * 30]",
+             cfgReq.dnodeId, flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3uploaddelaysec");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
   } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
     int32_t optLen = strlen("ttlpushinterval");
     int32_t flag = -1;
@@ -1375,7 +1432,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   char obj[50] = {0};
   sprintf(obj, "%d", cfgReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "alterDnode", "", obj, cfgReq.sql, cfgReq.sqlLen);
 
   tFreeSMCfgDnodeReq(&cfgReq);
 

source/dnode/mnode/impl/src/mndGrant.c

@@ -131,13 +131,9 @@ void    grantAdd(EGrantType grant, uint64_t value) {}
 void    grantRestore(EGrantType grant, uint64_t value) {}
 int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
 int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
-#ifndef TD_GRANT_OPTIMIZE
-int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; }
-#else
 int32_t grantAlterActiveCode(int32_t did, const char *old, const char *new, char *out, int8_t type) {
   return TSDB_CODE_SUCCESS;
 }
-#endif
 
 #endif
 

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -78,7 +78,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
   STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
   if (NULL == pMeta) {
     mError("invalid information schema table name:%s", tbName);
-    terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 
@@ -111,7 +111,7 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
   STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
   if (NULL == pMeta) {
     mError("invalid information schema table name:%s", tbName);
-    terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 

source/dnode/mnode/impl/src/mndMain.c

@@ -41,6 +41,7 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
+#include "mndView.h"
 
 static inline int32_t mndAcquireRpc(SMnode *pMnode) {
   int32_t code = 0;
@@ -281,7 +282,7 @@ static void *mndThreadFp(void *param) {
       mndCalMqRebalance(pMnode);
     }
 
-    if (sec % tsStreamCheckpointTickInterval == 0) {
+    if (sec % tsStreamCheckpointInterval == 0) {
       mndStreamCheckpointTick(pMnode, sec);
     }
 
@@ -447,6 +448,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;

source/dnode/mnode/impl/src/mndMnode.c

@@ -656,7 +656,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
   char obj[40] = {0};
   sprintf(obj, "%d", createReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "createMnode", obj, "", createReq.sql, createReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "createMnode", "", obj, createReq.sql, createReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -798,7 +798,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
   char obj[40] = {0};
   sprintf(obj, "%d", dropReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "dropMnode", obj, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropMnode", "", obj, dropReq.sql, dropReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -75,7 +75,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
   STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
   if (NULL == meta) {
     mError("invalid performance schema table name:%s", tbName);
-    terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 
@@ -101,7 +101,7 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
   STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
   if (NULL == pMeta) {
     mError("invalid performance schema table name:%s", tbName);
-    terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -23,6 +23,7 @@
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndUser.h"
+#include "mndView.h"
 #include "tglobal.h"
 #include "tversion.h"
 #include "audit.h"
@@ -311,13 +312,10 @@ _CONNECT:
 
   code = 0;
 
-  char obj[100] = {0};
-  sprintf(obj, "%s:%d", ip, pConn->port);
-
   char detail[1000] = {0};
-  sprintf(detail, "app:%s", connReq.app);
+  sprintf(detail, "%s:%d, app:%s", ip, pConn->port, connReq.app);
 
-  auditRecord(pReq, pMnode->clusterId, "login", connReq.user, obj, detail, strlen(detail));
+  auditRecord(pReq, pMnode->clusterId, "login", "", "", detail, strlen(detail));
 
 _OVER:
 
@@ -466,7 +464,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
   SClientHbRsp  hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
   SRpcConnInfo  connInfo = pMsg->info.conn;
 
+  if (0 != pHbReq->app.appId) {
     mndUpdateAppInfo(pMnode, pHbReq, &connInfo);
+  }
 
   if (pHbReq->query) {
     SQueryHbReqBasic *pBasic = pHbReq->query;
@@ -529,6 +529,28 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
     return -1;
   }
 
+#ifdef TD_ENTERPRISE
+  bool needCheck = true;
+  int32_t key = HEARTBEAT_KEY_DYN_VIEW;
+  SDynViewVersion* pDynViewVer = NULL;
+  SKv* pKv = taosHashGet(pHbReq->info, &key, sizeof(key));
+  if (NULL != pKv) {
+    pDynViewVer = pKv->value;
+    mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
+
+    SDynViewVersion* pRspVer = NULL;
+    if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) {
+      return -1;
+    }
+    
+    if (needCheck) {
+      SKv kv1 = {.key = HEARTBEAT_KEY_DYN_VIEW, .valueLen = sizeof(*pDynViewVer), .value = pRspVer};
+      taosArrayPush(hbRsp.info, &kv1);
+      mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, pRspVer->dynViewVer);
+    }
+  }
+#endif
+
   void *pIter = taosHashIterate(pHbReq->info, NULL);
   while (pIter != NULL) {
     SKv *kv = pIter;
@@ -564,6 +586,25 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
         }
         break;
       }
+#ifdef TD_ENTERPRISE
+      case HEARTBEAT_KEY_DYN_VIEW: {
+        break;
+      }
+      case HEARTBEAT_KEY_VIEWINFO: {
+        if (!needCheck) {
+          break;
+        }
+        
+        void   *rspMsg = NULL;
+        int32_t rspLen = 0;
+        mndValidateViewInfo(pMnode, kv->value, kv->valueLen / sizeof(SViewVersion), &rspMsg, &rspLen);
+        if (rspMsg && rspLen > 0) {
+          SKv kv1 = {.key = HEARTBEAT_KEY_VIEWINFO, .valueLen = rspLen, .value = rspMsg};
+          taosArrayPush(hbRsp.info, &kv1);
+        }
+        break;
+      }
+#endif
       default:
         mError("invalid kv key:%d", kv->key);
         hbRsp.status = TSDB_CODE_APP_ERROR;

source/dnode/mnode/impl/src/mndQnode.c

@@ -310,7 +310,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
   char obj[33] = {0};
   sprintf(obj, "%d", createReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "createQnode", obj, "", createReq.sql, createReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "createQnode", "", obj, createReq.sql, createReq.sqlLen);
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
     mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr());
@@ -424,7 +424,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
   char obj[33] = {0};
   sprintf(obj, "%d", dropReq.dnodeId);
 
-  auditRecord(pReq, pMnode->clusterId, "dropQnode", obj, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropQnode", "", obj, dropReq.sql, dropReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {

source/dnode/mnode/impl/src/mndScheduler.c

@@ -569,6 +569,10 @@ static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStr
 }
 
 static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) {
+  if (taosArrayGetSize(pTasksList) < SINK_NODE_LEVEL || pUpstreamTask == NULL) {
+    return;
+  }
+
   SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL);
   for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
     SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
@@ -628,7 +632,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
     }
 
     setSinkTaskUpstreamInfo(pStream->tasks, pAggTask);
+    if (pHAggTask != NULL) {
       setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
+    }
 
     // source level
     return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey);
@@ -722,7 +728,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     pVgEp->vgId = pVgroup->vgId;
     taosArrayPush(pSub->unassignedVgs, &pVgEp);
 
-    mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
+    mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
 
     sdbRelease(pSdb, pVgroup);
   }

source/dnode/mnode/impl/src/mndShow.c

@@ -116,6 +116,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
     type = TSDB_MGMT_TABLE_STREAM_TASKS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) {
     type = TSDB_MGMT_TABLE_PRIVILEGES;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) {
+    type = TSDB_MGMT_TABLE_VIEWS;
   } else {
     mError("invalid show name:%s len:%d", name, len);
   }
@@ -211,7 +213,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
     if (pMeta == NULL) {
       pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
       if (pMeta == NULL) {
-        terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
+        terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
         mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
         return -1;
       }

source/dnode/mnode/impl/src/mndStb.c

@@ -1903,8 +1903,51 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
   return 0;
 }
 
-static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp,
+static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) {
-                                 int32_t *smaVer) {
+  char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
+  snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName);
+
+  SDbObj *pDb = mndAcquireDb(pMnode, pStbVer->dbFName);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    return -1;
+  }
+
+  if (pDb->uid != pStbVer->dbId) {
+    mndReleaseDb(pMnode, pDb);
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    return -1;
+  }
+
+  SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
+  if (pStb == NULL) {
+    mndReleaseDb(pMnode, pDb);
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+    return -1;
+  }
+
+  taosRLockLatch(&pStb->lock);
+
+  if (pStbVer->sversion != pStb->colVer || pStbVer->tversion != pStb->tagVer) {
+    *schema = true;
+  } else {
+    *schema = false;
+  }
+  
+  if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) {
+    *sma = true;
+  } else {
+    *sma = false;
+  }
+
+  taosRUnLockLatch(&pStb->lock);
+
+  mndReleaseDb(pMnode, pDb);
+  mndReleaseStb(pMnode, pStb);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
   char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
   snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
 
@@ -1921,10 +1964,6 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char
     return -1;
   }
 
-  if (smaVer) {
-    *smaVer = pStb->smaVer;
-  }
-
   int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp);
   mndReleaseDb(pMnode, pDb);
   mndReleaseStb(pMnode, pStb);
@@ -2563,7 +2602,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) {
     }
   } else {
     mInfo("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName);
-    if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) {
+    if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) {
       goto _OVER;
     }
   }
@@ -2674,14 +2713,15 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
   for (int32_t i = 0; i < numOfStbs; ++i) {
     SSTableVersion *pStbVersion = &pStbVersions[i];
     pStbVersion->suid = be64toh(pStbVersion->suid);
-    pStbVersion->sversion = ntohs(pStbVersion->sversion);
+    pStbVersion->sversion = ntohl(pStbVersion->sversion);
-    pStbVersion->tversion = ntohs(pStbVersion->tversion);
+    pStbVersion->tversion = ntohl(pStbVersion->tversion);
     pStbVersion->smaVer = ntohl(pStbVersion->smaVer);
 
+    bool schema = false;
+    bool sma = false;
+    int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma);
+    if (TSDB_CODE_SUCCESS != code) {
       STableMetaRsp metaRsp = {0};
-    int32_t       smaVer = 0;
-    mInfo("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName);
-    if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) {
       metaRsp.numOfColumns = -1;
       metaRsp.suid = pStbVersion->suid;
       tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName));
@@ -2691,13 +2731,23 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
       continue;
     }
 
-    if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) {
+    if (schema) {
+      STableMetaRsp metaRsp = {0};
+      mInfo("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName);
+      if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) {
+        metaRsp.numOfColumns = -1;
+        metaRsp.suid = pStbVersion->suid;
+        tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName));
+        tstrncpy(metaRsp.tbName, pStbVersion->stbName, sizeof(metaRsp.tbName));
+        tstrncpy(metaRsp.stbName, pStbVersion->stbName, sizeof(metaRsp.stbName));
         taosArrayPush(hbRsp.pMetaRsp, &metaRsp);
-    } else {
+        continue;
-      tFreeSTableMetaRsp(&metaRsp);
       }
 
-    if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) {
+      taosArrayPush(hbRsp.pMetaRsp, &metaRsp);
+    }
+
+    if (sma) {
       bool           exist = false;
       char           tbFName[TSDB_TABLE_FNAME_LEN];
       STableIndexRsp indexRsp = {0};

source/dnode/mnode/impl/src/mndStream.c

@@ -42,14 +42,14 @@ typedef struct SNodeEntry {
   int64_t hbTimestamp;  // second
 } SNodeEntry;
 
-typedef struct SStreamExecNodeInfo {
+typedef struct SStreamExecInfo {
   SArray       *pNodeEntryList;
   int64_t       ts;                // snapshot ts
   int64_t       activeCheckpoint;  // active check point id
   SHashObj     *pTaskMap;
   SArray       *pTaskList;
   TdThreadMutex lock;
-} SStreamExecNodeInfo;
+} SStreamExecInfo;
 
 typedef struct SVgroupChangeInfo {
   SHashObj *pDBMap;
@@ -57,7 +57,7 @@ typedef struct SVgroupChangeInfo {
 } SVgroupChangeInfo;
 
 static int32_t         mndNodeCheckSentinel = 0;
-static SStreamExecNodeInfo     execNodeList;
+static SStreamExecInfo execInfo;
 
 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
@@ -77,19 +77,21 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
                                                   int64_t streamId, int32_t taskId);
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
 static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
-
 static SArray *extractNodeListFromStream(SMnode *pMnode);
-static SArray           *mndTakeVgroupSnapshot(SMnode *pMnode);
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady);
+
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
 
 static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name);
 static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans);
-static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset);
+static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo);
+                            int32_t retryCode);
-
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
-static void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode);
+static void    removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
-static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode);
+static void    keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
 static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
+static int32_t doKillActiveCheckpointTrans(SMnode *pMnode);
+static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList);
 
 int32_t mndInitStream(SMnode *pMnode) {
   SSdbTable table = {
@@ -129,18 +131,18 @@ int32_t mndInitStream(SMnode *pMnode) {
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
 
-  taosThreadMutexInit(&execNodeList.lock, NULL);
+  taosThreadMutexInit(&execInfo.lock, NULL);
-  execNodeList.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
+  execInfo.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
-  execNodeList.pTaskList = taosArrayInit(4, sizeof(STaskId));
+  execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
 
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupStream(SMnode *pMnode) {
-  taosArrayDestroy(execNodeList.pTaskList);
+  taosArrayDestroy(execInfo.pTaskList);
-  taosHashCleanup(execNodeList.pTaskMap);
+  taosHashCleanup(execInfo.pTaskMap);
-  taosThreadMutexDestroy(&execNodeList.lock);
+  taosThreadMutexDestroy(&execInfo.lock);
-  mDebug("mnd stream cleanup");
+  mDebug("mnd stream exec info cleanup");
 }
 
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@@ -286,7 +288,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
 static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
   int8_t status = atomic_load_8(&pStream->status);
   if (status == STREAM_STATUS__NORMAL) {
-    strcpy(dst, "normal");
+    strcpy(dst, "ready");
   } else if (status == STREAM_STATUS__STOP) {
     strcpy(dst, "stop");
   } else if (status == STREAM_STATUS__FAILED) {
@@ -294,7 +296,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
   } else if (status == STREAM_STATUS__RECOVER) {
     strcpy(dst, "recover");
   } else if (status == STREAM_STATUS__PAUSE) {
-    strcpy(dst, "pause");
+    strcpy(dst, "paused");
   }
 }
 
@@ -516,7 +518,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
 
   STransAction action = {0};
   action.mTraceId = pTrans->mTraceId;
-  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet);
+  initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(buf);
     return -1;
@@ -688,7 +690,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -847,20 +849,20 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   mndTransDrop(pTrans);
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
   mDebug("register to stream task node list");
-  keepStreamTasksInBuf(&streamObj, &execNodeList);
+  keepStreamTasksInBuf(&streamObj, &execInfo);
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
   SName name = {0};
-  tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB);
+  tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
   //reuse this function for stream
 
   //TODO
   if (createStreamReq.sql != NULL) {
-    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, "",
+    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, 
                 createStreamReq.sql, strlen(createStreamReq.sql));
   }
 _OVER:
@@ -882,9 +884,8 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
     return 0;
   }
 
-  int64_t                  checkpointId = taosGetTimestampMs();
   SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = checkpointId;
+  pMsg->checkpointId = taosGetTimestampMs();
 
   int32_t size = sizeof(SMStreamDoCheckpointMsg);
   SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@@ -936,7 +937,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
 }
 // static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) {
 //   int64_t timestampMs = taosGetTimestampMs();
-//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) {
+//   if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) {
 //     return -1;
 //   }
 
@@ -1069,7 +1070,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
 
         STransAction action = {0};
         SEpSet       epset = mndGetVgroupEpset(pMnode, pVgObj);
-        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset);
+        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
         mndReleaseVgroup(pMnode, pVgObj);
 
         if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -1084,6 +1085,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
   pStream->checkpointId = chkptId;
   pStream->checkpointFreq = taosGetTimestampMs();
   pStream->currentTick = 0;
+
   // 3. commit log: stream checkpoint info
   pStream->version = pStream->version + 1;
 
@@ -1133,31 +1135,37 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
   {  // check if the node update happens or not
     int64_t ts = taosGetTimestampSec();
 
-    if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
+    if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
-      if (execNodeList.pNodeEntryList != NULL) {
+      if (execInfo.pNodeEntryList != NULL) {
-        execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+        execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
       }
 
-      execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
+      execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
     }
 
-    if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
+    if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
       mDebug("stream task node change checking done, no vgroups exist, do nothing");
-      execNodeList.ts = ts;
+      execInfo.ts = ts;
       return 0;
     }
 
-    for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
+    for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
-      SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i);
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
       if (pNodeEntry->stageUpdated) {
         mDebug("stream task not ready due to node update detected, checkpoint not issued");
         return 0;
       }
     }
 
-    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+    bool allReady = true;
+    SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
+    if (!allReady) {
+      mWarn("not all vnodes are ready, ignore the checkpoint")
+      taosArrayDestroy(pNodeSnapshot);
+      return 0;
+    }
 
-    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+    SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
     bool              nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0);
     taosArrayDestroy(changeInfo.pUpdateNodeList);
     taosHashCleanup(changeInfo.pDBMap);
@@ -1169,26 +1177,25 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
     }
   }
 
-  {  // check if all tasks are in TASK_STATUS__NORMAL status
+  {  // check if all tasks are in TASK_STATUS__READY status
     bool ready = true;
 
-    taosThreadMutexLock(&execNodeList.lock);
+    taosThreadMutexLock(&execInfo.lock);
-    for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) {
+    for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
-      STaskId *p = taosArrayGet(execNodeList.pTaskList, i);
+      STaskId *p = taosArrayGet(execInfo.pTaskList, i);
-      STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p));
+      STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
       if (pEntry == NULL) {
         continue;
       }
 
-      if (pEntry->status != TASK_STATUS__NORMAL) {
+      if (pEntry->status != TASK_STATUS__READY) {
         mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
-              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status));
+              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
         ready = false;
         break;
       }
     }
-    taosThreadMutexUnlock(&execNodeList.lock);
+    taosThreadMutexUnlock(&execInfo.lock);
-
     if (!ready) {
       return 0;
     }
@@ -1202,7 +1209,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
     mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
     return -1;
   }
-  mDebug("start to trigger checkpoint, checkpointId: %" PRId64 "", checkpointId);
+
+  mDebug("start to trigger checkpoint, checkpointId: %" PRId64, checkpointId);
 
   const char *pDb = mndGetStreamDB(pMnode);
   mndTransSetDbName(pTrans, pDb, "checkpoint");
@@ -1228,11 +1236,16 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
 
   if (code == 0) {
     if (mndTransPrepare(pMnode, pTrans) != 0) {
-      mError("failed to prepre trans rebalance since %s", terrstr());
+      mError("failed to prepare trans rebalance since %s", terrstr());
     }
   }
 
   mndTransDrop(pTrans);
+
+  // only one trans here
+  taosThreadMutexLock(&execInfo.lock);
+  execInfo.activeCheckpoint = checkpointId;
+  taosThreadMutexUnlock(&execInfo.lock);
   return code;
 }
 
@@ -1310,13 +1323,13 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  removeStreamTasksInBuf(pStream, &execNodeList);
+  removeStreamTasksInBuf(pStream, &execInfo);
 
   SName name = {0};
-  tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB);
+  tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
   //reuse this function for stream
 
-  auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);
 
   sdbRelease(pMnode->pSdb, pStream);
   mndTransDrop(pTrans);
@@ -1561,12 +1574,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         char status[20 + VARSTR_HEADER_SIZE] = {0};
 
         STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-        STaskStatusEntry* pe = taosHashGet(execNodeList.pTaskMap, &id, sizeof(id));
+        STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
         if (pe == NULL) {
           continue;
         }
 
-        const char* pStatus = streamGetTaskStatusStr(pe->status);
+        const char* pStatus = streamTaskGetStatusStr(pe->status);
         STR_TO_VARSTR(status, pStatus);
 
         // status
@@ -1641,7 +1654,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -1774,7 +1787,7 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig
   pReq->igUntreated = igUntreated;
 
   STransAction action = {0};
-  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet);
+  initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet, 0);
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);
     return -1;
@@ -1878,18 +1891,19 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
-static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, int64_t streamId,
+static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
-                              int32_t taskId) {
+                              int32_t transId) {
-  pMsg->streamId = streamId;
+  pMsg->streamId = pId->streamId;
-  pMsg->taskId = taskId;
+  pMsg->taskId = pId->taskId;
+  pMsg->transId = transId;
   pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
   taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
 }
 
 static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
-                                          int64_t streamId, int32_t taskId) {
+                                          SStreamTaskId* pId, int32_t transId) {
   SStreamTaskNodeUpdateMsg req = {0};
-  initNodeUpdateMsg(&req, pInfo, streamId, taskId);
+  initNodeUpdateMsg(&req, pInfo, pId, transId);
 
   int32_t code = 0;
   int32_t blen;
@@ -1953,20 +1967,19 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans) {
   return 0;
 }
 
-void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset) {
+void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
+                     int32_t retryCode) {
   pAction->epSet = *pEpset;
   pAction->contLen = contLen;
   pAction->pCont = pCont;
   pAction->msgType = msgType;
+  pAction->retryCode = retryCode;
 }
 
 // todo extract method: traverse stream tasks
 // build trans to update the epset
-static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo) {
+static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
-  STrans* pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
+  mDebug("start to build stream:0x%" PRIx64 " tasks epset update", pStream->uid);
-  if (pTrans == NULL) {
-    return terrno;
-  }
 
   taosWLockLatch(&pStream->lock);
   int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
@@ -1981,38 +1994,20 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr
       void   *pBuf = NULL;
       int32_t len = 0;
       streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
-      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId);
+      doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
 
       STransAction action = {0};
-      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet);
+      initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
       if (mndTransAppendRedoAction(pTrans, &action) != 0) {
         taosMemoryFree(pBuf);
         taosWUnLockLatch(&pStream->lock);
-        mndTransDrop(pTrans);
         return -1;
       }
     }
   }
 
   taosWUnLockLatch(&pStream->lock);
-
+  return 0;
-  int32_t code = mndPersistTransLog(pStream, pTrans);
-  if (code != TSDB_CODE_SUCCESS) {
-    sdbRelease(pMnode->pSdb, pStream);
-    return -1;
-  }
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
-    sdbRelease(pMnode->pSdb, pStream);
-    mndTransDrop(pTrans);
-    return -1;
-  }
-
-  sdbRelease(pMnode->pSdb, pStream);
-  mndTransDrop(pTrans);
-
-  return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
 static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) {
@@ -2050,8 +2045,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
 
           char buf[256] = {0};
           EPSET_TO_STR(&pCurrent->epset, buf);
-          mDebug("nodeId:%d epset changed detected, old:%s:%d -> new:%s", pCurrent->nodeId, pPrevEp->fqdn,
+          mDebug("nodeId:%d restart/epset changed detected, old:%s:%d -> new:%s, stageUpdate:%d", pCurrent->nodeId,
-                 pPrevEp->port, buf);
+                 pPrevEp->fqdn, pPrevEp->port, buf, pPrevEntry->stageUpdated);
 
           SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId};
           epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset);
@@ -2071,11 +2066,12 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
   return info;
 }
 
-static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
   SVgObj *pVgroup = NULL;
 
+  *allReady = true;
   SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry));
 
   while (1) {
@@ -2087,8 +2083,26 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
     SNodeEntry entry = {0};
     entry.epset = mndGetVgroupEpset(pMnode, pVgroup);
     entry.nodeId = pVgroup->vgId;
-    entry.hbTimestamp = -1;
+    entry.hbTimestamp = pVgroup->updateTime;
 
+    if (*allReady) {
+      for (int32_t i = 0; i < pVgroup->replica; ++i) {
+        if (!pVgroup->vnodeGid[i].syncRestore) {
+          *allReady = false;
+          break;
+        }
+
+        ESyncState state = pVgroup->vnodeGid[i].syncState;
+        if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR) {
+          *allReady = false;
+          break;
+        }
+      }
+    }
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&entry.epset, buf);
+    mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
     taosArrayPush(pVgroupListSnapshot, &entry);
     sdbRelease(pSdb, pVgroup);
   }
@@ -2102,27 +2116,63 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
   // check all streams that involved this vnode should update the epset info
   SStreamObj *pStream = NULL;
   void       *pIter = NULL;
+  STrans     *pTrans = NULL;
+
   while (1) {
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) {
       break;
     }
 
+    // here create only one trans
+    if (pTrans == NULL) {
+      pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
+      if (pTrans == NULL) {
+        sdbRelease(pSdb, pStream);
+        sdbCancelFetch(pSdb, pIter);
+        return terrno;
+      }
+    }
+
     void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb));
     void *p1 = taosHashGet(pChangeInfo->pDBMap, pStream->sourceDb, strlen(pStream->sourceDb));
     if (p == NULL && p1 == NULL) {
-      mndReleaseStream(pMnode, pStream);
+      mDebug("stream:0x%" PRIx64 " %s not involved nodeUpdate, ignore", pStream->uid, pStream->name);
+      sdbRelease(pSdb, pStream);
       continue;
     }
 
-    mDebug("stream:0x%" PRIx64 " involved node changed, create update trans", pStream->uid);
+    mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
-    int32_t code = createStreamUpdateTrans(pMnode, pStream, pChangeInfo);
+           pStream->name, pTrans->id);
+
+    int32_t code = createStreamUpdateTrans(pStream, pChangeInfo, pTrans);
+
+    // todo: not continue, drop all and retry again
+    if (code != TSDB_CODE_SUCCESS) {
+      mError("stream:0x%" PRIx64 " build nodeUpdate trans failed, ignore and continue, code:%s", pStream->uid,
+             tstrerror(code));
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    code = mndPersistTransLog(pStream, pTrans);
+    sdbRelease(pSdb, pStream);
+
     if (code != TSDB_CODE_SUCCESS) {
       sdbCancelFetch(pSdb, pIter);
-      return code;
+      return -1;
     }
   }
 
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
+    sdbRelease(pMnode->pSdb, pStream);
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  sdbRelease(pMnode->pSdb, pStream);
+  mndTransDrop(pTrans);
   return 0;
 }
 
@@ -2165,6 +2215,10 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
   while ((pIter = taosHashIterate(pHash, pIter)) != NULL) {
     SNodeEntry *pEntry = (SNodeEntry *)pIter;
     taosArrayPush(plist, pEntry);
+
+    char buf[256] = {0};
+    EPSET_TO_STR(&pEntry->epset, buf);
+    mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf);
   }
   taosHashCleanup(pHash);
 
@@ -2182,28 +2236,31 @@ static void doExtractTasksFromStream(SMnode *pMnode) {
       break;
     }
 
-    keepStreamTasksInBuf(pStream, &execNodeList);
+    keepStreamTasksInBuf(pStream, &execInfo);
     sdbRelease(pSdb, pStream);
   }
 }
 
-static int32_t doRemoveFromTask(SStreamExecNodeInfo* pExecNode, STaskId* pRemovedId) {
+static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
   void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
+  if (p == NULL) {
+    return TSDB_CODE_SUCCESS;
+  }
 
-  if (p != NULL) {
   taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
 
-    for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+  for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
-      STaskId* pId = taosArrayGet(pExecNode->pTaskList, k);
+    STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
     if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) {
       taosArrayRemove(pExecNode->pTaskList, k);
-        mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t) pRemovedId->taskId,
+
-              (int32_t)taosArrayGetSize(pExecNode->pTaskList));
+      int32_t num = taosArrayGetSize(pExecNode->pTaskList);
+      mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)pRemovedId->taskId, num);
       break;
     }
   }
-  }
+
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 static bool taskNodeExists(SArray* pList, int32_t nodeId) {
@@ -2220,31 +2277,31 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) {
 }
 
 int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
-  SArray* pRemoveTaskList = taosArrayInit(4, sizeof(STaskId));
+  SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
 
-  int32_t numOfTask = taosArrayGetSize(execNodeList.pTaskList);
+  int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
   for(int32_t i = 0; i < numOfTask; ++i) {
-    STaskId* pId = taosArrayGet(execNodeList.pTaskList, i);
+    STaskId* pId = taosArrayGet(execInfo.pTaskList, i);
-    STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, pId, sizeof(*pId));
+    STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
 
     bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
     if (!existed) {
-      taosArrayPush(pRemoveTaskList, pId);
+      taosArrayPush(pRemovedTasks, pId);
     }
   }
 
-  for(int32_t i = 0; i < taosArrayGetSize(pRemoveTaskList); ++i) {
+  for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
-    STaskId* pId = taosArrayGet(pRemoveTaskList, i);
+    STaskId* pId = taosArrayGet(pRemovedTasks, i);
-    doRemoveFromTask(&execNodeList, pId);
+    doRemoveTasks(&execInfo, pId);
   }
 
-  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemoveTaskList),
+  mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
-         (int32_t) taosArrayGetSize(execNodeList.pTaskList));
+         (int32_t) taosArrayGetSize(execInfo.pTaskList));
 
   int32_t size = taosArrayGetSize(pNodeSnapshot);
   SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
-  for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) {
+  for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
-    SNodeEntry* p = taosArrayGet(execNodeList.pNodeEntryList, i);
+    SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i);
 
     for(int32_t j = 0; j < size; ++j) {
       SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j);
@@ -2255,10 +2312,11 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
     }
   }
 
-  execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+  execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
-  execNodeList.pNodeEntryList = pValidNodeEntryList;
+  execInfo.pNodeEntryList = pValidNodeEntryList;
 
-  taosArrayDestroy(pRemoveTaskList);
+  mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList));
+  taosArrayDestroy(pRemovedTasks);
   return 0;
 }
 
@@ -2275,43 +2333,55 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
   int64_t ts = taosGetTimestampSec();
 
   SMnode *pMnode = pMsg->info.node;
-  if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) {
+  if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
-    if (execNodeList.pNodeEntryList != NULL) {
+    if (execInfo.pNodeEntryList != NULL) {
-      execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList);
+      execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
+    }
+    execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
   }
 
-    execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
+  if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
-  }
-
-  if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) {
     mDebug("end to do stream task node change checking, no vgroup exists, do nothing");
-    execNodeList.ts = ts;
+    execInfo.ts = ts;
     atomic_store_32(&mndNodeCheckSentinel, 0);
     return 0;
   }
 
-  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode);
+  bool allVnodeReady = true;
+  SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady);
+  if (!allVnodeReady) {
+    taosArrayDestroy(pNodeSnapshot);
+    atomic_store_32(&mndNodeCheckSentinel, 0);
+    mWarn("not all vnodes are ready, ignore the exec nodeUpdate check");
+    return 0;
+  }
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
   removeExpirednodeEntryAndTask(pNodeSnapshot);
 
-  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot);
+  SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
   if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
+
+    // kill current active checkpoint transaction, since the transaction is vnode wide.
+    doKillActiveCheckpointTrans(pMnode);
     code = mndProcessVgroupChange(pMnode, &changeInfo);
 
     // keep the new vnode snapshot
     if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) {
       mDebug("create trans successfully, update cached node list");
-      taosArrayDestroy(execNodeList.pNodeEntryList);
+      taosArrayDestroy(execInfo.pNodeEntryList);
-      execNodeList.pNodeEntryList = pNodeSnapshot;
+      execInfo.pNodeEntryList = pNodeSnapshot;
-      execNodeList.ts = ts;
+      execInfo.ts = ts;
+    } else {
+      mDebug("unexpect code during create nodeUpdate trans, code:%s", tstrerror(code));
+      taosArrayDestroy(pNodeSnapshot);
     }
   } else {
     mDebug("no update found in nodeList");
     taosArrayDestroy(pNodeSnapshot);
   }
 
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
   taosArrayDestroy(changeInfo.pUpdateNodeList);
   taosHashCleanup(changeInfo.pDBMap);
 
@@ -2324,10 +2394,6 @@ typedef struct SMStreamNodeCheckMsg {
   int8_t placeHolder;  // // to fix windows compile error, define place holder
 } SMStreamNodeCheckMsg;
 
-typedef struct SMStreamTaskResetMsg {
-  int8_t placeHolder;
-} SMStreamTaskResetMsg;
-
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
   SMnode *pMnode = pReq->info.node;
   SSdb   *pSdb = pMnode->pSdb;
@@ -2343,7 +2409,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
   return 0;
 }
 
-void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
+void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
   int32_t level = taosArrayGetSize(pStream->tasks);
 
   for (int32_t i = 0; i < level; i++) {
@@ -2368,7 +2434,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
   }
 }
 
-void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode) {
+void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
   int32_t level = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < level; i++) {
     SArray *pLevel = taosArrayGetP(pStream->tasks, i);
@@ -2399,7 +2465,7 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode
   ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
 }
 
-static STrans* doCreateTrans(SMnode* pMnode, SStreamObj* pStream, const char* name) {
+STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) {
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, name);
   if (pTrans == NULL) {
     mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@@ -2451,7 +2517,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
       pReq->streamId = pTask->id.streamId;
 
       STransAction action = {0};
-      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet);
+      initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet, 0);
       if (mndTransAppendRedoAction(pTrans, &action) != 0) {
         taosMemoryFree(pReq);
         taosWUnLockLatch(&pStream->lock);
@@ -2482,14 +2548,12 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
-int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
-  // find the checkpoint trans id
   int32_t transId = 0;
-
-  {
   SSdb   *pSdb = pMnode->pSdb;
   STrans *pTrans = NULL;
-    void* pIter = NULL;
+  void   *pIter = NULL;
+
   while (1) {
     pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
     if (pIter == NULL) {
@@ -2505,15 +2569,22 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
 
     sdbRelease(pSdb, pTrans);
   }
-  }
 
   if (transId == 0) {
     mError("failed to find the checkpoint trans, reset not executed");
     return TSDB_CODE_SUCCESS;
   }
 
-  STrans* pTrans = mndAcquireTrans(pMnode, transId);
+  pTrans = mndAcquireTrans(pMnode, transId);
+  mInfo("kill checkpoint trans:%d", transId);
+
   mndKillTrans(pMnode, pTrans);
+  mndReleaseTrans(pMnode, pTrans);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+  doKillActiveCheckpointTrans(pMnode);
 
   // set all tasks status to be normal, refactor later to be stream level, instead of vnode level.
   SSdb *pSdb = pMnode->pSdb;
@@ -2525,6 +2596,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
       break;
     }
 
+    // todo this transaction should exist be only one
     mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid);
     int32_t code = createStreamResetStatusTrans(pMnode, pStream);
     if (code != TSDB_CODE_SUCCESS) {
@@ -2536,6 +2608,43 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
   return 0;
 }
 
+int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) {
+  int32_t num = taosArrayGetSize(pNodeList);
+
+  for (int k = 0; k < num; ++k) {
+    int32_t* pVgId = taosArrayGet(pNodeList, k);
+
+    int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+    for (int i = 0; i < numOfNodes; ++i) {
+      SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
+
+      if (pNodeEntry->nodeId == *pVgId) {
+        mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId);
+        pNodeEntry->stageUpdated = true;
+        break;
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
+  int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
+  for(int32_t j = 0; j < numOfNodes; ++j) {
+    SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j);
+    if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
+
+      mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
+            pTaskEntry->stage, stage, pTaskEntry->id.taskId);
+
+      pNodeEntry->stageUpdated = true;
+      pTaskEntry->stage = stage;
+      break;
+    }
+  }
+}
+
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
@@ -2555,35 +2664,26 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
 
   mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
 
-  taosThreadMutexLock(&execNodeList.lock);
+  taosThreadMutexLock(&execInfo.lock);
-  int32_t numOfExisted = taosHashGetSize(execNodeList.pTaskMap);
+  int32_t numOfExisted = taosHashGetSize(execInfo.pTaskMap);
   if (numOfExisted == 0) {
     doExtractTasksFromStream(pMnode);
   }
 
+  setNodeEpsetExpiredFlag(req.pUpdateNodes);
+
   for (int32_t i = 0; i < req.numOfTasks; ++i) {
     STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i);
-    STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, &p->id, sizeof(p->id));
+    STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id));
-    if (pEntry == NULL) {
+    if (pTaskEntry == NULL) {
       mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
       continue;
     }
 
-    if (p->stage != pEntry->stage && pEntry->stage != -1) {
+    if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) {
-      int32_t numOfNodes = taosArrayGetSize(execNodeList.pNodeEntryList);
+      updateStageInfo(pTaskEntry, p->stage);
-      for(int32_t j = 0; j < numOfNodes; ++j) {
-        SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, j);
-        if (pNodeEntry->nodeId == pEntry->nodeId) {
-          mInfo("vgId:%d stage updated, from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64,
-                pEntry->nodeId, pEntry->stage, p->stage, pEntry->id.taskId);
-
-          pNodeEntry->stageUpdated = true;
-          pEntry->stage = p->stage;
-          break;
-        }
-      }
     } else {
-      streamTaskStatusCopy(pEntry, p);
+      streamTaskStatusCopy(pTaskEntry, p);
       if (p->activeCheckpointId != 0) {
         if (activeCheckpointId != 0) {
           ASSERT(activeCheckpointId == p->activeCheckpointId);
@@ -2597,26 +2697,32 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
       }
     }
 
-    pEntry->status = p->status;
+    pTaskEntry->status = p->status;
-    if (p->status != TASK_STATUS__NORMAL) {
+    if (p->status != TASK_STATUS__READY) {
-      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamGetTaskStatusStr(p->status));
+      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
     }
   }
 
   // current checkpoint is failed, rollback from the checkpoint trans
   // kill the checkpoint trans and then set all tasks status to be normal
   if (checkpointFailed && activeCheckpointId != 0) {
-    if (execNodeList.activeCheckpoint != activeCheckpointId) {
+    bool allReady = true;
-      mInfo("checkpointId:%"PRId64" failed, issue task-reset trans to reset all tasks status", activeCheckpointId);
+    SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady);
-      execNodeList.activeCheckpoint = activeCheckpointId;
+    taosArrayDestroy(p);
+
+    if (allReady) {
+      // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
+      mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status",
+            execInfo.activeCheckpoint);
       mndResetFromCheckpoint(pMnode);
     } else {
-      mDebug("checkpoint:%"PRId64" reset has issued already, ignore it", activeCheckpointId);
+      mInfo("not all vgroups are ready, wait for next HB from stream tasks");
     }
   }
 
-  taosThreadMutexUnlock(&execNodeList.lock);
+  taosThreadMutexUnlock(&execInfo.lock);
 
   taosArrayDestroy(req.pTaskStatus);
+  taosArrayDestroy(req.pUpdateNodes);
   return TSDB_CODE_SUCCESS;
 }

source/dnode/mnode/impl/src/mndSync.c

@@ -95,7 +95,11 @@ static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STr
   }
 
 _OUT:
+  if (pRow) {
+    SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
+    if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
     taosMemoryFreeClear(pRow);
+  }
   return code;
 }
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -611,10 +611,10 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
   tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB);
 
   SName topicName = {0};
-  tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB);
+  tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
   //reuse this function for topic
 
-  auditRecord(pReq, pMnode->clusterId, "createTopic", topicName.dbname, dbname.dbname, 
+  auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname, 
               createTopicReq.sql, strlen(createTopicReq.sql));
 
 _OVER:
@@ -817,10 +817,10 @@ end:
   }
 
   SName name = {0};
-  tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB);
+  tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
   //reuse this function for topic
 
-  auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);
 
   tFreeSMDropTopicReq(&dropReq);
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -720,7 +720,7 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c
 
 void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) {
   if (dbname != NULL) {
-    tstrncpy(pTrans->dbname, dbname, TSDB_TABLE_FNAME_LEN);
+    tstrncpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN);
   }
   if (stbname != NULL) {
     tstrncpy(pTrans->stbname, stbname, TSDB_TABLE_FNAME_LEN);

source/dnode/mnode/impl/src/mndUser.c

@@ -28,9 +28,53 @@
 
 // clang-format on
 
-#define USER_VER_NUMBER   5
+#define USER_VER_NUMBER   6
 #define USER_RESERVE_SIZE 64
 
+#define BIT_FLAG_MASK(n)              (1 << n)
+#define BIT_FLAG_SET_MASK(val, mask)  ((val) |= (mask))
+#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
+
+#define PRIVILEGE_TYPE_ALL       BIT_FLAG_MASK(0)
+#define PRIVILEGE_TYPE_READ      BIT_FLAG_MASK(1)
+#define PRIVILEGE_TYPE_WRITE     BIT_FLAG_MASK(2)
+#define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3)
+#define PRIVILEGE_TYPE_ALTER     BIT_FLAG_MASK(4)
+
+#define ALTER_USER_ADD_PRIVS(_type) ((_type) == TSDB_ALTER_USER_ADD_PRIVILEGES)
+#define ALTER_USER_DEL_PRIVS(_type) ((_type) == TSDB_ALTER_USER_DEL_PRIVILEGES)
+
+#define ALTER_USER_ALL_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
+#define ALTER_USER_READ_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_READ) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
+#define ALTER_USER_WRITE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_WRITE) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
+#define ALTER_USER_ALTER_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALTER) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
+#define ALTER_USER_SUBSCRIBE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_SUBSCRIBE))
+
+#define ALTER_USER_TARGET_DB(_tbname) (0 == (_tbname)[0])
+#define ALTER_USER_TARGET_TB(_tbname) (0 != (_tbname)[0])
+
+#define ALTER_USER_ADD_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_DEL_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_ADD_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_DEL_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_ADD_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_DEL_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_ADD_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+#define ALTER_USER_DEL_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
+
+#define ALTER_USER_ADD_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_DEL_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_ADD_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_DEL_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_ADD_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_DEL_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_ADD_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+#define ALTER_USER_DEL_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
+
+#define ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
+#define ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
+
+
 static SIpWhiteList *createDefaultIpWhiteList();
 SIpWhiteList        *createIpWhiteList(void *buf, int32_t len);
 static bool          updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew);
@@ -656,8 +700,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
       pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16;
   int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
   int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
-  int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs);
+  int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
-  int32_t numOfWriteStbs = taosHashGetSize(pUser->writeTbs);
+  int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
+  int32_t numOfAlterTbs = taosHashGetSize(pUser->alterTbs);
+  int32_t numOfReadViews = taosHashGetSize(pUser->readViews);
+  int32_t numOfWriteViews = taosHashGetSize(pUser->writeViews);
+  int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews);
   int32_t numOfTopics = taosHashGetSize(pUser->topics);
   int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs);
   int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE +
@@ -692,6 +740,62 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
     stb = taosHashIterate(pUser->writeTbs, stb);
   }
 
+  stb = taosHashIterate(pUser->alterTbs, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    size += sizeof(int32_t);
+    size += keyLen;
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb);
+    size += sizeof(int32_t);
+    size += valueLen;
+    stb = taosHashIterate(pUser->alterTbs, stb);
+  }
+
+  stb = taosHashIterate(pUser->readViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    size += sizeof(int32_t);
+    size += keyLen;
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb);
+    size += sizeof(int32_t);
+    size += valueLen;
+    stb = taosHashIterate(pUser->readViews, stb);
+  }
+
+  stb = taosHashIterate(pUser->writeViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    size += sizeof(int32_t);
+    size += keyLen;
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb);
+    size += sizeof(int32_t);
+    size += valueLen;
+    stb = taosHashIterate(pUser->writeViews, stb);
+  }
+
+  stb = taosHashIterate(pUser->alterViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    size += sizeof(int32_t);
+    size += keyLen;
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb);
+    size += sizeof(int32_t);
+    size += valueLen;
+    stb = taosHashIterate(pUser->alterViews, stb);
+  }  
+
   SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
   if (pRaw == NULL) goto _OVER;
 
@@ -729,8 +833,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
     topic = taosHashIterate(pUser->topics, topic);
   }
 
-  SDB_SET_INT32(pRaw, dataPos, numOfReadStbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfReadTbs, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, numOfWriteStbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfWriteTbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfAlterTbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfReadViews, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfWriteViews, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfAlterViews, _OVER)
   SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER)
 
   stb = taosHashIterate(pUser->readTbs, NULL);
@@ -761,6 +869,62 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
     stb = taosHashIterate(pUser->writeTbs, stb);
   }
 
+  stb = taosHashIterate(pUser->alterTbs, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb) + 1;
+    SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
+    stb = taosHashIterate(pUser->alterTbs, stb);
+  }
+
+  stb = taosHashIterate(pUser->readViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb) + 1;
+    SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
+    stb = taosHashIterate(pUser->readViews, stb);
+  }
+
+  stb = taosHashIterate(pUser->writeViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb) + 1;
+    SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
+    stb = taosHashIterate(pUser->writeViews, stb);
+  }
+
+  stb = taosHashIterate(pUser->alterViews, NULL);
+  while (stb != NULL) {
+    size_t keyLen = 0;
+    void  *key = taosHashGetKey(stb, &keyLen);
+    SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+    size_t valueLen = 0;
+    valueLen = strlen(stb) + 1;
+    SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
+    SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
+    stb = taosHashIterate(pUser->alterViews, stb);
+  }
+
   int32_t *useDb = taosHashIterate(pUser->useDbs, NULL);
   while (useDb != NULL) {
     size_t keyLen = 0;
@@ -873,20 +1037,40 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
   }
 
   if (sver >= 3) {
-    int32_t numOfReadStbs = 0;
+    int32_t numOfReadTbs = 0;
-    int32_t numOfWriteStbs = 0;
+    int32_t numOfWriteTbs = 0;
+    int32_t numOfAlterTbs = 0;
+    int32_t numOfReadViews = 0;
+    int32_t numOfWriteViews = 0;
+    int32_t numOfAlterViews = 0;
     int32_t numOfUseDbs = 0;
-    SDB_GET_INT32(pRaw, dataPos, &numOfReadStbs, _OVER)
+    SDB_GET_INT32(pRaw, dataPos, &numOfReadTbs, _OVER)
-    SDB_GET_INT32(pRaw, dataPos, &numOfWriteStbs, _OVER)
+    SDB_GET_INT32(pRaw, dataPos, &numOfWriteTbs, _OVER)
+    if (sver >= 6) {
+      SDB_GET_INT32(pRaw, dataPos, &numOfAlterTbs, _OVER)
+      SDB_GET_INT32(pRaw, dataPos, &numOfReadViews, _OVER)
+      SDB_GET_INT32(pRaw, dataPos, &numOfWriteViews, _OVER)
+      SDB_GET_INT32(pRaw, dataPos, &numOfAlterViews, _OVER)
+    }
     SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER)
 
     pUser->readTbs =
-        taosHashInit(numOfReadStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+        taosHashInit(numOfReadTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
     pUser->writeTbs =
-        taosHashInit(numOfWriteStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+        taosHashInit(numOfWriteTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    pUser->alterTbs =
+        taosHashInit(numOfAlterTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
+    pUser->readViews =
+        taosHashInit(numOfReadViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    pUser->writeViews =
+        taosHashInit(numOfWriteViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+    pUser->alterViews =
+        taosHashInit(numOfAlterViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
     pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
 
-    for (int32_t i = 0; i < numOfReadStbs; ++i) {
+    for (int32_t i = 0; i < numOfReadTbs; ++i) {
       int32_t keyLen = 0;
       SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
 
@@ -906,7 +1090,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
       taosMemoryFree(value);
     }
 
-    for (int32_t i = 0; i < numOfWriteStbs; ++i) {
+    for (int32_t i = 0; i < numOfWriteTbs; ++i) {
       int32_t keyLen = 0;
       SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
 
@@ -926,6 +1110,88 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
       taosMemoryFree(value);
     }
 
+    if (sver >= 6) {
+      for (int32_t i = 0; i < numOfAlterTbs; ++i) {
+        int32_t keyLen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
+
+        char *key = taosMemoryCalloc(keyLen, sizeof(char));
+        memset(key, 0, keyLen);
+        SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+        int32_t valuelen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
+        char *value = taosMemoryCalloc(valuelen, sizeof(char));
+        memset(value, 0, valuelen);
+        SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
+
+        taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen);
+
+        taosMemoryFree(key);
+        taosMemoryFree(value);
+      }
+
+      for (int32_t i = 0; i < numOfReadViews; ++i) {
+        int32_t keyLen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
+
+        char *key = taosMemoryCalloc(keyLen, sizeof(char));
+        memset(key, 0, keyLen);
+        SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+        int32_t valuelen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
+        char *value = taosMemoryCalloc(valuelen, sizeof(char));
+        memset(value, 0, valuelen);
+        SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
+
+        taosHashPut(pUser->readViews, key, keyLen, value, valuelen);
+
+        taosMemoryFree(key);
+        taosMemoryFree(value);
+      }
+
+      for (int32_t i = 0; i < numOfWriteViews; ++i) {
+        int32_t keyLen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
+
+        char *key = taosMemoryCalloc(keyLen, sizeof(char));
+        memset(key, 0, keyLen);
+        SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+        int32_t valuelen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
+        char *value = taosMemoryCalloc(valuelen, sizeof(char));
+        memset(value, 0, valuelen);
+        SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
+
+        taosHashPut(pUser->writeViews, key, keyLen, value, valuelen);
+
+        taosMemoryFree(key);
+        taosMemoryFree(value);
+      }
+
+      for (int32_t i = 0; i < numOfAlterViews; ++i) {
+        int32_t keyLen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
+
+        char *key = taosMemoryCalloc(keyLen, sizeof(char));
+        memset(key, 0, keyLen);
+        SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
+
+        int32_t valuelen = 0;
+        SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
+        char *value = taosMemoryCalloc(valuelen, sizeof(char));
+        memset(value, 0, valuelen);
+        SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
+
+        taosHashPut(pUser->alterViews, key, keyLen, value, valuelen);
+
+        taosMemoryFree(key);
+        taosMemoryFree(value);
+      }
+    }
+
     for (int32_t i = 0; i < numOfUseDbs; ++i) {
       int32_t keyLen = 0;
       SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
@@ -975,6 +1241,10 @@ _OVER:
       taosHashCleanup(pUser->topics);
       taosHashCleanup(pUser->readTbs);
       taosHashCleanup(pUser->writeTbs);
+      taosHashCleanup(pUser->alterTbs);
+      taosHashCleanup(pUser->readViews);
+      taosHashCleanup(pUser->writeViews);
+      taosHashCleanup(pUser->alterViews);
       taosHashCleanup(pUser->useDbs);
       taosMemoryFreeClear(pUser->pIpWhiteList);
     }
@@ -1062,6 +1332,10 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
   pNew->writeDbs = mndDupDbHash(pUser->writeDbs);
   pNew->readTbs = mndDupTableHash(pUser->readTbs);
   pNew->writeTbs = mndDupTableHash(pUser->writeTbs);
+  pNew->alterTbs = mndDupTableHash(pUser->alterTbs);
+  pNew->readViews = mndDupTableHash(pUser->readViews);
+  pNew->writeViews = mndDupTableHash(pUser->writeViews);
+  pNew->alterViews = mndDupTableHash(pUser->alterViews);
   pNew->topics = mndDupTopicHash(pUser->topics);
   pNew->useDbs = mndDupUseDbHash(pUser->useDbs);
   pNew->pIpWhiteList = cloneIpWhiteList(pUser->pIpWhiteList);
@@ -1080,6 +1354,10 @@ void mndUserFreeObj(SUserObj *pUser) {
   taosHashCleanup(pUser->topics);
   taosHashCleanup(pUser->readTbs);
   taosHashCleanup(pUser->writeTbs);
+  taosHashCleanup(pUser->alterTbs);
+  taosHashCleanup(pUser->readViews);
+  taosHashCleanup(pUser->writeViews);
+  taosHashCleanup(pUser->alterViews);
   taosHashCleanup(pUser->useDbs);
   taosMemoryFreeClear(pUser->pIpWhiteList);
   pUser->readDbs = NULL;
@@ -1087,6 +1365,10 @@ void mndUserFreeObj(SUserObj *pUser) {
   pUser->topics = NULL;
   pUser->readTbs = NULL;
   pUser->writeTbs = NULL;
+  pUser->alterTbs = NULL;
+  pUser->readViews = NULL;
+  pUser->writeViews = NULL;
+  pUser->alterViews = NULL;
   pUser->useDbs = NULL;
 }
 
@@ -1110,6 +1392,10 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
   TSWAP(pOld->topics, pNew->topics);
   TSWAP(pOld->readTbs, pNew->readTbs);
   TSWAP(pOld->writeTbs, pNew->writeTbs);
+  TSWAP(pOld->alterTbs, pNew->alterTbs);
+  TSWAP(pOld->readViews, pNew->readViews);
+  TSWAP(pOld->writeViews, pNew->writeViews);
+  TSWAP(pOld->alterViews, pNew->alterViews);
   TSWAP(pOld->useDbs, pNew->useDbs);
 
   int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range);
@@ -1275,7 +1561,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
   code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-  auditRecord(pReq, pMnode->clusterId, "createUser", createReq.user, "", createReq.sql, createReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "createUser", "", createReq.user, createReq.sql, createReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -1429,7 +1715,7 @@ static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useD
       return -1;
     }
   } else {
-    if (taosHashPut(hash, tbFName, len, "t", 2) != 0) {
+    if (taosHashPut(hash, tbFName, len, alterReq->isView ? "v" : "t", 2) != 0) {
       return -1;
     }
   }
@@ -1460,7 +1746,11 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj
 
   int32_t  dbKeyLen = strlen(alterReq->objname) + 1;
   int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen);
-  if (NULL == currRef || 1 == *currRef) {
+  if (NULL == currRef) {
+    return 0;
+  }
+  
+  if (1 == *currRef) {
     if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) {
       return -1;
     }
@@ -1481,60 +1771,162 @@ static char *mndUserAuditTypeStr(int32_t type) {
   if (type == TSDB_ALTER_USER_SUPERUSER) {
     return "changeSuperUser";
   }
-  if (type == TSDB_ALTER_USER_ADD_READ_DB) {
-    return "addReadToDB";
-  }
-  if (type == TSDB_ALTER_USER_ADD_READ_DB) {
-    return "addReadToDB";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_READ_DB) {
-    return "removeReadFromDB";
-  }
-  if (type == TSDB_ALTER_USER_ADD_WRITE_DB) {
-    return "addWriteToDB";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
-    return "removeWriteFromDB";
-  }
-  if (type == TSDB_ALTER_USER_ADD_ALL_DB) {
-    return "addToAllDB";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_ALL_DB) {
-    return "removeFromAllDB";
-  }
   if (type == TSDB_ALTER_USER_ENABLE) {
     return "enableUser";
   }
   if (type == TSDB_ALTER_USER_SYSINFO) {
     return "userSysInfo";
   }
-  if (type == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
-    return "addSubscribeTopic";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
-    return "removeSubscribeTopic";
-  }
-  if (type == TSDB_ALTER_USER_ADD_READ_TABLE) {
-    return "addReadToTable";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_READ_TABLE) {
-    return "removeReadFromTable";
-  }
-  if (type == TSDB_ALTER_USER_ADD_WRITE_TABLE) {
-    return "addWriteToTable";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) {
-    return "removeWriteFromTable";
-  }
-  if (type == TSDB_ALTER_USER_ADD_ALL_TABLE) {
-    return "addToAllTable";
-  }
-  if (type == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
-    return "removeFromAllTable";
-  }
   return "error";
 }
 
+static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode *pMnode, SUserObj* pNewUser) {
+  SSdb         *pSdb = pMnode->pSdb;
+  void         *pIter = NULL;
+
+  if (ALTER_USER_ADD_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || 
+     ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (strcmp(pAlterReq->objname, "1.*") != 0) {
+      int32_t len = strlen(pAlterReq->objname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      if (taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      mndReleaseDb(pMnode, pDb);
+    } else {
+      while (1) {
+        SDbObj *pDb = NULL;
+        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
+        if (pIter == NULL) break;
+        int32_t len = strlen(pDb->name) + 1;
+        taosHashPut(pNewUser->readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
+        sdbRelease(pSdb, pDb);
+      }
+    }
+  }
+
+  if (ALTER_USER_ADD_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (strcmp(pAlterReq->objname, "1.*") != 0) {
+      int32_t len = strlen(pAlterReq->objname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      if (taosHashPut(pNewUser->writeDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      mndReleaseDb(pMnode, pDb);
+    } else {
+      while (1) {
+        SDbObj *pDb = NULL;
+        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
+        if (pIter == NULL) break;
+        int32_t len = strlen(pDb->name) + 1;
+        taosHashPut(pNewUser->writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
+        sdbRelease(pSdb, pDb);
+      }
+    }
+  }
+
+  if (ALTER_USER_DEL_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (strcmp(pAlterReq->objname, "1.*") != 0) {
+      int32_t len = strlen(pAlterReq->objname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      taosHashRemove(pNewUser->readDbs, pAlterReq->objname, len);
+      mndReleaseDb(pMnode, pDb);
+    } else {
+      taosHashClear(pNewUser->readDbs);
+    }
+  }
+
+  if (ALTER_USER_DEL_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (strcmp(pAlterReq->objname, "1.*") != 0) {
+      int32_t len = strlen(pAlterReq->objname) + 1;
+      SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
+      if (pDb == NULL) {
+        mndReleaseDb(pMnode, pDb);
+        return -1;
+      }
+      taosHashRemove(pNewUser->writeDbs, pAlterReq->objname, len);
+      mndReleaseDb(pMnode, pDb);
+    } else {
+      taosHashClear(pNewUser->writeDbs);
+    }
+  }
+
+  SHashObj* pReadTbs = pNewUser->readTbs;
+  SHashObj* pWriteTbs = pNewUser->writeTbs;
+  SHashObj* pAlterTbs = pNewUser->alterTbs;
+
+#ifdef TD_ENTERPRISE
+  if (pAlterReq->isView) {
+    pReadTbs = pNewUser->readViews;
+    pWriteTbs = pNewUser->writeViews;
+    pAlterTbs = pNewUser->alterViews;
+  }
+#endif
+
+  if (ALTER_USER_ADD_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_ADD_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_ADD_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_DEL_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
+      ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndRemoveTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_DEL_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
+      ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndRemoveTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_DEL_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
+      ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
+    if (mndRemoveTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
+  }
+
+  if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
+    int32_t      len = strlen(pAlterReq->objname) + 1;
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
+    if (pTopic == NULL) {
+      mndReleaseTopic(pMnode, pTopic);
+      return -1;
+    }
+    taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
+  }
+
+  if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
+    int32_t      len = strlen(pAlterReq->objname) + 1;
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
+    if (pTopic == NULL) {
+      mndReleaseTopic(pMnode, pTopic);
+      return -1;
+    }
+    taosHashRemove(pNewUser->topics, pAlterReq->objname, len);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
   SMnode       *pMnode = pReq->info.node;
   SSdb         *pSdb = pMnode->pSdb;
@@ -1602,122 +1994,8 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
     newUser.sysInfo = alterReq.sysInfo;
   }
 
-  if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
+  if (ALTER_USER_ADD_PRIVS(alterReq.alterType) || ALTER_USER_DEL_PRIVS(alterReq.alterType)) {
-    if (strcmp(alterReq.objname, "1.*") != 0) {
+    if (0 != mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser)) goto _OVER;
-      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
-      if (pDb == NULL) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      if (taosHashPut(newUser.readDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      mndReleaseDb(pMnode, pDb);
-    } else {
-      while (1) {
-        SDbObj *pDb = NULL;
-        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
-        if (pIter == NULL) break;
-        int32_t len = strlen(pDb->name) + 1;
-        taosHashPut(newUser.readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
-        sdbRelease(pSdb, pDb);
-      }
-    }
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
-    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
-      if (pDb == NULL) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      if (taosHashPut(newUser.writeDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      mndReleaseDb(pMnode, pDb);
-    } else {
-      while (1) {
-        SDbObj *pDb = NULL;
-        pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
-        if (pIter == NULL) break;
-        int32_t len = strlen(pDb->name) + 1;
-        taosHashPut(newUser.writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
-        sdbRelease(pSdb, pDb);
-      }
-    }
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
-    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
-      if (pDb == NULL) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      taosHashRemove(newUser.readDbs, alterReq.objname, len);
-      mndReleaseDb(pMnode, pDb);
-    } else {
-      taosHashClear(newUser.readDbs);
-    }
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
-    if (strcmp(alterReq.objname, "1.*") != 0) {
-      int32_t len = strlen(alterReq.objname) + 1;
-      SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
-      if (pDb == NULL) {
-        mndReleaseDb(pMnode, pDb);
-        goto _OVER;
-      }
-      taosHashRemove(newUser.writeDbs, alterReq.objname, len);
-      mndReleaseDb(pMnode, pDb);
-    } else {
-      taosHashClear(newUser.writeDbs);
-    }
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) {
-    if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) {
-    if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE ||
-      alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
-    if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE ||
-      alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
-    if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
-    int32_t      len = strlen(alterReq.objname) + 1;
-    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
-    if (pTopic == NULL) {
-      mndReleaseTopic(pMnode, pTopic);
-      goto _OVER;
-    }
-    taosHashPut(newUser.topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
-  }
-
-  if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
-    int32_t      len = strlen(alterReq.objname) + 1;
-    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
-    if (pTopic == NULL) {
-      mndReleaseTopic(pMnode, pTopic);
-      goto _OVER;
-    }
-    taosHashRemove(newUser.topics, alterReq.objname, len);
   }
 
   if (alterReq.alterType == TSDB_ALTER_USER_ADD_WHITE_LIST) {
@@ -1820,45 +2098,45 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
     sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, tabName:%s, password:xxx",
             mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo,
             alterReq.tabName);
-    auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", detail, strlen(detail));
+    auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, detail, strlen(detail));
   }
   else if(alterReq.alterType == TSDB_ALTER_USER_SUPERUSER ||
           alterReq.alterType == TSDB_ALTER_USER_ENABLE ||
           alterReq.alterType == TSDB_ALTER_USER_SYSINFO){
-    auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", alterReq.sql, alterReq.sqlLen);
+    auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, alterReq.sql, alterReq.sqlLen);
   }
-  else if(alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB||
+  else if(ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
-          alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB||
+          ALTER_USER_ADD_WRITE_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
-          alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB||
+          ALTER_USER_ADD_ALL_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
-          alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE||
+          ALTER_USER_ADD_READ_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
-          alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE||
+          ALTER_USER_ADD_WRITE_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
-          alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE){
+          ALTER_USER_ADD_ALL_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)){
     if (strcmp(alterReq.objname, "1.*") != 0){
       SName name = {0};
       tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB);
-      auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, name.dbname,
+      auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", name.dbname, alterReq.user,
                   alterReq.sql, alterReq.sqlLen);
     }else{
-      auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, "*",
+      auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", "", alterReq.user,
                   alterReq.sql, alterReq.sqlLen);
     }
   }
-  else if(alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC){
+  else if(ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
-    auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, alterReq.objname,
+    auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.objname, alterReq.user,
                     alterReq.sql, alterReq.sqlLen);
   }
-  else if(alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC){
+  else if(ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
-    auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, alterReq.objname,
+    auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.objname, alterReq.user,
                 alterReq.sql, alterReq.sqlLen);
   }
   else{
     if (strcmp(alterReq.objname, "1.*") != 0){
       SName name = {0};
       tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB);
-      auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, name.dbname,
+      auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", name.dbname, alterReq.user,
                   alterReq.sql, alterReq.sqlLen);
     }else{
-      auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, "*",
+      auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", "", alterReq.user,
                   alterReq.sql, alterReq.sqlLen);
     }
   }
@@ -1933,7 +2211,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) {
   code = mndDropUser(pMnode, pReq, pUser);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-  auditRecord(pReq, pMnode->clusterId, "dropUser", dropReq.user, "", dropReq.sql, dropReq.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "dropUser", "", dropReq.user, dropReq.sql, dropReq.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -2092,7 +2370,7 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false);
 
-    if (strcmp("t", value) != 0) {
+    if (strcmp("t", value) != 0 && strcmp("v", value) != 0) {
       SNode  *pAst = NULL;
       int32_t sqlLen = 0;
       size_t  bufSz = strlen(value) + 1;
@@ -2113,12 +2391,22 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3
       colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false);
       taosMemoryFree(obj);
       taosMemoryFree(sql);
+
+      char notes[2] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false);
     } else {
       char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false);
       taosMemoryFree(condition);
+
+      char notes[64 + VARSTR_HEADER_SIZE] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, value[0] == 'v' ? "view" : "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false);
     }
 
     (*numOfRows)++;
@@ -2155,11 +2443,15 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
     int32_t numOfTopics = taosHashGetSize(pUser->topics);
     int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
     int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
-    if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) {
+    int32_t numOfAlterTbs = taosHashGetSize(pUser->alterTbs);
+    int32_t numOfReadViews = taosHashGetSize(pUser->readViews);
+    int32_t numOfWriteViews = taosHashGetSize(pUser->writeViews);
+    int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews);
+    if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs + numOfAlterTbs + numOfReadViews + numOfWriteViews + numOfAlterViews >= rows) {
       mInfo(
           "will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables "
-          "%d",
+          "%d, alter tables %d, read views %d, write views %d, alter views %d",
-          numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs);
+          numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs, numOfAlterTbs, numOfReadViews, numOfWriteViews, numOfAlterViews);
       pShow->restore = true;
       sdbRelease(pSdb, pUser);
       break;
@@ -2193,6 +2485,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
       taosMemoryFree(condition);
 
+      char notes[2] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
+
       numOfRows++;
     }
 
@@ -2228,6 +2525,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
       taosMemoryFree(condition);
 
+      char notes[2] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
+
       numOfRows++;
       db = taosHashIterate(pUser->readDbs, db);
     }
@@ -2264,6 +2566,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
       taosMemoryFree(condition);
 
+      char notes[2] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
+
       numOfRows++;
       db = taosHashIterate(pUser->writeDbs, db);
     }
@@ -2272,6 +2579,14 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
 
     mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow);
 
+    mndLoopHash(pUser->alterTbs, "alter", pBlock, &numOfRows, pUser->user, pShow);
+
+    mndLoopHash(pUser->readViews, "read", pBlock, &numOfRows, pUser->user, pShow);
+
+    mndLoopHash(pUser->writeViews, "write", pBlock, &numOfRows, pUser->user, pShow);
+
+    mndLoopHash(pUser->alterViews, "alter", pBlock, &numOfRows, pUser->user, pShow);
+
     char *topic = taosHashIterate(pUser->topics, NULL);
     while (topic != NULL) {
       cols = 0;
@@ -2302,6 +2617,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
       taosMemoryFree(condition);
 
+      char notes[2] = {0};
+      STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
+
       numOfRows++;
       topic = taosHashIterate(pUser->topics, topic);
     }
@@ -2450,9 +2770,11 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) {
 
     bool inRead = (taosHashGet(newUser.readTbs, stb, len) != NULL);
     bool inWrite = (taosHashGet(newUser.writeTbs, stb, len) != NULL);
-    if (inRead || inWrite) {
+    bool inAlter = (taosHashGet(newUser.alterTbs, stb, len) != NULL);
+    if (inRead || inWrite || inAlter) {
       (void)taosHashRemove(newUser.readTbs, stb, len);
       (void)taosHashRemove(newUser.writeTbs, stb, len);
+      (void)taosHashRemove(newUser.alterTbs, stb, len);
 
       SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser);
       if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
@@ -2472,6 +2794,50 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) {
   return code;
 }
 
+int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) {
+  int32_t   code = 0;
+  SSdb     *pSdb = pMnode->pSdb;
+  int32_t   len = strlen(view) + 1;
+  void     *pIter = NULL;
+  SUserObj *pUser = NULL;
+  SUserObj  newUser = {0};
+
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser);
+    if (pIter == NULL) break;
+
+    code = -1;
+    if (mndUserDupObj(pUser, &newUser) != 0) {
+      break;
+    }
+
+    bool inRead = (taosHashGet(newUser.readViews, view, len) != NULL);
+    bool inWrite = (taosHashGet(newUser.writeViews, view, len) != NULL);
+    bool inAlter = (taosHashGet(newUser.alterViews, view, len) != NULL);
+    if (inRead || inWrite || inAlter) {
+      (void)taosHashRemove(newUser.readViews, view, len);
+      (void)taosHashRemove(newUser.writeViews, view, len);
+      (void)taosHashRemove(newUser.alterViews, view, len);
+
+      SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser);
+      if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
+        break;
+      }
+      (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
+    }
+
+    mndUserFreeObj(&newUser);
+    sdbRelease(pSdb, pUser);
+    code = 0;
+  }
+
+  if (pUser != NULL) sdbRelease(pSdb, pUser);
+  if (pIter != NULL) sdbCancelFetch(pSdb, pIter);
+  mndUserFreeObj(&newUser);
+  return code;
+}
+
+
 int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) {
   int32_t   code = 0;
   SSdb     *pSdb = pMnode->pSdb;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -754,10 +754,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup
   for (int32_t v = 0; v < pVgroup->replica; ++v) {
     SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
     SDnodeObj *pDnode = taosArrayGet(pArray, v);
-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
       terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
       return -1;
     }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }
 
     int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup);
     if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@@ -1180,10 +1184,14 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro
     }
     if (used) continue;
 
-    if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+    if (pDnode == NULL) {
       terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
       return -1;
     }
+    if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
+      return -1;
+    }
 
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
     if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@@ -1912,7 +1920,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew1->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes,
              pNew1->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
 
@@ -1935,7 +1943,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew2->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes,
              pNew2->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
@@ -1956,7 +1964,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (numOfVnodes >= pNew3->numOfSupportVnodes) {
       mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes,
              pNew3->numOfSupportVnodes);
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
       goto _OVER;
     }
     int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
@@ -2177,7 +2185,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
   char obj[33] = {0};
   sprintf(obj, "%d", req.vgId);
 
-  auditRecord(pReq, pMnode->clusterId, "RedistributeVgroup", obj, "", req.sql, req.sqlLen);
+  auditRecord(pReq, pMnode->clusterId, "RedistributeVgroup", "", obj, req.sql, req.sqlLen);
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {

source/dnode/mnode/impl/src/mndView.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndView.h"
+#include "mndShow.h"
+
+int32_t mndInitView(SMnode *pMnode) {
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_VIEW, mndProcessCreateViewReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_DROP_VIEW, mndProcessDropViewReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_VIEW_META, mndProcessGetViewMetaReq);
+
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VIEWS, mndRetrieveView);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VIEWS, mndCancelGetNextView);
+
+#ifdef TD_ENTERPRISE
+  initDynViewVersion();
+
+  SSdbTable table = {
+      .sdbType = SDB_VIEW,
+      .keyType = SDB_KEY_BINARY,
+      .encodeFp = (SdbEncodeFp)mndViewActionEncode,
+      .decodeFp = (SdbDecodeFp)mndViewActionDecode,
+      .insertFp = (SdbInsertFp)mndViewActionInsert,
+      .updateFp = (SdbUpdateFp)mndViewActionUpdate,
+      .deleteFp = (SdbDeleteFp)mndViewActionDelete,
+  };
+
+  return sdbSetTable(pMnode->pSdb, table);
+#else
+  return TSDB_CODE_SUCCESS;
+#endif
+}
+
+void mndCleanupView(SMnode *pMnode) {
+  mDebug("mnd view cleanup");
+}
+
+int32_t mndProcessCreateViewReq(SRpcMsg *pReq) {
+#ifndef TD_ENTERPRISE
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
+  SCMCreateViewReq   createViewReq = {0};
+  if (tDeserializeSCMCreateViewReq(pReq->pCont, pReq->contLen, &createViewReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  mInfo("start to create view:%s, sql:%s", createViewReq.fullname, createViewReq.sql);
+
+  return mndProcessCreateViewReqImpl(&createViewReq, pReq);
+#endif
+}
+
+int32_t mndProcessDropViewReq(SRpcMsg *pReq) {
+#ifndef TD_ENTERPRISE
+    return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
+    SCMDropViewReq dropViewReq = {0};
+    if (tDeserializeSCMDropViewReq(pReq->pCont, pReq->contLen, &dropViewReq) != 0) {
+      terrno = TSDB_CODE_INVALID_MSG;
+      return -1;
+    }
+  
+    mInfo("start to drop view:%s, sql:%s", dropViewReq.name, dropViewReq.sql);
+  
+    return mndProcessDropViewReqImpl(&dropViewReq, pReq);
+#endif
+}
+
+int32_t mndProcessGetViewMetaReq(SRpcMsg *pReq) {
+#ifndef TD_ENTERPRISE
+  return TSDB_CODE_OPS_NOT_SUPPORT;
+#else
+  SViewMetaReq  req = {0};
+
+  if (tDeserializeSViewMetaReq(pReq->pCont, pReq->contLen, &req) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  return mndProcessViewMetaReqImpl(&req, pReq);
+#endif  
+}
+
+
+int32_t mndRetrieveView(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
+#ifndef TD_ENTERPRISE
+  return 0;
+#else
+  return mndRetrieveViewImpl(pReq, pShow, pBlock, rows);
+#endif
+}
+
+void mndCancelGetNextView(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}
+
+
+

source/dnode/mnode/impl/test/user/user.cpp

@@ -250,7 +250,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
+    alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
+    alterReq.privileges = PRIVILEGE_TYPE_ALL；
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "1.*");
@@ -266,7 +267,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB;
+    alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
+    alterReq.privileges = PRIVILEGE_TYPE_ALL；
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "1.*");
@@ -282,7 +284,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB;
+    alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
+    alterReq.privileges = PRIVILEGE_TYPE_READ;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "d1");
@@ -329,7 +332,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB;
+    alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
+    alterReq.privileges = PRIVILEGE_TYPE_READ;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "1.d2");
@@ -345,7 +349,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB;
+    alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
+    alterReq.privileges = PRIVILEGE_TYPE_READ;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "1.d2");
@@ -388,7 +393,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
 
   {
     SAlterUserReq alterReq = {0};
-    alterReq.alterType = TSDB_ALTER_USER_REMOVE_READ_DB;
+    alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
+    alterReq.alterType = PRIVILEGE_TYPE_READ;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
     strcpy(alterReq.dbname, "1.d2");

source/dnode/mnode/sdb/inc/sdb.h

@@ -148,7 +148,8 @@ typedef enum {
   SDB_DB = 19,
   SDB_FUNC = 20,
   SDB_IDX = 21,
-  SDB_MAX = 22
+  SDB_VIEW = 22,
+  SDB_MAX = 23
 } ESdbType;
 
 typedef struct SSdbRaw {

source/dnode/mnode/sdb/src/sdbHash.c

@@ -62,6 +62,8 @@ const char *sdbTableName(ESdbType type) {
       return "func";
     case SDB_IDX:
       return "idx";
+    case SDB_VIEW:
+      return "view";
     default:
       return "undefine";
   }

source/dnode/snode/src/snode.c

@@ -92,11 +92,13 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
     }
   }
 
-  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64
+  char* p = NULL;
-         " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
+  streamTaskGetStatus(pTask, &p);
+
+  qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
+        " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
         SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-         pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+        pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam);
-         pTask->info.fillHistory, pTask->info.triggerParam);
 
   return 0;
 }
@@ -163,20 +165,26 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
   ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG);
 
   // 2.save task
-  taosWLockLatch(&pSnode->pMeta->lock);
+  streamMetaWLock(pSnode->pMeta);
 
   bool added = false;
   code = streamMetaRegisterTask(pSnode->pMeta, -1, pTask, &added);
   if (code < 0) {
-    taosWUnLockLatch(&pSnode->pMeta->lock);
+    streamMetaWUnLock(pSnode->pMeta);
     return -1;
   }
 
   int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta);
-  taosWUnLockLatch(&pSnode->pMeta->lock);
+  streamMetaWUnLock(pSnode->pMeta);
-  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr,
-         streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks);
 
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
+  qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE,
+         pTask->id.idStr, p, numOfTasks);
+
+  EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+  streamTaskHandleEvent(pTask->status.pSM, event);
   streamTaskCheckDownstream(pTask);
   return 0;
 }
@@ -187,14 +195,14 @@ int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
   streamMetaUnregisterTask(pSnode->pMeta, pReq->streamId, pReq->taskId);
 
   // commit the update
-  taosWLockLatch(&pSnode->pMeta->lock);
+  streamMetaWLock(pSnode->pMeta);
   int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta);
   qDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks);
 
   if (streamMetaCommit(pSnode->pMeta) < 0) {
     // persist to disk
   }
-  taosWUnLockLatch(&pSnode->pMeta->lock);
+  streamMetaWUnLock(pSnode->pMeta);
   return 0;
 }
 
@@ -352,10 +360,10 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
   if (pTask != NULL) {
     rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
     streamMetaReleaseTask(pSnode->pMeta, pTask);
-
+    char* p = NULL;
-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    streamTaskGetStatus(pTask, &p);
     qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
-            pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+            pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
   } else {
     rsp.status = TASK_DOWNSTREAM_NOT_READY;
     qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d",

source/dnode/vnode/src/inc/tq.h

@@ -126,7 +126,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
 int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
 int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
                       int32_t type, int32_t vgId);
-//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId);
 int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
 
 // tqMeta
@@ -134,7 +133,6 @@ int32_t tqMetaOpen(STQ* pTq);
 int32_t tqMetaClose(STQ* pTq);
 int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
 int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
-//int32_t tqMetaRestoreHandle(STQ* pTq);
 int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
 int32_t tqMetaRestoreCheckInfo(STQ* pTq);
@@ -160,8 +158,8 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
-int32_t tqCheckAndRunStreamTask(STQ* pTq);
+int32_t tqStartStreamTask(STQ* pTq);
-int32_t tqStartStreamTasks(STQ* pTq);
+int32_t tqResetStreamTaskStatus(STQ* pTq);
 int32_t tqStopStreamTasks(STQ* pTq);
 
 // tq util
@@ -171,6 +169,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
                         int32_t type, int64_t sver, int64_t ever);
 int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
 void    tqUpdateNodeStage(STQ* pTq, bool isLeader);
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
+                               SSubmitTbData* pTableData, const char* id);
+int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
+
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
+                                       SSDataBlock* pDataBlock, SArray* pTagArray);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/tsdb.h

@@ -309,7 +309,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STs
 void    tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive);
 
 // tsdbMerge.c ==============================================================================================
-int32_t tsdbMerge(void *arg);
+int32_t tsdbSchedMerge(STsdb *tsdb, int32_t fid);
 
 // tsdbDiskData ==============================================================================================
 int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
@@ -371,7 +371,7 @@ struct STsdb {
   char                *path;
   SVnode              *pVnode;
   STsdbKeepCfg         keepCfg;
-  TdThreadRwlock       rwLock;
+  TdThreadMutex        mutex;
   SMemTable           *mem;
   SMemTable           *imem;
   STsdbFS              fs;  // old
@@ -382,6 +382,8 @@ struct STsdb {
   TdThreadMutex        biMutex;
   SLRUCache           *bCache;
   TdThreadMutex        bMutex;
+  SLRUCache           *pgCache;
+  TdThreadMutex        pgMutex;
   struct STFileSystem *pFS;  // new
   SRocksCache          rCache;
 };
@@ -668,7 +670,7 @@ struct SDelFWriter {
 };
 
 #include "tarray2.h"
-//#include "tsdbFS2.h"
+// #include "tsdbFS2.h"
 //  struct STFileSet;
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
@@ -873,8 +875,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf);
 
 void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool tMergeTreeNext(SMergeTree *pMTree);
-void tMergeTreePinSttBlock(SMergeTree* pMTree);
+void tMergeTreePinSttBlock(SMergeTree *pMTree);
-void tMergeTreeUnpinSttBlock(SMergeTree* pMTree);
+void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
 bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
 void tMergeTreeClose(SMergeTree *pMTree);
 
@@ -909,7 +911,9 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa
 int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h);
 
 int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle);
-int32_t tsdbBCacheRelease(SLRUCache *pCache, LRUHandle *h);
+int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle);
+int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage);
+int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
 
 int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);

source/dnode/vnode/src/inc/vndCos.h

@@ -27,6 +27,8 @@ extern "C" {
 extern int8_t  tsS3Enabled;
 extern int32_t tsS3BlockSize;
 extern int32_t tsS3BlockCacheSize;
+extern int32_t tsS3PageCacheSize;
+extern int32_t tsS3UploadDelaySec;
 
 int32_t s3Init();
 void    s3CleanUp();

source/dnode/vnode/src/inc/vnodeInt.h

@@ -231,7 +231,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
 int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq);
+int32_t tqLaunchStreamTaskAsync(STQ* pTq);
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);

source/dnode/vnode/src/sma/smaOpen.c

@@ -33,7 +33,7 @@ static int32_t rsmaRestore(SSma *pSma);
 #define SMA_OPEN_RSMA_IMPL(v, l, force)                                                             \
   do {                                                                                              \
     SRetention *r = (SRetention *)VND_RETENTIONS(v) + l;                                            \
-    if (!RETENTION_VALID(r)) {                                                               \
+    if (!RETENTION_VALID(l, r)) {                                                                   \
       if (l == 0) {                                                                                 \
         code = TSDB_CODE_INVALID_PARA;                                                              \
         TSDB_CHECK_CODE(code, lino, _exit);                                                         \
@@ -79,20 +79,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
   freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
   keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);
 
-  int32_t nFreqTimes = (r + level)->freq / (r + TSDB_RETENTION_L0)->freq;
+  int32_t nFreqTimes = (r + level)->freq / (60 * 1000);  // use 60s for freq of 1st level
   days *= (nFreqTimes > 1 ? nFreqTimes : 1);
 
-  if (days > keepDuration) {
-    days = keepDuration;
-  }
-
-  if (days > TSDB_MAX_DURATION_PER_FILE) {
-    days = TSDB_MAX_DURATION_PER_FILE;
-  }
-
   if (days < freqDuration) {
     days = freqDuration;
   }
+
+  int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE);
+  if (days > maxKeepDuration) {
+    days = maxKeepDuration;
+  }
+
 _exit:
   smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration);
   return days;
@@ -157,6 +155,7 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
 _exit:
   if (code) {
     smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
+    terrno = code;
   }
   return code;
 }

source/dnode/vnode/src/sma/smaRollup.c

@@ -31,8 +31,6 @@ SSmaMgmt smaMgmt = {
 
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 
-extern int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now);
-
 static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static void       tdUidStoreDestory(STbUidStore *pStore);
 static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd);

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -35,8 +35,8 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
   return code;
 }
 
-int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) {
+int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) {
-  int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg);
+  int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg);
 
   return code;
 }
@@ -109,7 +109,7 @@ _exit:
  * @param pMsg
  * @return int32_t
  */
-static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
+static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg) {
   int32_t        code = 0;
   int32_t        lino = 0;
   SSmaCfg       *pCfg = (SSmaCfg *)pMsg;
@@ -118,7 +118,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
 
   if (TD_VID(pSma->pVnode) == pCfg->dstVgId) {
     // create tsma meta in dstVgId
-    if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) {
+    if (metaCreateTSma(SMA_META(pSma), ver, pCfg) < 0) {
       code = terrno;
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -130,7 +130,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
     pReq.schemaRow = pCfg->schemaRow;
     pReq.schemaTag = pCfg->schemaTag;
 
-    if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) {
+    if (metaCreateSTable(SMA_META(pSma), ver, &pReq) < 0) {
       code = terrno;
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -154,94 +154,36 @@ _exit:
   return code;
 }
 
-int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema,
+int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int64_t suid,
-                         SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName,
+                         const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
-                         SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
   int32_t      code = 0;
   int32_t      lino = 0;
   void        *pBuf = NULL;
   int32_t      len = 0;
   SSubmitReq2 *pReq = NULL;
   SArray      *tagArray = NULL;
-  SArray      *createTbArray = NULL;
-  SArray      *pVals = NULL;
 
-  int32_t sz = taosArrayGetSize(pBlocks);
+  int32_t numOfBlocks = taosArrayGetSize(pBlocks);
 
   tagArray = taosArrayInit(1, sizeof(STagVal));
-  createTbArray = taosArrayInit(sz, POINTER_BYTES);
   pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2));
-  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
 
-  if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) {
+  if (!tagArray || !pReq) {
     code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  // create table req
+  pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
-  if (createTb) {
+  if (pReq->aSubmitTbData == NULL) {
-    for (int32_t i = 0; i < sz; ++i) {
+    code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
-      SSDataBlock   *pDataBlock = taosArrayGet(pBlocks, i);
-      SVCreateTbReq *pCreateTbReq = NULL;
-      if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
-        taosArrayPush(createTbArray, &pCreateTbReq);
-        continue;
-      }
-
-      if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        TSDB_CHECK_CODE(code, lino, _exit);
-      };
-
-      // don't move to the end of loop as to destroy in the end of func when error occur
-      taosArrayPush(createTbArray, &pCreateTbReq);
-
-      // set const
-      pCreateTbReq->flags = 0;
-      pCreateTbReq->type = TSDB_CHILD_TABLE;
-      pCreateTbReq->ctb.suid = suid;
-
-      // set super table name
-      SName name = {0};
-      tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-      pCreateTbReq->ctb.stbName = taosStrdup((char *)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-      // set tag content
-      taosArrayClear(tagArray);
-      STagVal tagVal = {
-          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
-          .type = TSDB_DATA_TYPE_UBIGINT,
-          .i64 = (int64_t)pDataBlock->info.id.groupId,
-      };
-      taosArrayPush(tagArray, &tagVal);
-      pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
-
-      STag *pTag = NULL;
-      tTagNew(tagArray, 1, false, &pTag);
-      if (pTag == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
-      pCreateTbReq->ctb.pTag = (uint8_t *)pTag;
 
-      // set tag name
+  SHashObj *pTableIndexMap =
-      SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
+      taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = {0};
-      strcpy(tagNameStr, "group_id");
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
-
-      // set table name
-      if (pDataBlock->info.parTbName[0]) {
-        pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
-      } else {
-        pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
-      }
-    }
-  }
 
   // SSubmitTbData req
-  for (int32_t i = 0; i < sz; ++i) {
+  for (int32_t i = 0; i < numOfBlocks; ++i) {
     SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i);
     if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
       pDeleteReq->suid = suid;
@@ -250,62 +192,42 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
       continue;
     }
 
-    int32_t rows = pDataBlock->info.rows;
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE,};
 
-    SSubmitTbData tbData = {0};
+    int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
+    tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray);
 
-    if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) {
+    {
-      code = terrno;
+      uint64_t groupId = pDataBlock->info.id.groupId;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-    tbData.suid = suid;
-    tbData.uid = 0;  // uid is assigned by vnode
-    tbData.sver = pTSchema->version;
 
-    if (createTb) {
+      int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
-      tbData.pCreateTbReq = taosArrayGetP(createTbArray, i);
+      if (index == NULL) {  // no data yet, append it
-      if (tbData.pCreateTbReq) tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
-    }
+        if (code != TSDB_CODE_SUCCESS) {
-
+          continue;
-    if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) {
-      taosArrayDestroy(tbData.aRowP);
-      code = terrno;
-      TSDB_CHECK_CODE(code, lino, _exit);
-    }
-
-    for (int32_t j = 0; j < rows; ++j) {
-      taosArrayClear(pVals);
-      for (int32_t k = 0; k < pTSchema->numOfCols; k++) {
-        const STColumn  *pCol = &pTSchema->columns[k];
-        SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k);
-        if (colDataIsNull_s(pColData, j)) {
-          SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
-          taosArrayPush(pVals, &cv);
-        } else {
-          void *data = colDataGetData(pColData, j);
-          if (IS_STR_DATA_TYPE(pCol->type)) {
-            SValue  sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)};  // address copy, no value
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          } else {
-            SValue sv;
-            memcpy(&sv.val, data, tDataTypes[pCol->type].bytes);
-            SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
-            taosArrayPush(pVals, &cv);
-          }
-        }
-      }
-      SRow *pRow = NULL;
-      if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) {
-        tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
-      taosArrayPush(tbData.aRowP, &pRow);
         }
 
         taosArrayPush(pReq->aSubmitTbData, &tbData);
+
+        int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
+        taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
+      } else {
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
         }
 
+        SSubmitTbData *pExisted = taosArrayGet(pReq->aSubmitTbData, *index);
+        code = doMergeExistedRows(pExisted, &tbData, "id");
+        if (code != TSDB_CODE_SUCCESS) {
+          continue;
+        }
+      }
+    }
+  }
+
+  taosHashCleanup(pTableIndexMap);
+
   // encode
   tEncodeSize(tEncodeSubmitReq, pReq, len, code);
   if (TSDB_CODE_SUCCESS == code) {
@@ -327,11 +249,10 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
     }
     tEncoderClear(&encoder);
   }
+
 _exit:
-  taosArrayDestroy(createTbArray);
   taosArrayDestroy(tagArray);
-  taosArrayDestroy(pVals);
+  if (pReq != NULL) {
-  if (pReq) {
     tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
     taosMemoryFree(pReq);
   }
@@ -442,8 +363,8 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
   void           *pSubmitReq = NULL;
   int32_t         contLen = 0;
 
-  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true,
+  code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, pTsmaStat->pTSma->dstTbUid,
-                          pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
+                          pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) {

source/dnode/vnode/src/tq/tq.c

@@ -20,6 +20,12 @@ typedef struct {
   int8_t inited;
 } STqMgmt;
 
+typedef struct STaskUpdateEntry {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t transId;
+} STaskUpdateEntry;
+
 static STqMgmt tqMgmt = {0};
 
 // 0: not init
@@ -820,28 +826,29 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   }
 
   // sink
-  if (pTask->outputInfo.type == TASK_OUTPUT__SMA) {
+  STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
-    pTask->outputInfo.smaSink.vnode = pTq->pVnode;
+  if (pOutputInfo->type == TASK_OUTPUT__SMA) {
-    pTask->outputInfo.smaSink.smaSink = smaHandleRes;
+    pOutputInfo->smaSink.vnode = pTq->pVnode;
-  } else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
+    pOutputInfo->smaSink.smaSink = smaHandleRes;
-    pTask->outputInfo.tbSink.vnode = pTq->pVnode;
+  } else if (pOutputInfo->type == TASK_OUTPUT__TABLE) {
-    pTask->outputInfo.tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
+    pOutputInfo->tbSink.vnode = pTq->pVnode;
+    pOutputInfo->tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
 
     int32_t   ver1 = 1;
     SMetaInfo info = {0};
-    code = metaGetInfo(pTq->pVnode->pMeta, pTask->outputInfo.tbSink.stbUid, &info, NULL);
+    code = metaGetInfo(pTq->pVnode->pMeta, pOutputInfo->tbSink.stbUid, &info, NULL);
     if (code == TSDB_CODE_SUCCESS) {
       ver1 = info.skmVer;
     }
 
-    SSchemaWrapper* pschemaWrapper = pTask->outputInfo.tbSink.pSchemaWrapper;
+    SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper;
-    pTask->outputInfo.tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
+    pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
-    if (pTask->outputInfo.tbSink.pTSchema == NULL) {
+    if (pOutputInfo->tbSink.pTSchema == NULL) {
       return -1;
     }
 
-    pTask->outputInfo.tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
+    pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
-    tSimpleHashSetFreeFp(pTask->outputInfo.tbSink.pTblInfo, freePtr);
+    tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr);
   }
 
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@@ -849,11 +856,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
   }
 
-  // reset the task status from unfinished transaction
+//  // reset the task status from unfinished transaction
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+//  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
+//    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
-    pTask->status.taskStatus = TASK_STATUS__NORMAL;
+//    pTask->status.taskStatus = TASK_STATUS__READY;
-  }
+//  }
 
   streamTaskResetUpstreamStageInfo(pTask);
   streamSetupScheduleTrigger(pTask);
@@ -866,20 +873,23 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
            pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
   }
 
+  char* p = NULL;
+  streamTaskGetStatus(pTask, &p);
+
   if (pTask->info.fillHistory) {
     tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
            vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-           pTask->info.fillHistory, (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
+           (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
   } else {
     tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
            " nextProcessVer:%" PRId64
            " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
            vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
-           pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus),
+           pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
-           pTask->info.fillHistory, (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
+           (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
   }
 
   return 0;
@@ -921,9 +931,10 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
       rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
       streamMetaReleaseTask(pMeta, pTask);
 
-      const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+      char* p = NULL;
+      streamTaskGetStatus(pTask, &p);
       tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d",
-              pTask->id.idStr, pStatus, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+              pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
     } else {
       rsp.status = TASK_DOWNSTREAM_NOT_READY;
       tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
@@ -957,6 +968,12 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
           rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
 
+  if (!vnodeIsRoleLeader(pTq->pVnode)) {
+    tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
+            rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId);
+    return code;
+  }
+
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId);
   if (pTask == NULL) {
     tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
@@ -1006,10 +1023,10 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
   int64_t streamId = pTask->id.streamId;
   bool    added = false;
 
-  taosWLockLatch(&pStreamMeta->lock);
+  streamMetaWLock(pStreamMeta);
   code = streamMetaRegisterTask(pStreamMeta, sversion, pTask, &added);
   int32_t numOfTasks = streamMetaGetNumOfTasks(pStreamMeta);
-  taosWUnLockLatch(&pStreamMeta->lock);
+  streamMetaWUnLock(pStreamMeta);
 
   if (code < 0) {
     tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code));
@@ -1026,11 +1043,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
       SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId);
 
       bool restored = pTq->pVnode->restored;
-      if (p != NULL && restored) {
+      if (p != NULL && restored && p->info.fillHistory == 0) {
-        p->execInfo.init = taosGetTimestampMs();
+        EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
-        tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->execInfo.init);
+        streamTaskHandleEvent(p->status.pSM, event);
-
-        streamTaskCheckDownstream(p);
       } else if (!restored) {
         tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId);
       }
@@ -1049,6 +1064,47 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
   return code;
 }
 
+static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) {
+  const char* id = pTask->id.idStr;
+  int64_t     nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
+
+  // if it's an source task, extract the last version in wal.
+  SVersionRange *pRange = &pTask->dataRange.range;
+
+  bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
+  pTask->execInfo.step2Start = taosGetTimestampMs();
+
+  if (done) {
+    qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0);
+    streamTaskPutTranstateIntoInputQ(pTask);
+    streamExecTask(pTask);  // exec directly
+  } else {
+    STimeWindow* pWindow = &pTask->dataRange.window;
+    tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64
+                ", do secondary scan-history from WAL after halt the related stream task:%s",
+            id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey,
+            pStreamTask->id.idStr);
+    ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING);
+
+    streamSetParamForStreamScannerStep2(pTask, pRange, pWindow);
+
+    int64_t dstVer = pTask->dataRange.range.minVer;
+    pTask->chkInfo.nextProcessVer = dstVer;
+
+    walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer);
+    tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
+            pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE);
+
+    /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);
+
+    // now the fill-history task starts to scan data from wal files.
+    int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
+    if (code == TSDB_CODE_SUCCESS) {
+      tqScanWalAsync(pTq, false);
+    }
+  }
+}
+
 // this function should be executed by only one thread
 int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont;
@@ -1064,7 +1120,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
 
   // do recovery step1
   const char* id = pTask->id.idStr;
-  const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+  char*       pStatus = NULL;
+  streamTaskGetStatus(pTask, &pStatus);
 
   // avoid multi-thread exec
   while(1) {
@@ -1110,15 +1167,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
     return 0;
   }
 
-  if (pTask->info.fillHistory == 1) {
-    ASSERT(pTask->status.pauseAllowed == true);
-  }
-
   streamScanHistoryData(pTask);
 
   double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
-  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) {
-    int8_t status = streamTaskSetSchedStatusInActive(pTask);
+    int8_t status = streamTaskSetSchedStatusInactive(pTask);
     tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status);
 
     atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@@ -1127,22 +1180,18 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   // the following procedure should be executed, no matter status is stop/pause or not
-  tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el);
+  tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el);
 
   if (pTask->info.fillHistory) {
-    SVersionRange* pRange = NULL;
     SStreamTask*   pStreamTask = NULL;
-    bool           done = false;
 
     // 1. get the related stream task
     pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
     if (pStreamTask == NULL) {
-      // todo delete this task, if the related stream task is dropped
+      tqError("failed to find s-task:0x%"PRIx64", it may have been destroyed, drop related fill-history task:%s",
-      qError("failed to find s-task:0x%"PRIx64", it may have been destroyed, drop fill-history task:%s",
              pTask->streamTaskId.taskId, pTask->id.idStr);
 
       tqDebug("s-task:%s fill-history task set status to be dropping", id);
-
       streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
 
       atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@@ -1152,128 +1201,25 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
 
     ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
 
-    // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the
+    code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
-    // stream task get ready for scan history data
+    if (code == TSDB_CODE_SUCCESS) {
-    while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
+      doStartStep2(pTask, pStreamTask, pTq);
-      tqDebug(
-          "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms",
-          id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus));
-      taosMsleep(100);
-    }
-
-    // now we can stop the stream task execution
-    int64_t nextProcessedVer = 0;
-
-    while (1) {
-      taosThreadMutexLock(&pStreamTask->lock);
-      int8_t status = pStreamTask->status.taskStatus;
-      if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) {
-        // return; do nothing
-      }
-
-      if (status == TASK_STATUS__HALT) {
-//        tqDebug("s-task:%s level:%d sched-status:%d is halt by fill-history task:%s", pStreamTask->id.idStr,
-//                pStreamTask->info.taskLevel, pStreamTask->status.schedStatus, id);
-//        latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
-//
-//        taosThreadMutexUnlock(&pStreamTask->lock);
-//        break;
-      }
-
-      if (pStreamTask->status.taskStatus == TASK_STATUS__CK) {
-        qDebug("s-task:%s status:%s during generating checkpoint, wait for 1sec and retry set status:halt",
-               pStreamTask->id.idStr, streamGetTaskStatusStr(TASK_STATUS__CK));
-        taosThreadMutexUnlock(&pStreamTask->lock);
-        taosMsleep(1000);
-        continue;
-      }
-
-      // upgrade to halt status
-      if (status == TASK_STATUS__PAUSE) {
-        qDebug("s-task:%s upgrade status to %s from %s", pStreamTask->id.idStr, streamGetTaskStatusStr(TASK_STATUS__HALT),
-               streamGetTaskStatusStr(TASK_STATUS__PAUSE));
     } else {
-        qDebug("s-task:%s halt task, prev status:%s", pStreamTask->id.idStr, streamGetTaskStatusStr(status));
+      tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr);
     }
 
-      pStreamTask->status.keepTaskStatus = status;
-      pStreamTask->status.taskStatus = TASK_STATUS__HALT;
-
-      nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
-      if (nextProcessedVer == -1) {
-        nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1;
-      }
-
-      tqDebug("s-task:%s level:%d nextProcessedVer:%" PRId64 ", sched-status:%d is halt by fill-history task:%s",
-              pStreamTask->id.idStr, pStreamTask->info.taskLevel, nextProcessedVer, pStreamTask->status.schedStatus,
-              id);
-
-      taosThreadMutexUnlock(&pStreamTask->lock);
-      break;
-    }
-
-    // if it's an source task, extract the last version in wal.
-    pRange = &pTask->dataRange.range;
-    done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
-    pTask->execInfo.step2Start = taosGetTimestampMs();
-
-    if (done) {
-      qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0);
-      streamTaskPutTranstateIntoInputQ(pTask);
-
-      if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-        pTask->status.keepTaskStatus = TASK_STATUS__NORMAL;
-        qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
-               streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
-      }
-
-      streamExecTask(pTask);  // exec directly
-    } else {
-      STimeWindow* pWindow = &pTask->dataRange.window;
-      tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64
-              ", do secondary scan-history from WAL after halt the related stream task:%s",
-              id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey,
-              pStreamTask->id.idStr);
-      ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING);
-
-      streamSetParamForStreamScannerStep2(pTask, pRange, pWindow);
-
-      int64_t dstVer = pTask->dataRange.range.minVer;
-      pTask->chkInfo.nextProcessVer = dstVer;
-
-      walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer);
-      tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
-              pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE);
-
-      /*int8_t status = */streamTaskSetSchedStatusInActive(pTask);
-
-      // the fill-history task starts to process data in wal, let's set it status to be normal now
-      if (pTask->info.fillHistory == 1 && !streamTaskShouldStop(&pTask->status)) {
-        streamSetStatusNormal(pTask);
-      }
-
-      tqScanWalAsync(pTq, false);
-    }
     streamMetaReleaseTask(pMeta, pStreamTask);
 
   } else {
     STimeWindow* pWindow = &pTask->dataRange.window;
+    ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
 
-    if (pTask->hTaskInfo.id.taskId == 0) {
+    // Not update the fill-history time window until the state transfer is completed if the related fill-history task
-      *pWindow = (STimeWindow){INT64_MIN, INT64_MAX};
+    // exists.
     tqDebug(
-          "s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time "
+        "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64
-          "window:%" PRId64 " - %" PRId64,
+        ", window:%" PRId64 " - %" PRId64,
-          id, pWindow->skey, pWindow->ekey);
-      qStreamInfoResetTimewindowFilter(pTask->exec.pExecutor);
-    } else {
-      // when related fill-history task exists, update the fill-history time window only when the
-      // state transfer is completed.
-      tqDebug(
-          "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start "
-          "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64,
         id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
-    }
 
     code = streamTaskScanHistoryDataComplete(pTask);
   }
@@ -1352,7 +1298,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t vgId = TD_VID(pTq->pVnode);
 
   if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) {
-    tqCheckAndRunStreamTask(pTq);
+    tqStartStreamTask(pTq);
     return 0;
   }
 
@@ -1362,17 +1308,16 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId);
-  if (pTask != NULL) {
+  if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
-    // even in halt status, the data in inputQ must be processed
+    char* p = NULL;
-    int8_t st = pTask->status.taskStatus;
+    if (streamTaskReadyToRun(pTask, &p)) {
-    if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK) {
       tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr,
               pTask->chkInfo.nextProcessVer);
       streamExecTask(pTask);
     } else {
-      int8_t status = streamTaskSetSchedStatusInActive(pTask);
+      int8_t status = streamTaskSetSchedStatusInactive(pTask);
       tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
-              pTask->id.idStr, streamGetTaskStatusStr(st), status);
+              pTask->id.idStr, p, status);
     }
 
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1446,7 +1391,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
   if (pTask != NULL) {
     // drop the related fill-history task firstly
-    if (pTask->hTaskInfo.id.taskId != 0) {
+    if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
       STaskId* pHTaskId = &pTask->hTaskInfo.id;
       streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
       tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId);
@@ -1458,14 +1403,14 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
   streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
 
   // commit the update
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
   int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
   tqDebug("vgId:%d task:0x%x dropped, remain tasks:%d", vgId, pReq->taskId, numOfTasks);
 
   if (streamMetaCommit(pMeta) < 0) {
     // persist to disk
   }
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaWUnLock(pMeta);
 
   return 0;
 }
@@ -1486,7 +1431,7 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
   streamTaskPause(pTask, pMeta);
 
   SStreamTask* pHistoryTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
     if (pHistoryTask == NULL) {
       tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
@@ -1514,17 +1459,19 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
     return -1;
   }
 
-  // todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal
+  streamTaskResume(pTask);
-  streamTaskResume(pTask, pTq->pStreamMeta);
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
 
   int32_t level = pTask->info.taskLevel;
   if (level == TASK_LEVEL__SINK) {
+    if (status == TASK_STATUS__UNINIT) {
+
+    }
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
     return 0;
   }
 
-  int8_t  status = pTask->status.taskStatus;
+  if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
-  if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
     // no lock needs to secure the access of the version
     if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) {
       // discard all the data  when the stream task is suspended.
@@ -1537,14 +1484,18 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
               vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus);
     }
 
-    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory &&
+    if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
-        pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
       streamStartScanHistoryAsync(pTask, igUntreated);
-    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0)) {
+    } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
       tqScanWalAsync(pTq, false);
     } else {
       streamSchedExec(pTask);
     }
+  } else if (status == TASK_STATUS__UNINIT) {
+    if (pTask->info.fillHistory == 0) {
+      EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
+      streamTaskHandleEvent(pTask->status.pSM, event);
+    }
   }
 
   streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1683,7 +1634,6 @@ FAIL:
   return -1;
 }
 
-// todo error code cannot be return, since this is invoked by an mnode-launched transaction.
 int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
   int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
@@ -1694,7 +1644,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   // disable auto rsp to mnode
   pRsp->info.handle = NULL;
 
-  // todo: add counter to make sure other tasks would not be trapped in checkpoint state
   SStreamCheckpointSourceReq req = {0};
   if (!vnodeIsRoleLeader(pTq->pVnode)) {
     tqDebug("vgId:%d not leader, ignore checkpoint-source msg", vgId);
@@ -1725,6 +1674,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   }
   tDecoderClear(&decoder);
 
+  // todo handle failure to reset from checkpoint procedure
   SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
   if (pTask == NULL) {
     tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed already", vgId,
@@ -1735,6 +1685,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
     return TSDB_CODE_SUCCESS;
   }
 
+  // todo handle failure to reset from checkpoint procedure
   // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req.
   if (pTask->status.downstreamReady != 1) {
     pTask->chkInfo.failedId = req.checkpointId;   // record the latest failed checkpoint id
@@ -1750,8 +1701,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
     return TSDB_CODE_SUCCESS;
   }
 
+  // todo save the checkpoint failed info
   taosThreadMutexLock(&pTask->lock);
-  if (pTask->status.taskStatus == TASK_STATUS__HALT) {
+  ETaskStatus status = streamTaskGetStatus(pTask, NULL);
+
+  if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {
     qError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure",
            pTask->id.idStr, req.checkpointId);
     taosThreadMutexUnlock(&pTask->lock);
@@ -1767,7 +1721,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   taosThreadMutexUnlock(&pTask->lock);
 
   int32_t total = 0;
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
 
   // set the initial value for generating check point
   // set the mgmt epset info according to the checkout source msg from mnode, todo update mgmt epset if needed
@@ -1776,7 +1730,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
   }
 
   total = pMeta->numOfStreamTasks;
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaWUnLock(pMeta);
 
   qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", total checkpoint reqs:%d",
          pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, total);
@@ -1832,7 +1786,6 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   char*        msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t      len = pMsg->contLen - sizeof(SMsgHead);
   SRpcMsg      rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
-  bool         allStopped = false;
 
   SStreamTaskNodeUpdateMsg req = {0};
 
@@ -1848,7 +1801,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   tDecoderClear(&decoder);
 
   // update the nodeEpset when it exists
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
 
   // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status.
   STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
@@ -1857,23 +1810,50 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
     tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId,
             req.taskId);
     rsp.code = TSDB_CODE_SUCCESS;
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
+
     taosArrayDestroy(req.pNodeList);
     return rsp.code;
   }
 
   SStreamTask* pTask = *ppTask;
-  tqDebug("s-task:%s receive nodeEp update msg from mnode", pTask->id.idStr);
 
+  if (pMeta->updateInfo.transId != req.transId) {
+    pMeta->updateInfo.transId = req.transId;
+    tqInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId);
+    // info needs to be kept till the new trans to update the nodeEp arrived.
+    taosHashClear(pMeta->updateInfo.pTasks);
+  } else {
+    tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId);
+  }
+
+  STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId};
+  void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
+  if (exist != NULL) {
+    tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId,
+            req.transId);
+    rsp.code = TSDB_CODE_SUCCESS;
+    streamMetaWUnLock(pMeta);
+    taosArrayDestroy(req.pNodeList);
+    return rsp.code;
+  }
+
+  streamMetaWUnLock(pMeta);
+
+  // the following two functions should not be executed within the scope of meta lock to avoid deadlock
   streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
-  streamSetStatusNormal(pTask);
+  streamTaskResetStatus(pTask);
+
+  // continue after lock the meta again
+  streamMetaWLock(pMeta);
 
   SStreamTask** ppHTask = NULL;
-  if (pTask->hTaskInfo.id.taskId != 0) {
+  if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
     if (ppHTask == NULL || *ppHTask == NULL) {
       tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
               pMeta->vgId, req.taskId);
+      CLEAR_RELATED_FILLHISTORY_TASK(pTask);
     } else {
       tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
       streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList);
@@ -1892,12 +1872,14 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   streamTaskStop(pTask);
-  taosHashPut(pMeta->pUpdateTaskSet, &pTask->id, sizeof(pTask->id), NULL, 0);
+
+  // keep the already handled info
+  taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0);
 
   if (ppHTask != NULL) {
     streamTaskStop(*ppHTask);
     tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr);
-    taosHashPut(pMeta->pUpdateTaskSet, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
+    taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
   } else {
     tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr);
   }
@@ -1906,49 +1888,56 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
 
   // possibly only handle the stream task.
   int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
-  int32_t updateTasks = taosHashGetSize(pMeta->pUpdateTaskSet);
+  int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);
 
-  pMeta->startInfo.startedAfterNodeUpdate = 1;
+  pMeta->startInfo.startAllTasksFlag = 1;
 
   if (updateTasks < numOfTasks) {
     tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
             updateTasks, (numOfTasks - updateTasks));
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
   } else {
-    taosHashClear(pMeta->pUpdateTaskSet);
-
     if (!pTq->pVnode->restored) {
       tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId);
-      pMeta->startInfo.startedAfterNodeUpdate = 0;
+      pMeta->startInfo.startAllTasksFlag = 0;
-      taosWUnLockLatch(&pMeta->lock);
+      streamMetaWUnLock(pMeta);
     } else {
-      tqDebug("vgId:%d tasks are all updated and stopped, restart them", vgId);
+      tqInfo("vgId:%d tasks are all updated and stopped, restart them", vgId);
-
       terrno = 0;
+
+      streamMetaWUnLock(pMeta);
+
+      while (streamMetaTaskInTimer(pMeta)) {
+        tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
+        taosMsleep(100);
+      }
+
+      streamMetaWLock(pMeta);
+
       int32_t code = streamMetaReopen(pMeta);
       if (code != 0) {
         tqError("vgId:%d failed to reopen stream meta", vgId);
-        taosWUnLockLatch(&pMeta->lock);
+        streamMetaWUnLock(pMeta);
         taosArrayDestroy(req.pNodeList);
         return -1;
       }
 
       if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) {
         tqError("vgId:%d failed to load stream tasks", vgId);
-        taosWUnLockLatch(&pMeta->lock);
+        streamMetaWUnLock(pMeta);
         taosArrayDestroy(req.pNodeList);
         return -1;
       }
 
       if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
-        vInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
+        tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
-        tqStartStreamTasks(pTq);
+        tqResetStreamTaskStatus(pTq);
-        tqCheckAndRunStreamTaskAsync(pTq);
+        tqLaunchStreamTaskAsync(pTq);
       } else {
-        vInfo("vgId:%d, follower node not start stream tasks", vgId);
+        tqInfo("vgId:%d, follower node not start stream tasks", vgId);
       }
 
-      taosWUnLockLatch(&pMeta->lock);
+      streamMetaWUnLock(pMeta);
     }
   }
 
@@ -1970,10 +1959,10 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
 
   // clear flag set during do checkpoint, and open inputQ for all upstream tasks
-  if (pTask->status.taskStatus == TASK_STATUS__CK) {
+  if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
     streamTaskClearCheckInfo(pTask);
     taosArrayClear(pTask->pReadyMsgList);
-    streamSetStatusNormal(pTask);
+    streamTaskSetStatusReady(pTask);
   }
 
   streamMetaReleaseTask(pMeta, pTask);

source/dnode/vnode/src/tq/tqPush.c

@@ -35,11 +35,11 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
     tqProcessSubmitReqForSubscribe(pTq);
   }
 
-  taosRLockLatch(&pTq->pStreamMeta->lock);
+  streamMetaRLock(pTq->pStreamMeta);
   int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
-  taosRUnLockLatch(&pTq->pStreamMeta->lock);
+  streamMetaRUnLock(pTq->pStreamMeta);
 
-  tqDebug("handle submit, restore:%d, numOfTasks:%d", pTq->pVnode->restored, numOfTasks);
+//  tqTrace("vgId:%d handle submit, restore:%d, numOfTasks:%d", TD_VID(pTq->pVnode), pTq->pVnode->restored, numOfTasks);
 
   // push data for stream processing:
   // 1. the vnode has already been restored.

source/dnode/vnode/src/tq/tqRead.c

@@ -1111,7 +1111,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   taosWUnLockLatch(&pTq->lock);
 
   // update the table list handle for each stream scanner/wal reader
-  taosWLockLatch(&pTq->pStreamMeta->lock);
+  streamMetaWLock(pTq->pStreamMeta);
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasksMap, pIter);
     if (pIter == NULL) {
@@ -1128,6 +1128,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
     }
   }
 
-  taosWUnLockLatch(&pTq->pStreamMeta->lock);
+  streamMetaWUnLock(pTq->pStreamMeta);
   return 0;
 }

source/dnode/vnode/src/tq/tqSink.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <common/tmsg.h>
 #include "tcommon.h"
 #include "tmsg.h"
 #include "tq.h"
@@ -28,19 +29,19 @@ static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks);
 static int32_t tsAscendingSortFn(const void* p1, const void* p2);
 static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
                                   SSubmitTbData* pTableData);
-static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
-                                      SSubmitTbData* pTableData);
 static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                        int64_t suid);
 static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
 static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
-static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
+static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
 static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
                                         const char* dstTableName, int64_t* uid);
 static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id);
-static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
 static bool    isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
-static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock);
+static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags);
+static SArray* createDefaultTagColName();
+static void    setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                   int64_t gid);
 
 int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
                          const char* pIdStr) {
@@ -138,61 +139,68 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
+  pCreateTableReq->flags = 0;
+  pCreateTableReq->type = TSDB_CHILD_TABLE;
+  pCreateTableReq->ctb.suid = suid;
+
+  // set super table name
+  SName name = {0};
+  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+  pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
+
+  pCreateTableReq->ctb.tagNum = numOfTags;
+  return TSDB_CODE_SUCCESS;
+}
+
+SArray* createDefaultTagColName() {
+  SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
+  char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
+  taosArrayPush(pTagColNameList, tagNameStr);
+  return pTagColNameList;
+}
+
+void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
+                                int64_t gid) {
+  if (pDataBlock->info.parTbName[0]) {
+    pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
+  } else {
+    pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
+  }
+}
+
 static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
                                             int64_t suid) {
   tqDebug("s-task:%s build create table msg", pTask->id.idStr);
 
   STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
   int32_t   rows = pDataBlock->info.rows;
-  SArray*   tagArray = NULL;
+  SArray*   tagArray = taosArrayInit(4, sizeof(STagVal));;
   int32_t   code = 0;
 
   SVCreateTbBatchReq reqs = {0};
-
   SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
   if (NULL == reqs.pArray) {
+    tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
     goto _end;
   }
 
   for (int32_t rowId = 0; rowId < rows; rowId++) {
     SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0});
 
-    // set const
-    pCreateTbReq->flags = 0;
-    pCreateTbReq->type = TSDB_CHILD_TABLE;
-    pCreateTbReq->ctb.suid = suid;
-
-    // set super table name
-    SName name = {0};
-    tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-    pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));  // taosStrdup(stbFullName);
-
-    // set tag content
     int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
-    if (size == 2) {
+    int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
-      tagArray = taosArrayInit(1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
 
+    initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
+    taosArrayClear(tagArray);
+
+    if (size == 2) {
       STagVal tagVal = {
           .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
 
       taosArrayPush(tagArray, &tagVal);
-
+      pCreateTbReq->ctb.tagName = createDefaultTagColName();
-      // set tag name
-      SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-      char    tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
-      taosArrayPush(tagName, tagNameStr);
-      pCreateTbReq->ctb.tagName = tagName;
     } else {
-      tagArray = taosArrayInit(size - 1, sizeof(STagVal));
-      if (!tagArray) {
-        tdDestroySVCreateTbReq(pCreateTbReq);
-        goto _end;
-      }
-
       for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
         SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
 
@@ -209,29 +217,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
         taosArrayPush(tagArray, &tagVal);
       }
     }
-    pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
 
-    STag* pTag = NULL;
+    tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
-    tTagNew(tagArray, 1, false, &pTag);
     tagArray = taosArrayDestroy(tagArray);
-    if (pTag == NULL) {
+    if (pCreateTbReq->ctb.pTag == NULL) {
       tdDestroySVCreateTbReq(pCreateTbReq);
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _end;
     }
 
-    pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
+    uint64_t gid = pDataBlock->info.id.groupId;
-
+    if (taosArrayGetSize(pDataBlock->pDataBlock) > UD_GROUPID_COLUMN_INDEX) {
-    // set table name
-    if (!pDataBlock->info.parTbName[0]) {
       SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);
 
+      // todo remove this
       void* pGpIdData = colDataGetData(pGpIdColInfo, rowId);
-      pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData);
+      ASSERT(gid == *(int64_t*)pGpIdData);
-    } else {
-      pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
     }
 
+    setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid);
+
     taosArrayPush(reqs.pArray, pCreateTbReq);
     tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
   }
@@ -330,6 +335,9 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
 
   tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id,
           (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL));
+
+  tdDestroySVCreateTbReq(pNew->pCreateTbReq);
+  taosMemoryFree(pNew->pCreateTbReq);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -390,60 +398,33 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
   return true;
 }
 
-SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) {
+SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
-  char* ctbName = pDataBlock->info.parTbName;
+                                       SSDataBlock* pDataBlock, SArray* pTagArray) {
-
+  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
-  SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq));
   if (pCreateTbReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  // set tag content
+  taosArrayClear(pTagArray);
-  SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
+  initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
-  if (tagArray == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tdDestroySVCreateTbReq(pCreateTbReq);
-    taosMemoryFreeClear(pCreateTbReq);
-    return NULL;
-  }
-
-  // set const
-  pCreateTbReq->flags = 0;
-  pCreateTbReq->type = TSDB_CHILD_TABLE;
-  pCreateTbReq->ctb.suid = suid;
-
-  // set super table name
-  SName name = {0};
-  tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
 
   STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
-  taosArrayPush(tagArray, &tagVal);
+  taosArrayPush(pTagArray, &tagVal);
-  pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
 
-  STag* pTag = NULL;
+  tTagNew(pTagArray, 1, false, (STag**) &pCreateTbReq->ctb.pTag);
-  tTagNew(tagArray, 1, false, &pTag);
-  taosArrayDestroy(tagArray);
 
-  if (pTag == NULL) {
+  if (pCreateTbReq->ctb.pTag == NULL) {
     tdDestroySVCreateTbReq(pCreateTbReq);
     taosMemoryFreeClear(pCreateTbReq);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  pCreateTbReq->ctb.pTag = (uint8_t*)pTag;
+  pCreateTbReq->ctb.tagName = createDefaultTagColName();
-
-  // set tag name
-  SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
-  char k[TSDB_COL_NAME_LEN] = "group_id";
-  taosArrayPush(tagName, k);
-
-  pCreateTbReq->ctb.tagName = tagName;
 
   // set table name
-  pCreateTbReq->name = taosStrdup(ctbName);
+  setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId);
   return pCreateTbReq;
 }
 
@@ -514,7 +495,7 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) {
   }
 }
 
-int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
+int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
   int32_t numOfRows = pDataBlock->info.rows;
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -592,7 +573,7 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
   const char* id = pTask->id.idStr;
 
   while (pTableSinkInfo->uid == 0) {
-    if (streamTaskShouldStop(&pTask->status)) {
+    if (streamTaskShouldStop(pTask)) {
       tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName);
       return TSDB_CODE_STREAM_EXEC_CANCELLED;
     }
@@ -693,8 +674,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
 
       tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);
 
+      SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
+
       pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
-      pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock);
+      pTableData->pCreateTbReq =
+          buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray);
+      taosArrayDestroy(pTagArray);
+
       if (pTableData->pCreateTbReq == NULL) {
         tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno));
         taosMemoryFree(pTableSinkInfo);
@@ -724,17 +710,16 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
+int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
-                               SSubmitTbData* pTableData) {
+                               SSubmitTbData* pTableData, const char* id) {
   int32_t numOfRows = pDataBlock->info.rows;
-  const char* id = pTask->id.idStr;
 
   tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64,
-          id, blockIndex + 1, numOfRows, pTask->outputInfo.tbSink.stbUid);
+          id, blockIndex + 1, numOfRows, suid);
   char* dstTableName = pDataBlock->info.parTbName;
 
   // convert all rows
-  int32_t code = doConvertRows(pTableData, pTask->outputInfo.tbSink.pTSchema, pDataBlock, id);
+  int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id);
   if (code != TSDB_CODE_SUCCESS) {
     tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno));
     return code;
@@ -773,7 +758,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
             numOfBlocks);
 
     for(int32_t i = 0; i < numOfBlocks; ++i) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
         return;
       }
 
@@ -800,7 +785,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
@@ -823,7 +808,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
 
     bool hasSubmit = false;
     for (int32_t i = 0; i < numOfBlocks; i++) {
-      if (streamTaskShouldStop(&pTask->status)) {
+      if (streamTaskShouldStop(pTask)) {
         taosHashCleanup(pTableIndexMap);
         tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
         return;
@@ -847,7 +832,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           continue;
         }
 
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }
@@ -857,7 +842,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
         int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
         taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
       } else {
-        code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData);
+        code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
         }

source/dnode/vnode/src/tq/tqStreamTask.c

@@ -38,19 +38,17 @@ int32_t tqScanWal(STQ* pTq) {
     doScanWalForAllTasks(pTq->pStreamMeta, &shouldIdle);
 
     if (shouldIdle) {
-      taosWLockLatch(&pMeta->lock);
+      streamMetaWLock(pMeta);
-
       int32_t times = (--pMeta->walScanCounter);
       ASSERT(pMeta->walScanCounter >= 0);
+      streamMetaWUnLock(pMeta);
 
-      if (pMeta->walScanCounter <= 0) {
+      if (times <= 0) {
-        taosWUnLockLatch(&pMeta->lock);
         break;
-      }
+      } else {
-
-      taosWUnLockLatch(&pMeta->lock);
         tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION);
       }
+    }
 
     taosMsleep(SCAN_WAL_IDLE_DURATION);
   }
@@ -60,7 +58,8 @@ int32_t tqScanWal(STQ* pTq) {
   return 0;
 }
 
-int32_t tqCheckAndRunStreamTask(STQ* pTq) {
+int32_t tqStartStreamTask(STQ* pTq) {
+  int32_t      code = TSDB_CODE_SUCCESS;
   int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
 
@@ -71,11 +70,11 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
   }
 
   SArray* pTaskList = NULL;
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
   pTaskList = taosArrayDup(pMeta->pTaskList, NULL);
   taosHashClear(pMeta->startInfo.pReadyTaskSet);
   pMeta->startInfo.startTs = taosGetTimestampMs();
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaWUnLock(pMeta);
 
   // broadcast the check downstream tasks msg
   for (int32_t i = 0; i < numOfTasks; ++i) {
@@ -92,29 +91,33 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
     }
 
     if (pTask->status.downstreamReady == 1) {
+      if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
         tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
                 pTask->id.idStr);
         streamLaunchFillHistoryTask(pTask);
+      }
+
+      streamMetaUpdateTaskReadyInfo(pTask);
       streamMetaReleaseTask(pMeta, pTask);
       continue;
     }
 
-    pTask->execInfo.init = taosGetTimestampMs();
+    EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
-    tqDebug("s-task:%s start check downstream tasks, set the init ts:%"PRId64, pTask->id.idStr, pTask->execInfo.init);
+    int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event);
-
+    if (ret != TSDB_CODE_SUCCESS) {
-    streamSetStatusNormal(pTask);
+      code = ret;
-    streamTaskCheckDownstream(pTask);
+    }
 
     streamMetaReleaseTask(pMeta, pTask);
   }
 
   taosArrayDestroy(pTaskList);
-  return 0;
+  return code;
 }
 
-int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq) {
+int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
-  int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
+  int32_t      vgId = pMeta->vgId;
 
   int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
   if (numOfTasks == 0) {
@@ -148,12 +151,12 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
     return TSDB_CODE_SUCCESS;
   }
 
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
 
   int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
   if (numOfTasks == 0) {
     tqDebug("vgId:%d no stream tasks existed to run", vgId);
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
     return 0;
   }
 
@@ -164,7 +167,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
 
   if (pMeta->walScanCounter > 1) {
     tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter);
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
     return 0;
   }
 
@@ -174,7 +177,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
 
     // reset the counter value, since we do not launch the scan wal operation.
     pMeta->walScanCounter = 0;
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
     return 0;
   }
 
@@ -182,7 +185,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
   if (pRunReq == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tqError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr());
-    taosWUnLockLatch(&pMeta->lock);
+    streamMetaWUnLock(pMeta);
     return -1;
   }
 
@@ -193,7 +196,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
 
   SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)};
   tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg);
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaWUnLock(pMeta);
 
   return 0;
 }
@@ -209,9 +212,9 @@ int32_t tqStopStreamTasks(STQ* pTq) {
   }
 
   SArray* pTaskList = NULL;
-  taosWLockLatch(&pMeta->lock);
+  streamMetaWLock(pMeta);
   pTaskList = taosArrayDup(pMeta->pTaskList, NULL);
-  taosWUnLockLatch(&pMeta->lock);
+  streamMetaWUnLock(pMeta);
 
   for (int32_t i = 0; i < numOfTasks; ++i) {
     SStreamTaskId*   pTaskId = taosArrayGet(pTaskList, i);
@@ -228,7 +231,7 @@ int32_t tqStopStreamTasks(STQ* pTq) {
   return 0;
 }
 
-int32_t tqStartStreamTasks(STQ* pTq) {
+int32_t tqResetStreamTaskStatus(STQ* pTq) {
   SStreamMeta* pMeta = pTq->pStreamMeta;
   int32_t      vgId = TD_VID(pTq->pVnode);
   int32_t      numOfTasks = taosArrayGetSize(pMeta->pTaskList);
@@ -243,11 +246,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
 
     STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId};
     SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
-
+    streamTaskResetStatus(*pTask);
-    int8_t status = (*pTask)->status.taskStatus;
-    if (status == TASK_STATUS__STOP && (*pTask)->info.fillHistory != 1) {
-      streamSetStatusNormal(*pTask);
-    }
   }
 
   return 0;
@@ -328,15 +327,17 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
   }
 
   // not in ready state, do not handle the data from wal
-  int32_t status = pTask->status.taskStatus;
+//  int32_t status = pTask->status.taskStatus;
-  if (status != TASK_STATUS__NORMAL) {
+  char* p = NULL;
-    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
+  int32_t status = streamTaskGetStatus(pTask, &p);
+  if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) {
+    tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, p);
     return false;
   }
 
   // fill-history task has entered into the last phase, no need to anything
   if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
-    ASSERT(status == TASK_STATUS__NORMAL);
+    ASSERT(status == TASK_STATUS__READY);
     // the maximum version of data in the WAL has reached already, the step2 is done
     tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
             pTask->dataRange.range.maxVer);
@@ -344,13 +345,13 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
   }
 
   // check if input queue is full or not
-  if (streamQueueIsFull(pTask->inputInfo.queue)) {
+  if (streamQueueIsFull(pTask->inputq.queue)) {
     tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr);
     return false;
   }
 
   // the input queue of downstream task is full, so the output is blocked, stopped for a while
-  if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) {
+  if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
     tqDebug("s-task:%s inputQ is blocked, do nothing", pTask->id.idStr);
     return false;
   }
@@ -414,9 +415,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
 
   // clone the task list, to avoid the task update during scan wal files
   SArray* pTaskList = NULL;
-  taosWLockLatch(&pStreamMeta->lock);
+  streamMetaWLock(pStreamMeta);
   pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL);
-  taosWUnLockLatch(&pStreamMeta->lock);
+  streamMetaWUnLock(pStreamMeta);
 
   tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
 
@@ -444,14 +445,16 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
       continue;
     }
 
-    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputInfo.queue);
+    int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
     int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX;
 
     taosThreadMutexLock(&pTask->lock);
+    tqDebug("s-task:%s lock", pTask->id.idStr);
 
-    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    char* p = NULL;
-    if (pTask->status.taskStatus != TASK_STATUS__NORMAL) {
+    ETaskStatus status = streamTaskGetStatus(pTask, &p);
-      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus);
+    if (status != TASK_STATUS__READY) {
+      tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, p);
       taosThreadMutexUnlock(&pTask->lock);
       streamMetaReleaseTask(pStreamMeta, pTask);
       continue;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -87,6 +87,41 @@ static void tsdbCloseBCache(STsdb *pTsdb) {
   }
 }
 
+static int32_t tsdbOpenPgCache(STsdb *pTsdb) {
+  int32_t code = 0;
+  // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5);
+  int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
+
+  SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3PageCacheSize * szPage, 0, .5);
+  if (pCache == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  taosLRUCacheSetStrictCapacity(pCache, false);
+
+  taosThreadMutexInit(&pTsdb->pgMutex, NULL);
+
+_err:
+  pTsdb->pgCache = pCache;
+  return code;
+}
+
+static void tsdbClosePgCache(STsdb *pTsdb) {
+  SLRUCache *pCache = pTsdb->pgCache;
+  if (pCache) {
+    int32_t elems = taosLRUCacheGetElems(pCache);
+    tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems);
+    taosLRUCacheEraseUnrefEntries(pCache);
+    elems = taosLRUCacheGetElems(pCache);
+    tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems);
+
+    taosLRUCacheCleanup(pCache);
+
+    taosThreadMutexDestroy(&pTsdb->bMutex);
+  }
+}
+
 #define ROCKS_KEY_LEN (sizeof(tb_uid_t) + sizeof(int16_t) + sizeof(int8_t))
 
 typedef struct {
@@ -1191,6 +1226,12 @@ int32_t tsdbOpenCache(STsdb *pTsdb) {
     goto _err;
   }
 
+  code = tsdbOpenPgCache(pTsdb);
+  if (code != TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
   code = tsdbOpenRocksCache(pTsdb);
   if (code != TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1221,6 +1262,7 @@ void tsdbCloseCache(STsdb *pTsdb) {
 
   tsdbCloseBICache(pTsdb);
   tsdbCloseBCache(pTsdb);
+  tsdbClosePgCache(pTsdb);
   tsdbCloseRocksCache(pTsdb);
 }
 
@@ -3057,7 +3099,6 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
   }
   */
   int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
-  // int64_t size = 4096;
   code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock);
   if (code != TSDB_CODE_SUCCESS) {
     // taosMemoryFree(pBlock);
@@ -3123,10 +3164,42 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle)
   return code;
 }
 
-int32_t tsdbBCacheRelease(SLRUCache *pCache, LRUHandle *h) {
+int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle) {
   int32_t code = 0;
+  char    key[128] = {0};
+  int     keyLen = 0;
 
-  taosLRUCacheRelease(pCache, h, false);
+  getBCacheKey(pFD->fid, pFD->cid, pgno, key, &keyLen);
+  *handle = taosLRUCacheLookup(pCache, key, keyLen);
+
+  return code;
+}
+
+int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage) {
+  int32_t    code = 0;
+  char       key[128] = {0};
+  int        keyLen = 0;
+  LRUHandle *handle = NULL;
+
+  getBCacheKey(pFD->fid, pFD->cid, pgno, key, &keyLen);
+  taosThreadMutexLock(&pFD->pTsdb->pgMutex);
+  handle = taosLRUCacheLookup(pFD->pTsdb->pgCache, key, keyLen);
+  if (!handle) {
+    size_t              charge = pFD->szPage;
+    _taos_lru_deleter_t deleter = deleteBCache;
+    uint8_t            *pPg = taosMemoryMalloc(charge);
+    memcpy(pPg, pPage, charge);
+
+    LRUStatus status =
+        taosLRUCacheInsert(pCache, key, keyLen, pPg, charge, deleter, &handle, TAOS_LRU_PRIORITY_LOW, NULL);
+    if (status != TAOS_LRU_STATUS_OK) {
+      // ignore cache updating if not ok
+      // code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  taosThreadMutexUnlock(&pFD->pTsdb->pgMutex);
+
+  tsdbCacheRelease(pFD->pTsdb->pgCache, handle);
 
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -131,7 +131,7 @@ int32_t tsdbBegin(STsdb *pTsdb) {
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // lock
-  if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) {
+  if ((code = taosThreadMutexLock(&pTsdb->mutex))) {
     code = TAOS_SYSTEM_ERROR(code);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) {
   pTsdb->mem = pMemTable;
 
   // unlock
-  if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) {
+  if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) {
     code = TAOS_SYSTEM_ERROR(code);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -152,11 +152,11 @@ _exit:
 }
 
 int32_t tsdbPrepareCommit(STsdb *pTsdb) {
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
+  taosThreadMutexLock(&pTsdb->mutex);
   ASSERT(pTsdb->imem == NULL);
   pTsdb->imem = pTsdb->mem;
   pTsdb->mem = NULL;
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
+  taosThreadMutexUnlock(&pTsdb->mutex);
 
   return 0;
 }
@@ -171,9 +171,9 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) {
 
   // check
   if (pMemTable->nRow == 0 && pMemTable->nDel == 0) {
-    taosThreadRwlockWrlock(&pTsdb->rwLock);
+    taosThreadMutexLock(&pTsdb->mutex);
     pTsdb->imem = NULL;
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
+    taosThreadMutexUnlock(&pTsdb->mutex);
 
     tsdbUnrefMemTable(pMemTable, NULL, true);
     goto _exit;
@@ -501,6 +501,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   int32_t    lino = 0;
   STsdb     *pTsdb = pCommitter->pTsdb;
   SDFileSet *pRSet = NULL;
+
   // memory
   pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision);
   pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec());
@@ -798,6 +799,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
   int32_t    lino = 0;
   STsdb     *pTsdb = pCommitter->pTsdb;
   SMemTable *pMemTable = pTsdb->imem;
+
   // commit file data start
   code = tsdbCommitFileDataStart(pCommitter);
   TSDB_CHECK_CODE(code, lino, _exit);
@@ -1650,18 +1652,18 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) {
   SMemTable *pMemTable = pTsdb->imem;
 
   // lock
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
+  taosThreadMutexLock(&pTsdb->mutex);
 
   code = tsdbFSCommit(pTsdb);
   if (code) {
-    taosThreadRwlockUnlock(&pTsdb->rwLock);
+    taosThreadMutexUnlock(&pTsdb->mutex);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   pTsdb->imem = NULL;
 
   // unlock
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
+  taosThreadMutexUnlock(&pTsdb->mutex);
   if (pMemTable) {
     tsdbUnrefMemTable(pMemTable, NULL, true);
   }

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -46,6 +46,7 @@ typedef struct {
     STFileSet *fset;
     TABLEID    tbid[1];
     bool       hasTSData;
+    bool       skipTsRow;
   } ctx[1];
 
   // reader
@@ -127,18 +128,18 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
         continue;
       }
     }
-
+    /*
     extern int8_t tsS3Enabled;
 
     int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
-    bool    skipRow = false;
+    committer->ctx->skipTsRow = false;
     if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) {
-      skipRow = true;
+      committer->ctx->skipTsRow = true;
     }
-
+    */
     int64_t ts = TSDBROW_TS(&row->row);
 
-    if (skipRow && ts <= committer->ctx->maxKey) {
+    if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) {
       ts = committer->ctx->maxKey + 1;
     }
 
@@ -367,7 +368,12 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   int32_t lino = 0;
   STsdb  *tsdb = committer->tsdb;
 
-  committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
+  int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
+
+  // check if can commit
+  tsdbFSCheckCommit(tsdb, fid);
+
+  committer->ctx->fid = fid;
   committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
   tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
                   &committer->ctx->maxKey);
@@ -397,6 +403,32 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   // reset nextKey
   committer->ctx->nextKey = TSKEY_MAX;
 
+  committer->ctx->skipTsRow = false;
+
+  extern int8_t  tsS3Enabled;
+  extern int32_t tsS3UploadDelaySec;
+  long           s3Size(const char *object_name);
+  int32_t        nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs);
+  committer->ctx->skipTsRow = false;
+  if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) {
+    STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA];
+    if (fobj && fobj->f->did.level == nlevel - 1) {
+      // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
+      const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
+
+      if (taosCheckExistFile(fobj->fname)) {
+        int32_t mtime = 0;
+        taosStatFile(fobj->fname, NULL, &mtime, NULL);
+        if (mtime < committer->ctx->now - tsS3UploadDelaySec) {
+          committer->ctx->skipTsRow = true;
+        }
+      } else if (s3Size(object_name) > 0) {
+        committer->ctx->skipTsRow = true;
+      }
+    }
+    // new fset can be written with ts data
+  }
+
 _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
@@ -549,11 +581,11 @@ _exit:
 }
 
 int32_t tsdbPreCommit(STsdb *tsdb) {
-  taosThreadRwlockWrlock(&tsdb->rwLock);
+  taosThreadMutexLock(&tsdb->mutex);
   ASSERT(tsdb->imem == NULL);
   tsdb->imem = tsdb->mem;
   tsdb->mem = NULL;
-  taosThreadRwlockUnlock(&tsdb->rwLock);
+  taosThreadMutexUnlock(&tsdb->mutex);
   return 0;
 }
 
@@ -568,15 +600,13 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
   int64_t    nDel = imem->nDel;
 
   if (nRow == 0 && nDel == 0) {
-    taosThreadRwlockWrlock(&tsdb->rwLock);
+    taosThreadMutexLock(&tsdb->mutex);
     tsdb->imem = NULL;
-    taosThreadRwlockUnlock(&tsdb->rwLock);
+    taosThreadMutexUnlock(&tsdb->mutex);
     tsdbUnrefMemTable(imem, NULL, true);
   } else {
     SCommitter2 committer[1];
 
-    tsdbFSCheckCommit(tsdb->pFS);
-
     code = tsdbOpenCommitter(tsdb, info, committer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -605,14 +635,14 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
   if (tsdb->imem == NULL) goto _exit;
 
   SMemTable *pMemTable = tsdb->imem;
-  taosThreadRwlockWrlock(&tsdb->rwLock);
+  taosThreadMutexLock(&tsdb->mutex);
   code = tsdbFSEditCommit(tsdb->pFS);
   if (code) {
-    taosThreadRwlockUnlock(&tsdb->rwLock);
+    taosThreadMutexUnlock(&tsdb->mutex);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
   tsdb->imem = NULL;
-  taosThreadRwlockUnlock(&tsdb->rwLock);
+  taosThreadMutexUnlock(&tsdb->mutex);
   tsdbUnrefMemTable(pMemTable, NULL, true);
 
 _exit:

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -55,25 +55,11 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) {
   TARRAY2_INIT(fs[0]->fSetArr);
   TARRAY2_INIT(fs[0]->fSetArrTmp);
 
-  // background task queue
-  taosThreadMutexInit(fs[0]->mutex, NULL);
-  fs[0]->bgTaskQueue->next = fs[0]->bgTaskQueue;
-  fs[0]->bgTaskQueue->prev = fs[0]->bgTaskQueue;
-
-  taosThreadMutexInit(&fs[0]->commitMutex, NULL);
-  taosThreadCondInit(&fs[0]->canCommit, NULL);
-  fs[0]->blockCommit = false;
-
   return 0;
 }
 
 static int32_t destroy_fs(STFileSystem **fs) {
   if (fs[0] == NULL) return 0;
-  taosThreadMutexDestroy(&fs[0]->commitMutex);
-  taosThreadCondDestroy(&fs[0]->canCommit);
-  taosThreadMutexDestroy(fs[0]->mutex);
-
-  ASSERT(fs[0]->bgTaskNum == 0);
 
   TARRAY2_DESTROY(fs[0]->fSetArr, NULL);
   TARRAY2_DESTROY(fs[0]->fSetArrTmp, NULL);
@@ -264,10 +250,11 @@ static int32_t apply_commit(STFileSystem *fs) {
     if (fset1 && fset2) {
       if (fset1->fid < fset2->fid) {
         // delete fset1
-        TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove);
+        tsdbTFileSetRemove(fset1);
+        i1++;
       } else if (fset1->fid > fset2->fid) {
         // create new file set with fid of fset2->fid
-        code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1);
+        code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
         if (code) return code;
         code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
         if (code) return code;
@@ -282,10 +269,11 @@ static int32_t apply_commit(STFileSystem *fs) {
       }
     } else if (fset1) {
       // delete fset1
-      TARRAY2_REMOVE(fsetArray1, i1, tsdbTFileSetRemove);
+      tsdbTFileSetRemove(fset1);
+      i1++;
     } else {
       // create new file set with fid of fset2->fid
-      code = tsdbTFileSetInitDup(fs->tsdb, fset2, &fset1);
+      code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1);
       if (code) return code;
       code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn);
       if (code) return code;
@@ -512,7 +500,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
         TARRAY2_FOREACH(lvl->fobjArr, fobj) {
           code = tsdbFSDoScanAndFixFile(fs, fobj);
           if (code) {
-            fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
+            fset->maxVerValid =
+                (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
             corrupt = true;
           }
         }
@@ -592,7 +581,7 @@ static int32_t tsdbFSDupState(STFileSystem *fs) {
   const STFileSet *fset1;
   TARRAY2_FOREACH(src, fset1) {
     STFileSet *fset2;
-    code = tsdbTFileSetInitDup(fs->tsdb, fset1, &fset2);
+    code = tsdbTFileSetInitCopy(fs->tsdb, fset1, &fset2);
     if (code) return code;
     code = TARRAY2_APPEND(dst, fset2);
     if (code) return code;
@@ -665,12 +654,6 @@ static int32_t close_file_system(STFileSystem *fs) {
   return 0;
 }
 
-static int32_t apply_edit(STFileSystem *pFS) {
-  int32_t code = 0;
-  ASSERTS(0, "TODO: Not implemented yet");
-  return code;
-}
-
 static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSet *pSet2) {
   if (pSet1->fid < pSet2->fid) {
     return -1;
@@ -710,10 +693,23 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) {
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  // remove empty file set
+  // remove empty empty stt level and empty file set
   int32_t i = 0;
   while (i < TARRAY2_SIZE(fsetArray)) {
     fset = TARRAY2_GET(fsetArray, i);
+
+    SSttLvl *lvl;
+    int32_t  j = 0;
+    while (j < TARRAY2_SIZE(fset->lvlArr)) {
+      lvl = TARRAY2_GET(fset->lvlArr, j);
+
+      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
+        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
+      } else {
+        j++;
+      }
+    }
+
     if (tsdbTFileSetIsEmpty(fset)) {
       TARRAY2_REMOVE(fsetArray, i, tsdbTFileSetClear);
     } else {
@@ -753,13 +749,13 @@ _exit:
 
 static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) {
   task->numWait++;
-  taosThreadCondWait(task->done, fs->mutex);
+  taosThreadCondWait(task->done, &fs->tsdb->mutex);
   task->numWait--;
 
   if (task->numWait == 0) {
     taosThreadCondDestroy(task->done);
-    if (task->free) {
+    if (task->destroy) {
-      task->free(task->arg);
+      task->destroy(task->arg);
     }
     taosMemoryFree(task);
   }
@@ -770,8 +766,8 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) {
     taosThreadCondBroadcast(task->done);
   } else {
     taosThreadCondDestroy(task->done);
-    if (task->free) {
+    if (task->destroy) {
-      task->free(task->arg);
+      task->destroy(task->arg);
     }
     taosMemoryFree(task);
   }
@@ -780,23 +776,16 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) {
 int32_t tsdbCloseFS(STFileSystem **fs) {
   if (fs[0] == NULL) return 0;
 
-  taosThreadMutexLock(fs[0]->mutex);
+  tsdbFSDisableBgTask(fs[0]);
-  fs[0]->stop = true;
-
-  if (fs[0]->bgTaskRunning) {
-    tsdbDoWaitBgTask(fs[0], fs[0]->bgTaskRunning);
-  }
-  taosThreadMutexUnlock(fs[0]->mutex);
-
   close_file_system(fs[0]);
   destroy_fs(fs);
   return 0;
 }
 
 int64_t tsdbFSAllocEid(STFileSystem *fs) {
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   int64_t cid = ++fs->neid;
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
   return cid;
 }
 
@@ -837,27 +826,34 @@ _exit:
   return code;
 }
 
-static int32_t tsdbFSSetBlockCommit(STFileSystem *fs, bool block) {
+static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block) {
-  taosThreadMutexLock(&fs->commitMutex);
   if (block) {
-    fs->blockCommit = true;
+    fset->blockCommit = true;
   } else {
-    fs->blockCommit = false;
+    fset->blockCommit = false;
-    taosThreadCondSignal(&fs->canCommit);
+    if (fset->numWaitCommit > 0) {
+      taosThreadCondSignal(&fset->canCommit);
+    }
   }
-  taosThreadMutexUnlock(&fs->commitMutex);
   return 0;
 }
 
-int32_t tsdbFSCheckCommit(STFileSystem *fs) {
+int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) {
-  taosThreadMutexLock(&fs->commitMutex);
+  taosThreadMutexLock(&tsdb->mutex);
-  while (fs->blockCommit) {
+  STFileSet *fset;
-    taosThreadCondWait(&fs->canCommit, &fs->commitMutex);
+  tsdbFSGetFSet(tsdb->pFS, fid, &fset);
+  if (fset) {
+    while (fset->blockCommit) {
+      fset->numWaitCommit++;
+      taosThreadCondWait(&fset->canCommit, &tsdb->mutex);
+      fset->numWaitCommit--;
     }
-  taosThreadMutexUnlock(&fs->commitMutex);
+  }
+  taosThreadMutexUnlock(&tsdb->mutex);
   return 0;
 }
 
+// IMPORTANT: the caller must hold fs->tsdb->mutex
 int32_t tsdbFSEditCommit(STFileSystem *fs) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -867,36 +863,57 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // schedule merge
-  if (fs->tsdb->pVnode->config.sttTrigger > 1) {
-    STFileSet *fset;
   int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger;
-    bool       schedMerge = false;
+  if (sttTrigger > 1) {
-    bool       blockCommit = false;
+    STFileSet *fset;
-
     TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) {
-      if (TARRAY2_SIZE(fset->lvlArr) == 0) continue;
+      if (TARRAY2_SIZE(fset->lvlArr) == 0) {
+        tsdbFSSetBlockCommit(fset, false);
+        continue;
+      }
 
       SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr);
-      if (lvl->level != 0) continue;
+      if (lvl->level != 0) {
+        tsdbFSSetBlockCommit(fset, false);
+        continue;
+      }
 
       int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
       if (numFile >= sttTrigger) {
-        schedMerge = true;
+        // launch merge
-      }
+        code = tsdbSchedMerge(fs->tsdb, fset->fid);
-
-      if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
-        blockCommit = true;
-      }
-
-      if (schedMerge && blockCommit) break;
-    }
-
-    if (schedMerge) {
-      code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, NULL, fs->tsdb, NULL);
         TSDB_CHECK_CODE(code, lino, _exit);
       }
 
-    tsdbFSSetBlockCommit(fs, blockCommit);
+      if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) {
+        tsdbFSSetBlockCommit(fset, true);
+      } else {
+        tsdbFSSetBlockCommit(fset, false);
+      }
+    }
+  }
+
+  // clear empty level and fset
+  int32_t i = 0;
+  while (i < TARRAY2_SIZE(fs->fSetArr)) {
+    STFileSet *fset = TARRAY2_GET(fs->fSetArr, i);
+
+    int32_t j = 0;
+    while (j < TARRAY2_SIZE(fset->lvlArr)) {
+      SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j);
+
+      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
+        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
+      } else {
+        j++;
+      }
+    }
+
+    if (tsdbTFileSetIsEmpty(fset) && fset->bgTaskRunning == NULL) {
+      TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear);
+    } else {
+      i++;
+    }
   }
 
 _exit:
@@ -933,15 +950,15 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
 
   TARRAY2_INIT(fsetArr[0]);
 
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   TARRAY2_FOREACH(fs->fSetArr, fset) {
-    code = tsdbTFileSetInitDup(fs->tsdb, fset, &fset1);
+    code = tsdbTFileSetInitCopy(fs->tsdb, fset, &fset1);
     if (code) break;
 
     code = TARRAY2_APPEND(fsetArr[0], fset1);
     if (code) break;
   }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   if (code) {
     TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear);
@@ -961,9 +978,9 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) {
 }
 
 int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr);
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
   return code;
 }
 
@@ -1017,7 +1034,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
     }
   }
 
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   TARRAY2_FOREACH(fs->fSetArr, fset) {
     int64_t ever = VERSION_MAX;
     if (pHash) {
@@ -1034,7 +1051,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
     code = TARRAY2_APPEND(fsetArr[0], fset1);
     if (code) break;
   }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
 
 _out:
   if (code) {
@@ -1089,7 +1106,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
     }
   }
 
-  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   TARRAY2_FOREACH(fs->fSetArr, fset) {
     int64_t sver1 = sver;
     int64_t ever1 = ever;
@@ -1118,7 +1135,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     fsr1 = NULL;
   }
-  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   if (code) {
     tsdbTSnapRangeClear(&fsr1);
@@ -1137,59 +1154,69 @@ _out:
 const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"};
 
 static int32_t tsdbFSRunBgTask(void *arg) {
-  STFileSystem *fs = (STFileSystem *)arg;
+  STFSBgTask   *task = (STFSBgTask *)arg;
+  STFileSystem *fs = task->fs;
+  STFileSet    *fset;
 
-  ASSERT(fs->bgTaskRunning != NULL);
+  tsdbFSGetFSet(fs, task->fid, &fset);
 
-  fs->bgTaskRunning->launchTime = taosGetTimestampMs();
+  ASSERT(fset != NULL && fset->bgTaskRunning == task);
-  fs->bgTaskRunning->run(fs->bgTaskRunning->arg);
+
-  fs->bgTaskRunning->finishTime = taosGetTimestampMs();
+  task->launchTime = taosGetTimestampMs();
+  task->run(task->arg);
+  task->finishTime = taosGetTimestampMs();
 
   tsdbDebug("vgId:%d bg task:%s task id:%" PRId64 " finished, schedule time:%" PRId64 " launch time:%" PRId64
             " finish time:%" PRId64,
-            TD_VID(fs->tsdb->pVnode), gFSBgTaskName[fs->bgTaskRunning->type], fs->bgTaskRunning->taskid,
+            TD_VID(fs->tsdb->pVnode), gFSBgTaskName[task->type], task->taskid, task->scheduleTime, task->launchTime,
-            fs->bgTaskRunning->scheduleTime, fs->bgTaskRunning->launchTime, fs->bgTaskRunning->finishTime);
+            task->finishTime);
 
-  taosThreadMutexLock(fs->mutex);
+  taosThreadMutexLock(&fs->tsdb->mutex);
 
   // free last
-  tsdbDoDoneBgTask(fs, fs->bgTaskRunning);
+  tsdbDoDoneBgTask(fs, task);
-  fs->bgTaskRunning = NULL;
+  fset->bgTaskRunning = NULL;
 
   // schedule next
-  if (fs->bgTaskNum > 0) {
+  if (fset->bgTaskNum > 0) {
     if (fs->stop) {
-      while (fs->bgTaskNum > 0) {
+      while (fset->bgTaskNum > 0) {
-        STFSBgTask *task = fs->bgTaskQueue->next;
+        STFSBgTask *nextTask = fset->bgTaskQueue->next;
-        task->prev->next = task->next;
+        nextTask->prev->next = nextTask->next;
-        task->next->prev = task->prev;
+        nextTask->next->prev = nextTask->prev;
-        fs->bgTaskNum--;
+        fset->bgTaskNum--;
-        tsdbDoDoneBgTask(fs, task);
+        tsdbDoDoneBgTask(fs, nextTask);
       }
     } else {
       // pop task from head
-      fs->bgTaskRunning = fs->bgTaskQueue->next;
+      fset->bgTaskRunning = fset->bgTaskQueue->next;
-      fs->bgTaskRunning->prev->next = fs->bgTaskRunning->next;
+      fset->bgTaskRunning->prev->next = fset->bgTaskRunning->next;
-      fs->bgTaskRunning->next->prev = fs->bgTaskRunning->prev;
+      fset->bgTaskRunning->next->prev = fset->bgTaskRunning->prev;
-      fs->bgTaskNum--;
+      fset->bgTaskNum--;
-      vnodeScheduleTaskEx(1, tsdbFSRunBgTask, arg);
+      vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fset->bgTaskRunning);
     }
   }
 
-  taosThreadMutexUnlock(fs->mutex);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
   return 0;
 }
 
-static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int32_t (*run)(void *),
+// IMPORTANT: the caller must hold the fs->tsdb->mutex
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *),
                              void (*destroy)(void *), void *arg, int64_t *taskid) {
   if (fs->stop) {
     if (destroy) {
       destroy(arg);
     }
-    return 0;  // TODO: use a better error code
+    return 0;
   }
 
-  for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) {
+  STFileSet *fset;
+  tsdbFSGetFSet(fs, fid, &fset);
+
+  ASSERT(fset != NULL);
+
+  for (STFSBgTask *task = fset->bgTaskQueue->next; task != fset->bgTaskQueue; task = task->next) {
     if (task->type == type) {
       if (destroy) {
         destroy(arg);
@@ -1203,22 +1230,24 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int
   if (task == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   taosThreadCondInit(task->done, NULL);
 
+  task->fs = fs;
+  task->fid = fid;
   task->type = type;
   task->run = run;
-  task->free = destroy;
+  task->destroy = destroy;
   task->arg = arg;
   task->scheduleTime = taosGetTimestampMs();
   task->taskid = ++fs->taskid;
 
-  if (fs->bgTaskRunning == NULL && fs->bgTaskNum == 0) {
+  if (fset->bgTaskRunning == NULL && fset->bgTaskNum == 0) {
     // launch task directly
-    fs->bgTaskRunning = task;
+    fset->bgTaskRunning = task;
-    vnodeScheduleTaskEx(1, tsdbFSRunBgTask, fs);
+    vnodeScheduleTaskEx(1, tsdbFSRunBgTask, task);
   } else {
     // add to the queue tail
-    fs->bgTaskNum++;
+    fset->bgTaskNum++;
-    task->next = fs->bgTaskQueue;
+    task->next = fset->bgTaskQueue;
-    task->prev = fs->bgTaskQueue->prev;
+    task->prev = fset->bgTaskQueue->prev;
     task->prev->next = task;
     task->next->prev = task;
   }
@@ -1227,68 +1256,30 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int
   return 0;
 }
 
-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
+int32_t tsdbFSDisableBgTask(STFileSystem *fs) {
-                             int64_t *taskid) {
+  taosThreadMutexLock(&fs->tsdb->mutex);
-  taosThreadMutexLock(fs->mutex);
+  for (;;) {
-  int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, free, arg, taskid);
+    fs->stop = true;
-  taosThreadMutexUnlock(fs->mutex);
+    bool done = true;
-  return code;
-}
 
-int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid) {
+    STFileSet *fset;
-  STFSBgTask *task = NULL;
+    TARRAY2_FOREACH(fs->fSetArr, fset) {
-
+      if (fset->bgTaskRunning) {
-  taosThreadMutexLock(fs->mutex);
+        tsdbDoWaitBgTask(fs, fset->bgTaskRunning);
-
+        done = false;
-  if (fs->bgTaskRunning && fs->bgTaskRunning->taskid == taskid) {
-    task = fs->bgTaskRunning;
-  } else {
-    for (STFSBgTask *taskt = fs->bgTaskQueue->next; taskt != fs->bgTaskQueue; taskt = taskt->next) {
-      if (taskt->taskid == taskid) {
-        task = taskt;
         break;
       }
     }
-  }
 
-  if (task) {
+    if (done) break;
-    tsdbDoWaitBgTask(fs, task);
   }
-
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
-  taosThreadMutexUnlock(fs->mutex);
   return 0;
 }
 
-int32_t tsdbFSWaitAllBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
-
-  while (fs->bgTaskRunning) {
-    taosThreadCondWait(fs->bgTaskRunning->done, fs->mutex);
-  }
-
-  taosThreadMutexUnlock(fs->mutex);
-  return 0;
-}
-
-static int32_t tsdbFSDoDisableBgTask(STFileSystem *fs) {
-  fs->stop = true;
-
-  if (fs->bgTaskRunning) {
-    tsdbDoWaitBgTask(fs, fs->bgTaskRunning);
-  }
-  return 0;
-}
-
-int32_t tsdbFSDisableBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
-  int32_t code = tsdbFSDoDisableBgTask(fs);
-  taosThreadMutexUnlock(fs->mutex);
-  return code;
-}
-
 int32_t tsdbFSEnableBgTask(STFileSystem *fs) {
-  taosThreadMutexLock(fs->mutex);
+  taosThreadMutexLock(&fs->tsdb->mutex);
   fs->stop = false;
-  taosThreadMutexUnlock(fs->mutex);
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
   return 0;
 }

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -22,22 +22,11 @@
 extern "C" {
 #endif
 
-/* Exposed Handle */
-typedef struct STFileSystem STFileSystem;
-typedef struct STFSBgTask   STFSBgTask;
-// typedef TARRAY2(STFileSet *) TFileSetArray;
-
 typedef enum {
   TSDB_FEDIT_COMMIT = 1,  //
   TSDB_FEDIT_MERGE
 } EFEditT;
 
-typedef enum {
-  TSDB_BG_TASK_MERGER = 1,
-  TSDB_BG_TASK_RETENTION,
-  TSDB_BG_TASK_COMPACT,
-} EFSBgTaskT;
-
 typedef enum {
   TSDB_FCURRENT = 1,
   TSDB_FCURRENT_C,  // for commit
@@ -67,37 +56,17 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
 int32_t tsdbFSEditCommit(STFileSystem *fs);
 int32_t tsdbFSEditAbort(STFileSystem *fs);
 // background task
-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, int32_t fid, EFSBgTaskT type, int32_t (*run)(void *),
-                             int64_t *taskid);
+                             void (*destroy)(void *), void *arg, int64_t *taskid);
-int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid);
-int32_t tsdbFSWaitAllBgTask(STFileSystem *fs);
 int32_t tsdbFSDisableBgTask(STFileSystem *fs);
 int32_t tsdbFSEnableBgTask(STFileSystem *fs);
 // other
 int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
-int32_t tsdbFSCheckCommit(STFileSystem *fs);
+int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid);
 // utils
 int32_t save_fs(const TFileSetArray *arr, const char *fname);
 int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
 
-struct STFSBgTask {
-  EFSBgTaskT type;
-  int32_t (*run)(void *arg);
-  void (*free)(void *arg);
-  void *arg;
-
-  TdThreadCond done[1];
-  int32_t      numWait;
-
-  int64_t taskid;
-  int64_t scheduleTime;
-  int64_t launchTime;
-  int64_t finishTime;
-
-  struct STFSBgTask *prev;
-  struct STFSBgTask *next;
-};
-
 /* Exposed Structs */
 struct STFileSystem {
   STsdb        *tsdb;
@@ -109,17 +78,8 @@ struct STFileSystem {
   TFileSetArray fSetArrTmp[1];
 
   // background task queue
-  TdThreadMutex mutex[1];
   bool    stop;
   int64_t taskid;
-  int32_t       bgTaskNum;
-  STFSBgTask    bgTaskQueue[1];
-  STFSBgTask   *bgTaskRunning;
-
-  // block commit variables
-  TdThreadMutex commitMutex;
-  TdThreadCond  canCommit;
-  bool          blockCommit;
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -342,11 +342,6 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) {
       int32_t    idx = TARRAY2_SEARCH_IDX(lvl->fobjArr, &tfobjp, tsdbTFileObjCmpr, TD_EQ);
       ASSERT(idx >= 0);
       TARRAY2_REMOVE(lvl->fobjArr, idx, tsdbSttLvlClearFObj);
-
-      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
-        // TODO: remove the stt level if no file exists anymore
-        // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear);
-      }
     } else {
       ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f));
       tsdbTFileObjUnref(fset->farr[op->of.type]);
@@ -454,10 +449,22 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
   fset[0]->fid = fid;
   fset[0]->maxVerValid = VERSION_MAX;
   TARRAY2_INIT(fset[0]->lvlArr);
+
+  // background task queue
+  fset[0]->bgTaskNum = 0;
+  fset[0]->bgTaskQueue->next = fset[0]->bgTaskQueue;
+  fset[0]->bgTaskQueue->prev = fset[0]->bgTaskQueue;
+  fset[0]->bgTaskRunning = NULL;
+
+  // block commit variables
+  taosThreadCondInit(&fset[0]->canCommit, NULL);
+  fset[0]->numWaitCommit = 0;
+  fset[0]->blockCommit = false;
+
   return 0;
 }
 
-int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) {
+int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) {
   int32_t code = tsdbTFileSetInit(fset1->fid, fset);
   if (code) return code;
 
@@ -588,21 +595,25 @@ int32_t tsdbTFileSetClear(STFileSet **fset) {
 
   TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear);
 
+  taosThreadCondDestroy(&fset[0]->canCommit);
   taosMemoryFree(fset[0]);
   fset[0] = NULL;
 
   return 0;
 }
 
-int32_t tsdbTFileSetRemove(STFileSet **fset) {
+int32_t tsdbTFileSetRemove(STFileSet *fset) {
+  if (fset == NULL) return 0;
+
   for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
-    if (fset[0]->farr[ftype] == NULL) continue;
+    if (fset->farr[ftype] != NULL) {
-    tsdbTFileObjRemove(fset[0]->farr[ftype]);
+      tsdbTFileObjRemove(fset->farr[ftype]);
+      fset->farr[ftype] = NULL;
+    }
   }
 
-  TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlRemove);
+  TARRAY2_DESTROY(fset->lvlArr, tsdbSttLvlRemove);
-  taosMemoryFree(fset[0]);
+
-  fset[0] = NULL;
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -28,6 +28,8 @@ typedef struct SSttLvl   SSttLvl;
 typedef TARRAY2(STFileObj *) TFileObjArray;
 typedef TARRAY2(SSttLvl *) TSttLvlArray;
 typedef TARRAY2(STFileOp) TFileOpArray;
+typedef struct STFileSystem STFileSystem;
+typedef struct STFSBgTask   STFSBgTask;
 
 typedef enum {
   TSDB_FOP_NONE = 0,
@@ -41,10 +43,10 @@ typedef enum {
 
 // init/clear
 int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset);
-int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
+int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetClear(STFileSet **fset);
-int32_t tsdbTFileSetRemove(STFileSet **fset);
+int32_t tsdbTFileSetRemove(STFileSet *fset);
 
 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
                                     TFileOpArray *fopArr);
@@ -58,6 +60,7 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset);
 // cmpr
 int32_t tsdbTFileSetCmprFn(const STFileSet **fset1, const STFileSet **fset2);
 // edit
+int32_t tsdbSttLvlClear(SSttLvl **lvl);
 int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op);
 int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *fset);
 // max commit id
@@ -70,6 +73,33 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset);
 int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl);
 int32_t tsdbSttLvlClear(SSttLvl **lvl);
 
+typedef enum {
+  TSDB_BG_TASK_MERGER = 1,
+  TSDB_BG_TASK_RETENTION,
+  TSDB_BG_TASK_COMPACT,
+} EFSBgTaskT;
+
+struct STFSBgTask {
+  STFileSystem *fs;
+  int32_t       fid;
+
+  EFSBgTaskT type;
+  int32_t (*run)(void *arg);
+  void (*destroy)(void *arg);
+  void *arg;
+
+  TdThreadCond done[1];
+  int32_t      numWait;
+
+  int64_t taskid;
+  int64_t scheduleTime;
+  int64_t launchTime;
+  int64_t finishTime;
+
+  struct STFSBgTask *prev;
+  struct STFSBgTask *next;
+};
+
 struct STFileOp {
   tsdb_fop_t optype;
   int32_t    fid;
@@ -87,6 +117,16 @@ struct STFileSet {
   int64_t      maxVerValid;
   STFileObj   *farr[TSDB_FTYPE_MAX];  // file array
   TSttLvlArray lvlArr[1];             // level array
+
+  // background task queue
+  int32_t     bgTaskNum;
+  STFSBgTask  bgTaskQueue[1];
+  STFSBgTask *bgTaskRunning;
+
+  // block commit variables
+  TdThreadCond canCommit;
+  int32_t      numWaitCommit;
+  bool         blockCommit;
 };
 
 struct STSnapRange {

source/dnode/vnode/src/tsdb/tsdbFile2.c

@@ -303,6 +303,7 @@ bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
   if (f1->did.id != f2->did.id) return false;
   if (f1->fid != f2->fid) return false;
   if (f1->cid != f2->cid) return false;
+  if (f1->s3flag != f2->s3flag) return false;
   return true;
 }
 

source/dnode/vnode/src/tsdb/tsdbFile2.h

@@ -58,6 +58,7 @@ int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2);
 struct STFile {
   tsdb_ftype_t type;
   SDiskID      did;  // disk id
+  int32_t      s3flag;
   int32_t      fid;  // file id
   int64_t      cid;  // commit id
   int64_t      size;