Merge branch '3.0' of https://github.com/taosdata/TDengine into mark/3.0

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.2.3.0.alpha")
+  SET(TD_VER_NUMBER "3.2.4.0.alpha")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

docs/en/08-client-libraries/07-python.mdx

@@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
 
 In addition to native connections, the client library also supports subscriptions via websockets.
 
-The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
+The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
 
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>
@@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll`
 
 ```python
 while True:
-    res = consumer.poll(1)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -902,16 +902,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket connection">
 
-The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data.
+The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. 
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
-    if err is not None:
-        raise err
+    
     for block in message:
         for row in block:
             print(row)

docs/en/12-taos-sql/06-select.md

@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
 
 select_list:
     select_expr [, select_expr] ...
@@ -87,12 +87,13 @@ Hints are a means of user control over query optimization for individual stateme
 
 The list of currently supported Hints is as follows:
 
-|    **Hint**   |    **Params**  |         **Comment**        |       **Scopt**            |
+|    **Hint**   |    **Params**  |         **Comment**        |       **Scope**            |
 | :-----------: | -------------- | -------------------------- | -----------------------------------|
 | BATCH_SCAN    | None           | Batch table scan           | JOIN statment for stable           |
 | NO_BATCH_SCAN | None           | Sequential table scan      | JOIN statment for stable           |
 | SORT_FOR_GROUP| None           | Use sort for partition, conflict with PARTITION_FIRST     | With normal column in partition by list |
 | PARTITION_FIRST| None          | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list |
+| PARA_TABLES_SORT| None         | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp  |
 
 For example:
 
@@ -100,6 +101,7 @@ For example:
 SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts;
 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
+SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
 ```
 
 ## Lists

docs/en/12-taos-sql/14-stream.md

@@ -41,16 +41,28 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
 `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
 
+`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.
+
+`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
+
 For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
 
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## Partitions of Stream

docs/en/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
 
 import Release from "/components/ReleaseV3";
 
+## 3.2.3.0
+
+<Release type="tdengine" version="3.2.3.0" />
+
 ## 3.2.2.0
 
 <Release type="tdengine" version="3.2.2.0" />

docs/zh/08-connector/30-python.mdx

@@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>
@@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo
 
 ```python
 while True:
-    res = consumer.poll(1)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -911,16 +911,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket 连接">
 
-Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。
+Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
-    if err is not None:
-        raise err
+    
     for block in message:
         for row in block:
             print(row)

docs/zh/12-taos-sql/06-select.md

@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
 
 select_list:
     select_expr [, select_expr] ...
@@ -93,13 +93,14 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适
 | NO_BATCH_SCAN | 无             | 采用顺序读表的方式         | 超级表 JOIN 语句             |
 | SORT_FOR_GROUP| 无             | 采用sort方式进行分组, 与PARTITION_FIRST冲突  | partition by 列表有普通列时  |
 | PARTITION_FIRST| 无             | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时  |
-
+| PARA_TABLES_SORT| 无             | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时  |
 举例： 
 
 ```sql
 SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts;
 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
+SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
 ```
 
 ## 列表

docs/zh/12-taos-sql/14-stream.md

@@ -49,10 +49,14 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
+EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 
@@ -61,6 +65,12 @@ window_clause: {
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## 流式计算的 partition

docs/zh/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
 
 import Release from "/components/ReleaseV3";
 
+## 3.2.3.0
+
+<Release type="tdengine" version="3.2.3.0" />
+
 ## 3.2.2.0
 
 <Release type="tdengine" version="3.2.2.0" />

include/common/tdatablock.h

@@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo {
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
+#define BLOCK_VERSION_1          1
+#define BLOCK_VERSION_2          2
+
 #define NBIT                     (3u)
 #define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
 #define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])
@@ -264,7 +267,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
 
 bool  alreadyAddGroupId(char* ctbName);
 bool  isAutoTableName(char* ctbName);
-void  buildCtbNameAddGruopId(char* ctbName, uint64_t groupId);
+void  buildCtbNameAddGroupId(char* ctbName, uint64_t groupId);
 char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
 int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);
 

include/common/tglobal.h

@@ -106,6 +106,9 @@ extern char     tsMonitorFqdn[];
 extern uint16_t tsMonitorPort;
 extern int32_t  tsMonitorMaxLogs;
 extern bool     tsMonitorComp;
+extern bool     tsMonitorLogProtocol;
+extern int32_t  tsMonitorIntervalForBasic;
+extern bool     tsMonitorForceV2;
 
 // audit
 extern bool     tsEnableAudit;
@@ -230,10 +233,10 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer);
 
 struct SConfig *taosGetCfg();
 
-void    taosSetAllDebugFlag(int32_t flag);
-void    taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal);
-void    taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
-int8_t  taosGranted(int8_t type);
+void   taosSetGlobalDebugFlag(int32_t flag);
+void   taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal);
+void   taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
+int8_t taosGranted(int8_t type);
 
 #ifdef __cplusplus
 }

include/common/tgrant.h

@@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant);
 
 // #ifndef GRANTS_CFG
 #ifdef TD_ENTERPRISE
-#define GRANTS_SCHEMA                                                                             \
-  static const SSysDbTableSchema grantsSchema[] = {                                               \
-      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},  \
-      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
-      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},         \
-      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},   \
-      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    \
+#define GRANTS_SCHEMA                                                                                              \
+  static const SSysDbTableSchema grantsSchema[] = {                                                                \
+      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},  \
+      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},         \
+      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},   \
+      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},    \
   }
 #else
-#define GRANTS_SCHEMA                                                                             \
-  static const SSysDbTableSchema grantsSchema[] = {                                               \
-      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},  \
-      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
-      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},         \
-      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},   \
-      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
-      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    \
+#define GRANTS_SCHEMA                                                                                              \
+  static const SSysDbTableSchema grantsSchema[] = {                                                                \
+      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},  \
+      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},         \
+      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},   \
+      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
+      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},    \
   }
 #endif
 // #define GRANT_CFG_ADD

include/common/tmsg.h

@@ -581,9 +581,9 @@ typedef struct {
   };
 } SSubmitRsp;
 
-int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
-int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void    tFreeSSubmitBlkRsp(void* param);
+// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
+// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
+// void    tFreeSSubmitBlkRsp(void* param);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON     ((int8_t)0x1)
@@ -885,8 +885,8 @@ typedef struct {
   int64_t maxStorage;
 } SCreateAcctReq, SAlterAcctReq;
 
-int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
-int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
 
 typedef struct {
   char    user[TSDB_USER_LEN];
@@ -1324,14 +1324,13 @@ typedef struct {
 } SDnodeListReq;
 
 int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
-int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
 
 typedef struct {
   int32_t useless;  // useless
 } SServerVerReq;
 
 int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
-int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
+// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
 
 typedef struct {
   char ver[TSDB_VERSION_LEN];
@@ -1582,6 +1581,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
 int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
 void    tFreeSStatusReq(SStatusReq* pReq);
 
+typedef struct {
+  int32_t     contLen;
+  char*       pCont;
+} SStatisReq;
+
+int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+void tFreeSStatisReq(SStatisReq *pReq);
+
 typedef struct {
   int32_t dnodeId;
   int64_t clusterId;
@@ -1610,7 +1618,6 @@ typedef struct {
   SEp     ep;
   char    active[TSDB_ACTIVE_KEY_LEN];
   char    connActive[TSDB_CONN_ACTIVE_KEY_LEN];
-  char    machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeInfo;
 
 typedef struct {
@@ -1630,14 +1637,14 @@ typedef struct {
 } SMTimerReq;
 
 int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
-int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
+// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
 
 typedef struct {
   int64_t tick;
 } SMStreamTickReq;
 
 int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
-int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
+// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
 
 typedef struct {
   int32_t  id;
@@ -1908,7 +1915,7 @@ typedef struct {
 } SShowVariablesReq;
 
 int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
-int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
+// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
 
 typedef struct {
   char name[TSDB_CONFIG_OPTION_LEN + 1];
@@ -1938,7 +1945,7 @@ typedef struct {
 } SShowReq;
 
 int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
-int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
+// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
 void    tFreeSShowReq(SShowReq* pReq);
 
 typedef struct {
@@ -1946,9 +1953,9 @@ typedef struct {
   STableMetaRsp tableMeta;
 } SShowRsp, SVShowTablesRsp;
 
-int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-void    tFreeSShowRsp(SShowRsp* pRsp);
+// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
@@ -2206,7 +2213,7 @@ typedef struct {
 } SForceBecomeFollowerReq;
 
 int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
-int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
+// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
 
 typedef struct {
   int32_t vgId;
@@ -2223,8 +2230,8 @@ typedef struct {
   char ckey[TSDB_PASSWORD_LEN];
 } SAuthReq, SAuthRsp;
 
-int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
-int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
 
 typedef struct {
   int32_t statusCode;
@@ -2757,8 +2764,8 @@ typedef struct {
 int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp);
 int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp);
 
-int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
-int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
 
 // TDMT_VND_DROP_TABLE =================
 typedef struct {
@@ -3213,8 +3220,8 @@ typedef struct {
   int32_t hasCheckPoint;
 } SMVStreamGatherInfoReq;
 
-int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
-int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
+// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
+// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
 
 typedef struct {
   int64_t leftForVer;
@@ -3439,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR
 
 typedef SMDropSmaReq SDropTagIndexReq;
 
-int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
+// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 
 typedef struct {
@@ -3491,7 +3498,7 @@ void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
 int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
 int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
 int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
-int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
+// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
 
 typedef struct {
   int32_t number;
@@ -3560,8 +3567,8 @@ typedef struct {
   int8_t igNotExists;
 } SMDropFullTextReq;
 
-int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
-int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
 
 typedef struct {
   char indexFName[TSDB_INDEX_FNAME_LEN];

include/common/tmsgdef.h

@@ -215,6 +215,7 @@
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)

include/common/trow.h

@@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
  */
 
 static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
-  pBuilder->rowType = TD_ROW_TP;  // default STpRow
+  pBuilder->rowType = pBuilder->rowType;
   pBuilder->sver = sver;
 }
 int32_t                  tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);
@@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow);
 bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
 bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal);
 
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow);
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType);
 bool    tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
 void    tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag);
 

include/dnode/vnode/tqCommon.h

@@ -26,6 +26,9 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
 int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
                                      bool isLeader, bool restored);
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);

include/libs/monitor/clientMonitor.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_CLIENT_MONITOR_H
+#define TDENGINE_CLIENT_MONITOR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taos_monitor.h"
+#include "thash.h"
+#include "query.h"
+
+typedef enum SQL_RESULT_CODE {
+  SQL_RESULT_SUCCESS = 0,
+  SQL_RESULT_FAILED = 1,
+  SQL_RESULT_CANCEL = 2,
+} SQL_RESULT_CODE;
+
+const char* resultStr(SQL_RESULT_CODE code);
+
+typedef struct {
+  char                       clusterKey[512];
+  SEpSet                     epSet;
+  void*                      pTransporter;
+  taos_collector_registry_t* registry;
+  taos_collector_t*          colector;
+  SHashObj*                  counters;
+} ClientMonitor;
+
+void            clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter);
+void            clusterMonitorClose(const char* clusterKey);
+taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
+                                     const char** label_keys);
+int             taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values);
+void            cluster_monitor_stop();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_CLIENT_MONITOR_H

include/libs/monitor/monitor.h

@@ -76,7 +76,8 @@ typedef struct {
 } SMonBasicInfo;
 
 typedef struct {
-  float        uptime;  // day
+  //float        uptime;  // day
+  int64_t      uptime;  // second
   int8_t       has_mnode;
   int8_t       has_qnode;
   int8_t       has_snode;
@@ -100,13 +101,15 @@ typedef struct {
   int32_t mnode_id;
   char    mnode_ep[TSDB_EP_LEN];
   char    role[MON_ROLE_LEN];
+  int32_t syncState;
 } SMonMnodeDesc;
 
 typedef struct {
   char    first_ep[TSDB_EP_LEN];
   int32_t first_ep_dnode_id;
   char    version[MON_VER_LEN];
-  float   master_uptime;     // day
+  //float   master_uptime;     // day
+  int64_t master_uptime;        //second
   int32_t monitor_interval;  // sec
   int32_t dbs_total;
   int32_t stbs_total;
@@ -125,6 +128,7 @@ typedef struct {
 typedef struct {
   int32_t dnode_id;
   char    vnode_role[MON_ROLE_LEN];
+  int32_t syncState;
 } SMonVnodeDesc;
 
 typedef struct {
@@ -149,7 +153,7 @@ typedef struct {
 } SMonStbInfo;
 
 typedef struct {
-  uint32_t expire_time;
+  int64_t  expire_time;
   int64_t  timeseries_used;
   int64_t  timeseries_total;
 } SMonGrantInfo;
@@ -221,7 +225,9 @@ void    monSetVmInfo(SMonVmInfo *pInfo);
 void    monSetQmInfo(SMonQmInfo *pInfo);
 void    monSetSmInfo(SMonSmInfo *pInfo);
 void    monSetBmInfo(SMonBmInfo *pInfo);
-void    monSendReport();
+void    monGenAndSendReport();
+void    monGenAndSendReportBasic();
+void    monSendContent(char *pCont);
 
 void tFreeSMonMmInfo(SMonMmInfo *pInfo);
 void tFreeSMonVmInfo(SMonVmInfo *pInfo);

include/libs/monitorfw/taos_alloc.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_alloc.h
+ * @brief memory management
+ */
+
+#ifndef TAOS_ALLOC_H
+#define TAOS_ALLOC_H
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is malloc.
+ */
+#define taos_malloc malloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is realloc.
+ */
+#define taos_realloc realloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is strdup.
+ */
+#define taos_strdup strdup
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is free.
+ */
+#define taos_free free
+
+#endif  // TAOS_ALLOC_H

include/libs/monitorfw/taos_collector.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_COLLECTOR_H
+#define TAOS_COLLECTOR_H
+
+#include "taos_map.h"
+#include "taos_metric.h"
+
+/**
+ * @file taos_collector.h
+ * @brief A Prometheus collector returns a collection of metrics
+ */
+
+/**
+ * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is
+ * registered.
+ */
+typedef struct taos_collector taos_collector_t;
+
+/**
+ * @brief The function responsible for preparing metric data and returning metrics for a given collector.
+ *
+ * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may
+ * set this function on your collector to do additional work before returning the contained metrics.
+ *
+ * @param self The target taos_collector_t*
+ * @return The taos_map_t* containing the collected metrics
+ */
+typedef taos_map_t *taos_collect_fn(taos_collector_t *self);
+
+/**
+ * @brief Create a collector
+ * @param name The name of the collector. The name MUST NOT be default or process.
+ * @return The constructed taos_collector_t*
+ */
+taos_collector_t *taos_collector_new(const char *name);
+
+/**
+ * @brief Destroy a collector. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy(taos_collector_t *self);
+
+/**
+ * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ */
+void taos_collector_free_generic(void *gen);
+
+/**
+ * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy_generic(void *gen);
+
+/**
+ * @brief Add a metric to a collector
+ * @param self The target taos_collector_t*
+ * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric);
+
+int taos_collector_remove_metric(taos_collector_t *self, const char* key);
+
+taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name);
+
+/**
+ * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them
+ *        for metric exposition.
+ * @param self The target taos_collector_t*
+ * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before
+ *           returning the collected metrics for exposition.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn);
+
+#endif  // TAOS_COLLECTOR_H

include/libs/monitorfw/taos_collector_registry.h

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_collector_registry.h
+ * @brief The collector registry registers collectors for metric exposition.
+ */
+
+#ifndef TAOS_REGISTRY_H
+#define TAOS_REGISTRY_H
+
+#include "taos_collector.h"
+#include "taos_metric.h"
+
+/**
+ * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format
+ */
+typedef struct taos_collector_registry taos_collector_registry_t;
+
+/**
+ * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT
+ * modify this value.
+ */
+extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT;
+
+/**
+ * @brief Initializes the default collector registry and enables metric collection on the executing process
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_default_init(void);
+
+/**
+ * @brief Constructs a taos_collector_registry_t*
+ * @param name The name of the collector registry. It MUST NOT be default.
+ * @return The constructed taos_collector_registry_t*
+ */
+taos_collector_registry_t *taos_collector_registry_new(const char *name);
+
+/**
+ * @brief Destroy a collector registry. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_destroy(taos_collector_registry_t *self);
+
+/**
+ * @brief Enable process metrics on the given collector registry
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY
+ *
+ * The metric to be registered MUST NOT already be registered with the given . If so, the program will
+ * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore,
+ * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this
+ * function. The metric will be added to the default registry's default collector.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return The registered taos_metric_t*
+ */
+taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer
+ * value on failure.
+ *
+ * See taos_collector_registry_must_register_metric.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_metric(taos_metric_t *metric);
+
+int taos_collector_registry_deregister_metric(const char *key);
+
+taos_metric_t *taos_collector_registry_get_metric(char* metric_name);
+
+/**
+ * @brief Register a collector with the given registry. Returns a non-zero integer value on failure.
+ * @param self The target taos_collector_registry_t*
+ * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector);
+
+/**
+ * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap
+ * memory growth.
+ *
+ * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/
+ *
+ * @param self The target taos_collector_registry_t*
+ * @return The string int he default metric exposition format.
+ */
+const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format);
+
+int taos_collector_registry_clear_batch(taos_collector_registry_t *self);
+
+const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str);
+
+/**
+ *@brief Validates that the given metric name complies with the specification:
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ *
+ * Returns a non-zero integer value on failure.
+ *
+ * @param self The target taos_collector_registry_t*
+ * @param metric_name The metric name to validate
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name);
+
+#endif  // TAOS_H

include/libs/monitorfw/taos_counter.h

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_COUNTER_H
+#define TAOS_COUNTER_H
+
+#include <stdlib.h>
+
+#include "taos_metric.h"
+
+/**
+ * @file taos_counter.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+
+/**
+ * @brief A prometheus counter.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+typedef taos_metric_t taos_counter_t;
+
+/**
+ * @brief Construct a taos_counter_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ *                        require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ *                   no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ *                   literal.
+ * @return The constructed taos_counter_t*
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" });
+ *
+ *     // An example without labels
+ *     taos_counter_new("foo", "foo is a counter without labels", 0, NULL);
+ */
+taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ *        returned on failure.
+ * @param self A taos_counter_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_counter_destroy(taos_counter_t *self);
+
+/**
+ * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure.
+ * @param self The target  taos_counter_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_inc(foo_counter, (const char**) { "bar", "bang" });
+ **
+ *     // An example without labels
+ *     taos_counter_inc(foo_counter, NULL);
+ */
+int taos_counter_inc(taos_counter_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure.
+ * @param self The target  taos_counter_t*
+ * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_counter_add(foo_counter, 22, NULL);
+ */
+int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values);
+
+#endif  // TAOS_COUNTER_H

include/libs/monitorfw/taos_gauge.h

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_gauge.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+
+#ifndef TAOS_GAUGE_H
+#define TAOS_GAUGE_H
+
+#include <stdlib.h>
+
+#include "taos_metric.h"
+
+/**
+ * @brief A prometheus gauge.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+typedef taos_metric_t taos_gauge_t;
+
+/**
+ * @brief Constructs a taos_gauge_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ *                        require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ *                   no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ *                   literal.
+ * @return The constructed taos_guage_t*
+ *
+ *     // An example with labels
+ *     taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" });
+ *
+ *     // An example without labels
+ *     taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL);
+ */
+taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ *        returned on failure.
+ * @param self The target taos_gauge_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_gauge_destroy(taos_gauge_t *self);
+
+/**
+ * @brief Increment the taos_gauge_t* by 1.
+ * @param self The target  taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_inc(foo_gauge, NULL);
+ */
+int taos_gauge_inc(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Decrement the taos_gauge_t* by 1.
+ * @param self The target  taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_dec(foo_gauge, NULL);
+ */
+int taos_gauge_dec(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_gauge_t*.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_add(foo_gauge, 22, NULL);
+ */
+int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_sub(foo_gauge, 22, NULL);
+ */
+int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Set the value for the taos_gauge_t*
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to which the taos_gauge_t* passed as self will be set
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_set(foo_gauge, 22, NULL);
+ */
+int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values);
+
+#endif  // TAOS_GAUGE_H

include/libs/monitorfw/taos_linked_list.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_LIST_H
+#define TAOS_LIST_H
+
+#include <stdlib.h>
+
+struct taos_linked_list;
+/**
+ * @brief Provides a generic linked list
+ */
+typedef struct taos_linked_list taos_linked_list_t;
+
+#endif  // TAOS_LIST_H

include/libs/monitorfw/taos_map.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_MAP_H
+#define TAOS_MAP_H
+
+struct taos_map;
+typedef struct taos_map taos_map_t;
+
+struct taos_map_node;
+typedef struct taos_map_node taos_map_node_t;
+
+#endif  // TAOS_MAP_H

include/libs/monitorfw/taos_metric.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_metric.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_METRIC_H
+#define TAOS_METRIC_H
+
+#include "taos_metric_sample.h"
+
+struct taos_metric;
+/**
+ * @brief A prometheus metric.
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model
+ */
+typedef struct taos_metric taos_metric_t;
+
+/**
+ * @brief Returns a taos_metric_sample_t*. The order of label_values is significant.
+ *
+ * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map
+ * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be
+ * preferrable in performance-sensitive situations.
+ *
+ * @param self The target taos_metric_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A taos_metric_sample_t*
+ */
+taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values);
+
+#endif  // TAOS_METRIC_H

include/libs/monitorfw/taos_metric_sample.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_metric_sample.h
+ * @brief Functions for interfacting with metric samples directly
+ */
+
+#ifndef TAOS_METRIC_SAMPLE_H
+#define TAOS_METRIC_SAMPLE_H
+
+struct taos_metric_sample;
+/**
+ * @brief Contains the specific metric and value given the name and label set
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ */
+typedef struct taos_metric_sample taos_metric_sample_t;
+
+/**
+ * @brief Add the r_value to the sample. The value must be greater than or equal to zero.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to add to taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_add(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Subtract the r_value from the sample.
+ *
+ * This operation MUST be called a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to subtract from the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Set the r_value of the sample.
+ *
+ * This operation MUST be called on a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double which will be set to the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_set(taos_metric_sample_t *self, double r_value);
+
+int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value);
+
+#endif  // TAOS_METRIC_SAMPLE_H

include/libs/monitorfw/taos_monitor.h

@@ -0,0 +1,132 @@
+/*
+Copyright 2019 DigitalOcean Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @file taos_monitor.h
+ * @brief Include taos_monitor.h to include the entire public API
+ * @mainpage Welcome to the documentation site for prometheus-client-c!
+ * @tableofcontents
+ * @section Introduction
+ *
+ * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language.
+ * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics
+ * over HTTP.
+ *
+ * @section Creating-and-Registering-Metrics Creating and Registering Metrics
+ *
+ * prometheus-client-c supports the following metric types:
+ *
+ * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter)
+ * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge)
+ * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)
+ *
+ * To get started using one of the metric types, declare the metric at file scope. For example:
+ *
+ * @code{.c}
+ *
+ * #incldue "taos_monitor.h"
+ *
+ * taos_counter_t *my_counter;
+ *
+ * @endcode
+ *
+ * Next, create a metric initialization function. You can create the metric and register it with the default metric
+ * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning
+ * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later...
+ *
+ * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric
+ * constructor into the taos_collector_registry_must_register_metric function. For example:
+ *
+ * @code{.c}
+ *
+ * void foo_metric_init(void) {
+ *   my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL));
+ * }
+ *
+ * @endcode
+ *
+ * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third
+ * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass
+ * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we
+ * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics
+ * for the file we just created to the default prometheus metric collector registery called
+ * TAOS_COLLECTOR_REGISTRY_DEFAULT
+ *
+ * @section Updating-Metric-Sample-Values Updating Metric Sample Values
+ *
+ * Now that we have a metric configured for creation and registration, we can update our metric within any of the
+ * functions of the file in which it was declared. For example:
+ *
+ * @code{.c}
+ *
+ * void my_lib_do_something(void) {
+ *   printf("I did a really important thing!\n");
+ *   taos_counter_inc(my_counter, NULL);
+ * }
+ * @endcode
+ *
+ * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass
+ * NULL as the second argument.
+ *
+ * @section Program-Initialization Program Initialization
+ *
+ * At the start of the program's main function you need to do two things:
+ *
+ * * Initialize the default metric collector registry:
+ *
+ * @code{.c}
+ *
+ * taos_collector_registry_default_init();
+ *
+ * @endcode
+ *
+ * * For each file containing prometheus metrics, call its corresponding metric initialization function
+ *
+ * @code{.c}
+ *
+ * foo_metric_init()
+ *
+ * @endcode
+ *
+ * After initialization is complete, you may proceed to do work and update your metrics.
+ *
+ * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP
+ *
+ * @todo Describe how to use libpromhttp to expose metrics over HTTP
+ *
+ * @section Where-To-Go-From-Here Where to Go From Here?
+ *
+ * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html)
+ * tab in this documentation site for more information about the public API available to you. Also, you can take a look
+ * at the examples directory at the
+ * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration.
+ */
+
+#ifndef TAOS_INCLUDED
+#define TAOS_INCLUDED
+
+#include "taos_alloc.h"
+#include "taos_collector.h"
+#include "taos_collector_registry.h"
+#include "taos_counter.h"
+#include "taos_linked_list.h"
+#include "taos_map.h"
+#include "taos_metric.h"
+#include "taos_metric_sample.h"
+#include "taos_monitor_util.h"
+#include "taos_gauge.h"
+
+#endif //  TAOS_INCLUDED

include/libs/monitorfw/taos_monitor_util.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_monitor_util.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_MONITOR_UTIL_H
+#define TAOS_MONITOR_UTIL_H
+
+#include "taos_metric.h"
+#include "tjson.h"
+
+void taos_monitor_split_str(char** arr, char* str, const char* del);
+void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf);
+char* taos_monitor_get_metric_name(taos_metric_t* metric);
+
+#endif  // TAOS_MONITOR_UTIL_H

include/libs/stream/tstream.h

@@ -56,7 +56,6 @@ extern "C" {
 #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4)
 #define STREAM_EXEC_T_STOP_ALL_TASKS    (-5)
 #define STREAM_EXEC_T_RESUME_TASK       (-6)
-#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7)
 
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
@@ -783,11 +782,14 @@ bool    streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
-int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
 
 int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
-int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
-void    streamTaskRestoreStatus(SStreamTask* pTask);
+
+typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
+int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
+int32_t streamTaskRestoreStatus(SStreamTask* pTask);
 
 int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
                            SRpcHandleInfo* pRpcInfo, int32_t taskId);

include/os/osAtomic.h

@@ -48,11 +48,13 @@ void    atomic_store_8(int8_t volatile *ptr, int8_t val);
 void    atomic_store_16(int16_t volatile *ptr, int16_t val);
 void    atomic_store_32(int32_t volatile *ptr, int32_t val);
 void    atomic_store_64(int64_t volatile *ptr, int64_t val);
+double  atomic_store_double(double volatile *ptr, double val);
 void    atomic_store_ptr(void *ptr, void *val);
 int8_t  atomic_exchange_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
+double  atomic_exchange_double(double volatile *ptr, int64_t val);
 void   *atomic_exchange_ptr(void *ptr, void *val);
 int8_t  atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
 int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
@@ -68,6 +70,7 @@ int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
+double  atomic_fetch_add_double(double volatile *ptr, double val);
 void   *atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
@@ -78,6 +81,7 @@ int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
+double atomic_fetch_sub_double(double volatile *ptr, double val);
 void   *atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t  atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);

include/util/taoserror.h

@@ -433,7 +433,7 @@ int32_t* taosGetErrno();
 
 //mnode-compact
 #define TSDB_CODE_MND_INVALID_COMPACT_ID        TAOS_DEF_ERROR_CODE(0, 0x04B1)
-
+#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST  TAOS_DEF_ERROR_CODE(0, 0x04B2)
 
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
@@ -757,6 +757,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_COL_QUERY_MISMATCH        TAOS_DEF_ERROR_CODE(0, 0x266D)
 #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE  TAOS_DEF_ERROR_CODE(0, 0x266E)
 #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS         TAOS_DEF_ERROR_CODE(0, 0x266F)
+#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT  TAOS_DEF_ERROR_CODE(0, 0x2670)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner

include/util/tbase58.h

@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#define TBASE_MAX_ILEN 4096
+#define TBASE_MAX_OLEN 5653
+
 uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen);
 char    *base58_encode(const uint8_t *value, int32_t vlen);
 

include/util/tdef.h

@@ -187,6 +187,8 @@ typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
+#define TSDB_INT32_ID_LEN 11
+
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24
@@ -196,6 +198,8 @@ typedef enum ELogicConditionType {
 // ACCOUNT is a 32 bit positive integer
 // this is the length of its string representation, including the terminator zero
 #define TSDB_ACCT_ID_LEN 11
+#define TSDB_NODE_ID_LEN 11
+#define TSDB_VGROUP_ID_LEN 11
 
 #define TSDB_MAX_COLUMNS 4096
 #define TSDB_MIN_COLUMNS 2  // PRIMARY COLUMN(timestamp) + other columns
@@ -547,6 +551,10 @@ enum {
 #define VNODE_TIMEOUT_SEC 60
 #define MNODE_TIMEOUT_SEC 60
 
+#define MONITOR_TABLENAME_LEN     200
+#define MONITOR_TAG_NAME_LEN      100
+#define MONITOR_TAG_VALUE_LEN     300
+#define MONITOR_METRIC_NAME_LEN   100
 #ifdef __cplusplus
 }
 #endif

include/util/tjson.h

@@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
 SJson*      tjsonParse(const char* pStr);
 bool        tjsonValidateJson(const char* pJson);
 const char* tjsonGetError();
+void        tjsonDeleteItemFromObject(const SJson* pJson, const char* pName);
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -120,6 +120,7 @@ void    taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
 #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
 #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
 #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
+#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }}
 
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }

include/util/tqueue.h

@@ -72,40 +72,6 @@ struct STaosQnode {
   char        item[];
 };
 
-struct STaosQueue {
-  STaosQnode   *head;
-  STaosQnode   *tail;
-  STaosQueue   *next;     // for queue set
-  STaosQset    *qset;     // for queue set
-  void         *ahandle;  // for queue set
-  FItem         itemFp;
-  FItems        itemsFp;
-  TdThreadMutex mutex;
-  int64_t       memOfItems;
-  int32_t       numOfItems;
-  int64_t       threadId;
-  int64_t       memLimit;
-  int64_t       itemLimit;
-};
-
-struct STaosQset {
-  STaosQueue   *head;
-  STaosQueue   *current;
-  TdThreadMutex mutex;
-  tsem_t        sem;
-  int32_t       numOfQueues;
-  int32_t       numOfItems;
-};
-
-struct STaosQall {
-  STaosQnode *current;
-  STaosQnode *start;
-  int32_t     numOfItems;
-  int64_t     memOfItems;
-  int32_t     unAccessedNumOfItems;
-  int64_t     unAccessMemOfItems;
-};
-
 STaosQueue *taosOpenQueue();
 void        taosCloseQueue(STaosQueue *queue);
 void        taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
@@ -140,6 +106,8 @@ int32_t    taosGetQueueNumber(STaosQset *qset);
 int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo);
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
+void    taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId);
+int64_t taosQueueGetThreadId(STaosQueue *pQueue);
 
 #ifdef __cplusplus
 }

include/util/tscalablebf.h

@@ -26,6 +26,8 @@ typedef struct SScalableBf {
   SArray  *bfArray;  // array of bloom filters
   uint32_t growth;
   uint64_t numBits;
+  uint32_t maxBloomFilters;
+  int8_t   status;
   _hash_fn_t hashFn1;
   _hash_fn_t hashFn2;
 } SScalableBf;

source/client/CMakeLists.txt

@@ -20,7 +20,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+    PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
 )
 
 if(TD_DARWIN_ARM64)
@@ -61,7 +61,7 @@ target_include_directories(
 target_link_libraries(
     taos_static
     INTERFACE api
-    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+    PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -355,6 +355,7 @@ SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
 int32_t      removeRequest(int64_t rid);
 void         doDestroyRequest(void* p);
+int64_t      removeFromMostPrevReq(SRequestObj* pRequest);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
@@ -397,6 +398,8 @@ void       hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr);
 void       destroyAllRequests(SHashObj* pRequests);
 void       stopAllRequests(SHashObj* pRequests);
 
+SAppInstInfo* getAppInstInfo(const char* clusterKey);
+
 // conn level
 int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey);
@@ -433,6 +436,21 @@ void    freeQueryParam(SSyncQueryParam* param);
 int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
 #endif
 
+void clientSlowQueryMonitorInit(const char* clusterKey);
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost);
+
+void clientSQLReqMonitorInit(const char* clusterKey);
+
+enum {
+  MONITORSQLTYPESELECT = 0,
+  MONITORSQLTYPEINSERT = 1,
+  MONITORSQLTYPEDELETE = 2
+};
+
+void sqlReqLog(int64_t rid,  bool killed, int32_t code, int8_t type);
+
+void clientMonitorClose(const char* clusterKey);
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -18,6 +18,7 @@
 #include "clientLog.h"
 #include "functionMgt.h"
 #include "os.h"
+#include "osSleep.h"
 #include "query.h"
 #include "qworker.h"
 #include "scheduler.h"
@@ -105,7 +106,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
 
       atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
       reqType = SLOW_LOG_TYPE_QUERY;
-    }
+    } 
+  }
+
+  if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
+  } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT);
+  } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE);
   }
 
   if (duration >= (tsSlowLogThreshold * 1000000UL)) {
@@ -114,6 +123,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
       taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
                        taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
                        pRequest->sqlstr);
+      SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
     }
   }
 
@@ -224,6 +234,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) {
 
   taosThreadMutexLock(&appInfo.mutex);
 
+  clientMonitorClose(pAppInfo->instKey);
   hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
   taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
 
@@ -374,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
 
 int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
 
+/// return the most previous req ref id
+int64_t removeFromMostPrevReq(SRequestObj* pRequest) {
+  int64_t mostPrevReqRefId = pRequest->self;
+  SRequestObj* pTmp = pRequest;
+  while (pTmp->relation.prevRefId) {
+    pTmp = acquireRequest(pTmp->relation.prevRefId);
+    if (pTmp) {
+      mostPrevReqRefId = pTmp->self;
+      releaseRequest(mostPrevReqRefId);
+    } else {
+      break;
+    }
+  }
+  removeRequest(mostPrevReqRefId);
+  return mostPrevReqRefId;
+}
+
+void destroyNextReq(int64_t nextRefId) {
+  if (nextRefId) {
+    SRequestObj* pObj = acquireRequest(nextRefId);
+    if (pObj) {
+      releaseRequest(nextRefId);
+      releaseRequest(nextRefId);
+    }
+  }
+}
+
 void destroySubRequests(SRequestObj *pRequest) {
   int32_t      reqIdx = -1;
   SRequestObj *pReqList[16] = {NULL};
@@ -424,7 +462,7 @@ void doDestroyRequest(void *p) {
   uint64_t reqId = pRequest->requestId;
   tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 
-  destroySubRequests(pRequest);
+  int64_t nextReqRefId = pRequest->relation.nextRefId;
 
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
 
@@ -460,6 +498,7 @@ void doDestroyRequest(void *p) {
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFree(pRequest);
   tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);
+  destroyNextReq(nextReqRefId);
 }
 
 void destroyRequest(SRequestObj *pRequest) {
@@ -468,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) {
   }
 
   taos_stop_query(pRequest);
-  removeRequest(pRequest->self);
+  removeFromMostPrevReq(pRequest);
 }
 
 void taosStopQueryImpl(SRequestObj *pRequest) {

source/client/src/clientImpl.c

@@ -15,6 +15,7 @@
 
 #include "cJSON.h"
 #include "clientInt.h"
+#include "clientMonitor.h"
 #include "clientLog.h"
 #include "command.h"
 #include "scheduler.h"
@@ -157,6 +158,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
     tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port);
 
     pInst = &p;
+
+    clientSlowQueryMonitorInit(p->instKey);
+    clientSQLReqMonitorInit(p->instKey);
   } else {
     ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
     // reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
@@ -166,9 +170,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
   taosThreadMutexUnlock(&appInfo.mutex);
 
   taosMemoryFreeClear(key);
+
   return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
 }
 
+SAppInstInfo* getAppInstInfo(const char* clusterKey) {
+  SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
+  if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
+    return *ppAppInstInfo;
+  } else {
+    return NULL;
+  }
+}
+
 void freeQueryParam(SSyncQueryParam* param) {
   if (param == NULL) return;
   tsem_destroy(&param->sem);
@@ -1852,7 +1866,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
 
   char* pStart = p + len;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
 
     if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
       int32_t* offset = (int32_t*)pStart;
@@ -1951,8 +1965,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
   char* pStart = p;
   char* pStart1 = p1;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
-    int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i];
     if (ASSERT(colLen < dataLen)) {
       tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen);
       return TSDB_CODE_TSC_INTERNAL_ERROR;
@@ -2011,7 +2025,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       }
       colLen1 = len;
       totalLen += colLen1;
-      colLength1[i] = (blockVersion == 1) ? htonl(len) : len;
+      colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len;
     } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       len = numOfRows * sizeof(int32_t);
       memcpy(pStart1, pStart, len);
@@ -2100,7 +2114,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
 
   char* pStart = p;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(blockVersion == 1){
+    if(blockVersion == BLOCK_VERSION_1){
       colLength[i] = htonl(colLength[i]);
     }
     if (colLength[i] >= dataLen) {

source/client/src/clientMain.c

@@ -279,6 +279,7 @@ void taos_close_internal(void *taos) {
 
   STscObj *pTscObj = (STscObj *)taos;
   tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
+  // clientMonitorClose(pTscObj->pAppInfo->instKey);
 
   taosRemoveRef(clientConnRefPool, pTscObj->id);
 }
@@ -1253,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 }
 
 void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
-  int32_t      reqIdx = 0;
-  SRequestObj *pReqList[16] = {NULL};
-  SRequestObj *pUserReq = NULL;
-  pReqList[0] = pRequest;
-  uint64_t     tmpRefId = 0;
-  SRequestObj *pTmp = pRequest;
-  while (pTmp->relation.prevRefId) {
-    tmpRefId = pTmp->relation.prevRefId;
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      pReqList[++reqIdx] = pTmp;
-      releaseRequest(tmpRefId);
-    } else {
-      tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
+  tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
+  SRequestObj* pUserReq = pRequest;
+  acquireRequest(pRequest->self);
+  while (pUserReq) {
+    if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
       break;
-    }
-  }
-
-  tmpRefId = pRequest->relation.nextRefId;
-  while (tmpRefId) {
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      tmpRefId = pTmp->relation.nextRefId;
-      removeRequest(pTmp->self);
-      releaseRequest(pTmp->self);
     } else {
-      tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
-      break;
+      int64_t nextRefId = pUserReq->relation.nextRefId;
+      releaseRequest(pUserReq->self);
+      if (nextRefId) {
+        pUserReq = acquireRequest(nextRefId);
+      }
     }
   }
-
-  for (int32_t i = reqIdx; i >= 0; i--) {
-    destroyCtxInRequest(pReqList[i]);
-    if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) {
-      pUserReq = pReqList[i];
-    } else {
-      removeRequest(pReqList[i]->self);
-    }
-  }
-
+  bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0;
   if (pUserReq) {
+    destroyCtxInRequest(pUserReq);
     pUserReq->prevCode = code;
     memset(&pUserReq->relation, 0, sizeof(pUserReq->relation));
   } else {
-    tscError("user req is missing");
+    tscError("User req is missing");
+    removeFromMostPrevReq(pRequest);
     return;
   }
-
+  if (hasSubRequest)
+    removeFromMostPrevReq(pRequest);
+  else
+    releaseRequest(pUserReq->self);
   doAsyncQuery(pUserReq, true);
 }
 

source/client/src/clientSqlMonitor.c

@@ -0,0 +1,79 @@
+ /*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* selectMonitorName = "taos_sql_req:count";
+const char* selectMonitorHelp = "count for select sql";
+const int   selectMonitorLabelCount = 4;
+const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"};
+
+static const char* defaultClusterID = "";
+
+void clientSQLReqMonitorInit(const char* clusterKey) {
+  if (!tsEnableMonitor) return;
+  SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+  SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+  clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+  createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels);
+}
+
+void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) {
+  const char* typeStr;
+  switch (type) {
+    case MONITORSQLTYPEDELETE:
+      typeStr = "delete";
+      break;
+    case MONITORSQLTYPEINSERT:
+      typeStr = "insert";
+      break;
+    default:
+      typeStr = "select";
+      break;
+  }
+  const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)};
+  taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues);
+}
+
+void sqlReqLog(int64_t rid,  bool killed, int32_t code, int8_t type) {
+  if (!tsEnableMonitor) return;
+  SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+  if (TSDB_CODE_SUCCESS != code) {
+    result = SQL_RESULT_FAILED;
+  }
+  // to do Distinguish active Kill events
+  // else if (killed) {
+  //   result = SQL_RESULT_CANCEL;
+  // }
+
+  STscObj* pTscObj = acquireTscObj(rid);
+  if (pTscObj != NULL) {
+    if (pTscObj->pAppInfo == NULL) {
+      tscLog("sqlReqLog, not found pAppInfo");
+    } else {
+      clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type);
+    }
+    releaseTscObj(rid);
+  } else {
+    tscLog("sqlReqLog, not found rid");
+  }
+}
+
+void clientMonitorClose(const char* clusterKey) {
+  tscLog("clientMonitorClose, key:%s", clusterKey);
+  clusterMonitorClose(clusterKey);
+}

source/client/src/clientTmq.c

@@ -876,12 +876,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
   STaosQall* qall = taosAllocateQall();
   taosReadAllQitems(pTmq->delayedTask, qall);
 
-  if (qall->numOfItems == 0) {
+  int32_t numOfItems = taosQallItemSize(qall);
+  if (numOfItems == 0) {
     taosFreeQall(qall);
     return TSDB_CODE_SUCCESS;
   }
 
-  tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems);
+  tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems);
   int8_t* pTaskType = NULL;
   taosGetQitem(qall, (void**)&pTaskType);
 
@@ -1009,19 +1010,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
   }
   taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-  int32_t     rsp;
-  int32_t     retryCnt = 0;
   tmq_list_t* lst = tmq_list_new();
-  while (1) {
-    rsp = tmq_subscribe(tmq, lst);
-    if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-      break;
-    } else {
-      retryCnt++;
-      taosMsleep(500);
-    }
-  }
-
+  int32_t rsp = tmq_subscribe(tmq, lst);
   tmq_list_destroy(lst);
   return rsp;
 }
@@ -1271,10 +1261,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   }
 
   int32_t retryCnt = 0;
-  while (syncAskEp(tmq) != 0) {
-    if (retryCnt++ > MAX_RETRY_COUNT) {
+  while ((code = syncAskEp(tmq)) != 0) {
+    if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
       tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
-      code = TSDB_CODE_MND_CONSUMER_NOT_READY;
       goto FAIL;
     }
 
@@ -1611,17 +1600,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
-SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
-  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
-  pRspObj->resType = RES_TYPE__TMQ;
 
+void changeByteEndian(char* pData){
+  char* p = pData;
+
+  // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
+  // version:
+  int32_t blockVersion = *(int32_t*)p;
+  ASSERT(blockVersion == BLOCK_VERSION_1);
+  *(int32_t*)p = BLOCK_VERSION_2;
+
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  int32_t cols = *(int32_t*)p;
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(uint64_t);
+  // check fields
+  p += cols * (sizeof(int8_t) + sizeof(int32_t));
+
+  int32_t* colLength = (int32_t*)p;
+  for (int32_t i = 0; i < cols; ++i) {
+    colLength[i] = htonl(colLength[i]);
+  }
+}
+
+static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
   (*numOfRows) = 0;
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
 
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
-  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
@@ -1633,41 +1644,44 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
   }
   // extract the rows in this data packet
   for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t                  rows = htobe64(pRetrieve->numOfRows);
+    void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i);
+    void* rawData = NULL;
+    int64_t rows = 0;
+    // deal with compatibility
+    if(*(int64_t*)pRetrieve == 0){
+      rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
+    }else if(*(int64_t*)pRetrieve == 1){
+      rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
+    }
+
     pVg->numOfRows += rows;
     (*numOfRows) += rows;
-
-    if (needTransformSchema) {  // withSchema is false if subscribe subquery, true if subscribe db or stable
-      SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
-      if (schema) {
+    changeByteEndian(rawData);
+    if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
+      SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
+      if(schema){
         taosArrayPush(pRspObj->rsp.blockSchema, &schema);
       }
     }
   }
+}
 
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
+  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
+  pRspObj->resType = RES_TYPE__TMQ;
+  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj);
   return pRspObj;
 }
 
 SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
   SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
   pRspObj->resType = RES_TYPE__TMQ_METADATA;
-  tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
-  tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
-  pRspObj->vgId = pWrapper->vgHandle->vgId;
-  pRspObj->resIter = -1;
   memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
 
-  pRspObj->resInfo.totalRows = 0;
-  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-
-  // extract the rows in this data packet
-  for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t                  rows = htobe64(pRetrieve->numOfRows);
-    pVg->numOfRows += rows;
-    (*numOfRows) += rows;
-  }
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj);
   return pRspObj;
 }
 
@@ -1814,7 +1828,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal
 }
 
 static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
-  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
+  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall));
 
   while (1) {
     SMqRspWrapper* pRspWrapper = NULL;
@@ -2122,26 +2136,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     // if auto commit is set, commit before close consumer. Otherwise, do nothing.
     if (tmq->autoCommit) {
-      int32_t rsp = tmq_commit_sync(tmq, NULL);
-      if (rsp != 0) {
-        return rsp;
+      int32_t code = tmq_commit_sync(tmq, NULL);
+      if (code != 0) {
+        return code;
       }
     }
     taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-    int32_t     retryCnt = 0;
     tmq_list_t* lst = tmq_list_new();
-    while (1) {
-      int32_t rsp = tmq_subscribe(tmq, lst);
-      if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-        break;
-      } else {
-        retryCnt++;
-        taosMsleep(500);
-      }
-    }
-
+    int32_t code = tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
+    if (code != 0) {
+      return code;
+    }
   } else {
     tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
   }

source/client/src/slowQueryMonitor.c

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* slowQueryName = "taos_slow_sql:count";
+const char* slowQueryHelp = "slow query log when cost over than config duration";
+const int   slowQueryLabelCount = 4;
+const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"};
+static const char* defaultClusterID = "";
+
+const int64_t usInSeconds = 1000 * 1000;
+const int64_t msInMinutes = 60 * 1000;
+
+static const char* getSlowQueryLableCostDesc(int64_t cost) {
+  if (cost >= 1000 * usInSeconds) {
+    return "1000s-";
+  } else if (cost >= 100 * usInSeconds) {
+    return "100-1000s";
+  } else if (cost >= 10 * usInSeconds) {
+    return "10-100s";
+  } else if (cost >= 3 * usInSeconds) {
+    return "3-10s";
+  }
+  return "0-3s";
+}
+
+void clientSlowQueryMonitorInit(const char* clusterKey) {
+  if (!tsEnableMonitor) return;
+  SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+  SEpSet        epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+  clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+  createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels);
+}
+
+void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) {
+  const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)};
+  taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues);
+}
+
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
+  if (!tsEnableMonitor) return;
+  SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+  if (TSDB_CODE_SUCCESS != code) {
+    result = SQL_RESULT_FAILED;
+  }
+  // to do Distinguish active Kill events
+  // else if (killed) {
+  //   result = SQL_RESULT_CANCEL;
+  // }
+
+  STscObj* pTscObj = acquireTscObj(rid);
+  if (pTscObj != NULL) {
+    if(pTscObj->pAppInfo == NULL) {
+      tscLog("SlowQueryLog, not found pAppInfo");
+    } else {
+      clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost);
+    }
+    releaseTscObj(rid);
+  } else {
+    tscLog("SlowQueryLog, not found rid");
+  }
+}

source/client/test/CMakeLists.txt

@@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES(
         PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
 )
 
+ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
+TARGET_LINK_LIBRARIES(
+        clientMonitorTest
+        PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
+)
+
 TARGET_INCLUDE_DIRECTORIES(
         clientTest
         PUBLIC "${TD_SOURCE_DIR}/include/client/"
@@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES(
         PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
 )
 
+TARGET_INCLUDE_DIRECTORIES(
+        clientMonitorTest
+        PUBLIC "${TD_SOURCE_DIR}/include/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
+)
+
 add_test(
         NAME smlTest
         COMMAND smlTest
 )
+
+# add_test(
+#         NAME clientMonitorTest
+#         COMMAND clientMonitorTest
+# )

source/client/test/clientMonitorTests.cpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "taoserror.h"
+#include "tglobal.h"
+#include "thash.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "executor.h"
+#include "taos.h"
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(clientMonitorTest, monitorTest) {
+  const char* cluster1 = "cluster1";
+  const char* cluster2 = "cluster2";
+  SEpSet epSet;
+  clusterMonitorInit(cluster1, epSet, NULL);
+  const char* counterName1 = "slow_query";
+  const char* counterName2 = "select_count";
+  const char* help1 = "test for slowQuery";
+  const char* help2 = "test for selectSQL";
+  const char* lables[] = {"lable1"};
+  taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables);
+  ASSERT_TRUE(c1 != NULL);
+  taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables);
+  ASSERT_TRUE(c2 != NULL);
+  ASSERT_TRUE(c1 != c2);
+  taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+  ASSERT_TRUE(c21 == NULL);
+  clusterMonitorInit(cluster2, epSet, NULL);
+  c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+  ASSERT_TRUE(c21 != NULL);
+  int i = 0;
+  while (i < 12) {
+      taosMsleep(10);
+      ++i;
+  }
+  clusterMonitorClose(cluster1);
+  clusterMonitorClose(cluster2);
+}
+
+TEST(clientMonitorTest, sendTest) {
+  TAOS*       taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
+  ASSERT_TRUE(taos != NULL);
+  printf("connect taosd sucessfully.\n");
+
+  int64_t rid = *(int64_t *)taos;
+  SlowQueryLog(rid, false, -1, 1000);
+  int i = 0;
+  while (i < 20) {
+    SlowQueryLog(rid, false, 0, i * 1000);
+    taosMsleep(10);
+    ++i;
+  }
+
+  taos_close(taos);
+}

source/common/src/systable.c

@@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = {
 };
 
 static const SSysDbTableSchema userUsersSchema[] = {
-    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
-    {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
-    {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
-    {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+    {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+    {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
+    {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 GRANTS_SCHEMA;
@@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = {
 };
 
 static const SSysDbTableSchema variablesSchema[] = {
-    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = {
 static const SSysDbTableSchema vnodesSchema[] = {
     {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true},
     {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
@@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
 };
 
 static const SSysDbTableSchema userUserPrivilegesSchema[] = {
-    {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema userViewsSchema[] = {
@@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
 };
 
 static const SSysDbTableSchema useGrantsFullSchema[] = {
-    {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema useGrantsLogsSchema[] = {
-    {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema useMachinesSchema[] = {
-    {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysTableMeta infosMeta[] = {
@@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
     {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
     {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
-    {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false},
-    {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false},
-    {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false},
+    {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true},
+    {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true},
+    {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tdatablock.c

@@ -452,20 +452,21 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    int32_t newLen = pSource->varmeta.length;
     memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
-    if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
-      char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
+    if (pColumnInfoData->varmeta.allocLen < newLen) {
+      char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen);
       if (tmp == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
       }
 
       pColumnInfoData->pData = tmp;
-      pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
+      pColumnInfoData->varmeta.allocLen = newLen;
     }
 
-    pColumnInfoData->varmeta.length = pSource->varmeta.length;
+    pColumnInfoData->varmeta.length = newLen;
     if (pColumnInfoData->pData != NULL && pSource->pData != NULL) {
-      memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
+      memcpy(pColumnInfoData->pData, pSource->pData, newLen);
     }
   } else {
     memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
@@ -1687,7 +1688,29 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 }
 
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
+  if (n >= total || n == 0) return;
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+    if (pColInfoData->varmeta.length != 0) {
+      int32_t newLen = pColInfoData->varmeta.offset[n];
+      if (-1 == newLen) {
+        for (int i = n - 1; i >= 0; --i) {
+          newLen = pColInfoData->varmeta.offset[i];
+          if (newLen != -1) {
+            if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+              newLen += getJsonValueLen(pColInfoData->pData + newLen);
+            } else {
+              newLen += varDataTLen(pColInfoData->pData + newLen);
+            }
+            break;
+          }
+        }
+      }
+      if (newLen <= -1) {
+        uFatal("colDataKeepFirstNRows: newLen:%d  old:%d", newLen, pColInfoData->varmeta.length);
+      } else {
+        pColInfoData->varmeta.length = newLen;
+      }
+    }
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   }
@@ -2118,28 +2141,28 @@ _end:
   return TSDB_CODE_SUCCESS;
 }
 
-void  buildCtbNameAddGruopId(char* ctbName, uint64_t groupId){
+void  buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){
   char tmp[TSDB_TABLE_NAME_LEN] = {0};
   snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
   ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0;  // put groupId to the end
   strcat(ctbName, tmp);
 }
 
-bool  isAutoTableName(char* ctbName){
-  return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_');
-}
+// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.
+// the total length is fixed to be 34 bytes.
+bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); }
 
-bool  alreadyAddGroupId(char* ctbName){
+bool alreadyAddGroupId(char* ctbName) {
   size_t len = strlen(ctbName);
   size_t _location = len - 1;
-  while(_location > 0){
-    if(ctbName[_location] < '0' || ctbName[_location] > '9'){
+  while (_location > 0) {
+    if (ctbName[_location] < '0' || ctbName[_location] > '9') {
       break;
     }
     _location--;
   }
 
-  return ctbName[_location] == '_' &&  len - 1 - _location > 15;  //15 means the min length of groupid
+  return ctbName[_location] == '_' && len - 1 - _location >= 15;  // 15 means the min length of groupid
 }
 
 char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
@@ -2196,7 +2219,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
 
   // todo extract method
   int32_t* version = (int32_t*)data;
-  *version = 2;
+  *version = BLOCK_VERSION_1;
   data += sizeof(int32_t);
 
   int32_t* actualLen = (int32_t*)data;
@@ -2277,7 +2300,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
       data += colSizes[col];
     }
 
-//    colSizes[col] = htonl(colSizes[col]);
+    colSizes[col] = htonl(colSizes[col]);
     //    uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
     //    htonl(colSizes[col]), colSizes[col]);
   }
@@ -2342,9 +2365,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
   pStart += sizeof(int32_t) * numOfCols;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(version == 1){
-      colLen[i] = htonl(colLen[i]);
-    }
+    colLen[i] = htonl(colLen[i]);
     ASSERT(colLen[i] >= 0);
 
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

source/common/src/tglobal.c

@@ -94,6 +94,9 @@ char     tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
 uint16_t tsMonitorPort = 6043;
 int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
+bool     tsMonitorLogProtocol = false;
+int32_t  tsMonitorIntervalForBasic = 30;
+bool     tsMonitorForceV2 = true;
 
 // audit
 bool    tsEnableAudit = true;
@@ -535,6 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   return 0;
 }
 
@@ -693,6 +698,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+    return -1;
+  if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -1081,6 +1090,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64;
   tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32;
   tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32;
+  tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
+  tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
   if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
     return -1;
   }
@@ -1155,6 +1166,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
   tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
   tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
+  tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval;
+  tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32;
+  tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32;
 
   tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
   tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
@@ -1244,6 +1258,8 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; }
 int32_t taosSetReleaseCfg(SConfig *pCfg);
 #endif
 
+static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag);
+
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
                       const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
   if (tsCfg == NULL) osDefaultInit();
@@ -1288,7 +1304,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
     taosSetServerLogCfg(pCfg);
   }
 
-  taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
+  taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32);
 
   if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1366,7 +1382,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
   taosSetSystemCfg(tsCfg);
   if (taosSetFileHandlesLimit() != 0) return -1;
 
-  taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32);
+  taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
 
   cfgDumpCfg(tsCfg, tsc, false);
 
@@ -1459,7 +1475,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
   }
 
   if (strncasecmp(name, "debugFlag", 9) == 0) {
-    taosSetAllDebugFlag(pItem->i32);
+    taosSetAllDebugFlag(pCfg, pItem->i32);
     return 0;
   }
 
@@ -1533,7 +1549,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
   switch (lowcaseName[0]) {
     case 'd': {
       if (strcasecmp("debugFlag", name) == 0) {
-        taosSetAllDebugFlag(pItem->i32);
+        taosSetAllDebugFlag(pCfg, pItem->i32);
         matched = true;
       }
       break;
@@ -1757,11 +1773,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag
   taosSetDebugFlag(pFlagPtr, name, flag);
 }
 
-void taosSetAllDebugFlag(int32_t flag) {
+void taosSetGlobalDebugFlag(int32_t flag) { taosSetAllDebugFlag(tsCfg, flag); }
+
+static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) {
   if (flag <= 0) return;
 
   SArray      *noNeedToSetVars = NULL;
-  SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag");
+  SConfigItem *pItem = cfgGetItem(pCfg, "debugFlag");
   if (pItem != NULL) {
     pItem->i32 = flag;
     noNeedToSetVars = pItem->array;
@@ -1811,4 +1829,4 @@ int8_t taosGranted(int8_t type) {
       break;
   }
   return 0;
-}
+}

source/common/src/tmsg.c

@@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR
   tDecoderClear(&decoder);
   return 0;
 }
-int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
-  if (tStartEncode(&encoder) < 0) return -1;
-  tEndEncode(&encoder);
+// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   tEndEncode(&encoder);
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
@@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *
 
   return 0;
 }
+
 int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) {
   // impl later
   return;
 }
-int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
 
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  tEndEncode(&encoder);
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
-int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+//   tEndEncode(&encoder);
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
+// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
   SEncoder encoder = {0};
@@ -1436,19 +1437,15 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
 
 void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); }
 
-int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1456,25 +1453,67 @@ int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq
   return tlen;
 }
 
-int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
-  tEndDecode(&decoder);
 
+  if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1;
+  if (pReq->contLen > 0) {
+    pReq->pCont = taosMemoryMalloc(pReq->contLen + 1);
+    if (pReq->pCont == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
 }
 
+void tFreeSStatisReq(SStatisReq *pReq) {
+  taosMemoryFreeClear(pReq->pCont);
+}
+
+// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
+
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
+//   if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
+//   tEndEncode(&encoder);
+
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
+
+// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
+
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
+//   tEndDecode(&decoder);
+
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
+
 int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -3280,18 +3319,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq)
   return tlen;
 }
 
-int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1;
-  tEndDecode(&decoder);
-
-  tDecoderClear(&decoder);
-  return 0;
-}
-
 int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -3305,17 +3332,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq)
   return tlen;
 }
 
-int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) {
   SEncoder encoder = {0};
@@ -4233,17 +4260,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq
   return tlen;
 }
 
-int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) {
   if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1;
@@ -4325,24 +4352,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   return tlen;
 }
 
-int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
-  if (pReq->payloadLen > 0) {
-    pReq->payload = taosMemoryMalloc(pReq->payloadLen);
-    if (pReq->payload == NULL) return -1;
-    if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
-  }
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
+//   if (pReq->payloadLen > 0) {
+//     pReq->payload = taosMemoryMalloc(pReq->payloadLen);
+//     if (pReq->payload == NULL) return -1;
+//     if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
+//   }
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); }
 
@@ -4595,34 +4622,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) {
   taosArrayDestroy(pRsp->pIndexRsp);
 }
 
-int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
-  if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
-  tEndEncode(&encoder);
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
+//   if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 
-int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
-  if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
+//   if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
-void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
+// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
 
 int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
@@ -4930,17 +4957,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
   return tlen;
 }
 
-int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
-  tEndDecode(&decoder);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
   SEncoder encoder = {0};
@@ -4955,17 +4982,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR
   return tlen;
 }
 
-int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
-  tEndDecode(&decoder);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
   if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
@@ -5212,11 +5239,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo
 
   if (tStartDecode(&decoder) < 0) return -1;
 
-  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4;
+  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5;
+  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6;
 
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
@@ -5830,17 +5857,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom
   return tlen;
 }
 
-int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  tEndDecode(&decoder);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -5889,38 +5916,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq
   return 0;
 }
 
-int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
-  if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
-  if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
-  tEndEncode(&encoder);
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
+//   if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
+//   if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 
-int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
-  tEndDecode(&decoder);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) {
   SEncoder encoder = {0};
@@ -6958,30 +6985,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
 
 void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
 
-int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
-  // SEncoder encoder = {0};
-  // tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+//   // SEncoder encoder = {0};
+//   // tEncoderInit(&encoder, buf, bufLen);
 
-  // if (tStartEncode(&encoder) < 0) return -1;
-  // if (pRsp->rspList) {
-  //   int32_t num = taosArrayGetSize(pRsp->rspList);
-  //   if (tEncodeI32(&encoder, num) < 0) return -1;
-  //   for (int32_t i = 0; i < num; ++i) {
-  //     SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
-  //     if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
-  //   }
-  // } else {
-  //   if (tEncodeI32(&encoder, 0) < 0) return -1;
-  // }
-  // tEndEncode(&encoder);
+//   // if (tStartEncode(&encoder) < 0) return -1;
+//   // if (pRsp->rspList) {
+//   //   int32_t num = taosArrayGetSize(pRsp->rspList);
+//   //   if (tEncodeI32(&encoder, num) < 0) return -1;
+//   //   for (int32_t i = 0; i < num; ++i) {
+//   //     SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
+//   //     if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
+//   //   }
+//   // } else {
+//   //   if (tEncodeI32(&encoder, 0) < 0) return -1;
+//   // }
+//   // tEndEncode(&encoder);
 
-  // int32_t tlen = encoder.pos;
-  // tEncoderClear(&encoder);
-  // reture tlen;
-  return 0;
-}
+//   // int32_t tlen = encoder.pos;
+//   // tEncoderClear(&encoder);
+//   // reture tlen;
+//   return 0;
+// }
 
-int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
   // SDecoder  decoder = {0};
   // int32_t num = 0;
   // tDecoderInit(&decoder, buf, bufLen);
@@ -7002,8 +7029,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc
   // tEndDecode(&decoder);
 
   // tDecoderClear(&decoder);
-  return 0;
-}
+//  return 0;
+//}
 
 int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
   int32_t        nRsps = taosArrayGetSize(pRsp->pArray);
@@ -7138,15 +7165,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
   return 0;
 }
 
-int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
+// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
+//   if (tStartDecode(pCoder) < 0) return -1;
 
-  if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
-  if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
+//   if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
 
-  tEndDecode(pCoder);
-  return 0;
-}
+//   tEndDecode(pCoder);
+//   return 0;
+// }
 
 int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
@@ -7431,34 +7458,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq *
 
 void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); }
 
-int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 
-int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
   if (NULL == pReq) {
@@ -7858,37 +7885,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) {
   return 0;
 }
 
-static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
-  if (tStartEncode(pCoder) < 0) return -1;
+// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
+//   if (tStartEncode(pCoder) < 0) return -1;
 
-  if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
-  if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
-  if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
-  if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
+//   if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
+//   if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
+//   if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
+//   if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
 
-  if (flags & TD_AUTO_CREATE_TABLE) {
-    if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
-  }
+//   if (flags & TD_AUTO_CREATE_TABLE) {
+//     if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+//   }
 
-  tEndEncode(pCoder);
-  return 0;
-}
+//   tEndEncode(pCoder);
+//   return 0;
+// }
 
-static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
-  if (tStartDecode(pCoder) < 0) return -1;
+// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
+//   if (tStartDecode(pCoder) < 0) return -1;
 
-  if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
-  if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
-  if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
-  if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
+//   if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
+//   if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
 
-  if (flags & TD_AUTO_CREATE_TABLE) {
-    if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
-  }
+//   if (flags & TD_AUTO_CREATE_TABLE) {
+//     if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+//   }
 
-  tEndDecode(pCoder);
-  return 0;
-}
+//   tEndDecode(pCoder);
+//   return 0;
+// }
 
 static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) {
   if (tStartEncode(pEncoder) < 0) return -1;
@@ -7912,78 +7939,78 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
   return 0;
 }
 
-static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
-  pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
-  if (NULL == pBlock->tblFName) return -1;
-  if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
-  if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
+//   if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
+//   if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
+//   pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
+//   if (NULL == pBlock->tblFName) return -1;
+//   if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
+//   if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
 
-  int32_t meta = 0;
-  if (tDecodeI32(pDecoder, &meta) < 0) return -1;
-  if (meta) {
-    pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pBlock->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
-  } else {
-    pBlock->pMeta = NULL;
-  }
+//   int32_t meta = 0;
+//   if (tDecodeI32(pDecoder, &meta) < 0) return -1;
+//   if (meta) {
+//     pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     if (NULL == pBlock->pMeta) return -1;
+//     if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
+//   } else {
+//     pBlock->pMeta = NULL;
+//   }
 
-  tEndDecode(pDecoder);
-  return 0;
-}
+//   tEndDecode(pDecoder);
+//   return 0;
+// }
 
-int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
-  int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
+// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
+//   int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
 
-  if (tStartEncode(pEncoder) < 0) return -1;
+//   if (tStartEncode(pEncoder) < 0) return -1;
 
-  if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
-  for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
-    if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
-  }
+//   if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
+//   for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
+//     if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
+//   }
 
-  tEndEncode(pEncoder);
-  return 0;
-}
+//   tEndEncode(pEncoder);
+//   return 0;
+// }
 
-int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
-  pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
-  if (pRsp->pBlocks == NULL) return -1;
-  for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
-    if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
-  }
+//   if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
+//   pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
+//   if (pRsp->pBlocks == NULL) return -1;
+//   for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
+//     if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
+//   }
 
-  tEndDecode(pDecoder);
-  tDecoderClear(pDecoder);
-  return 0;
-}
+//   tEndDecode(pDecoder);
+//   tDecoderClear(pDecoder);
+//   return 0;
+// }
 
-void tFreeSSubmitBlkRsp(void *param) {
-  if (NULL == param) {
-    return;
-  }
+// void tFreeSSubmitBlkRsp(void *param) {
+//   if (NULL == param) {
+//     return;
+//   }
 
-  SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
+//   SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
 
-  taosMemoryFree(pRsp->tblFName);
-  if (pRsp->pMeta) {
-    taosMemoryFree(pRsp->pMeta->pSchemas);
-    taosMemoryFree(pRsp->pMeta);
-  }
-}
+//   taosMemoryFree(pRsp->tblFName);
+//   if (pRsp->pMeta) {
+//     taosMemoryFree(pRsp->pMeta->pSchemas);
+//     taosMemoryFree(pRsp->pMeta);
+//   }
+// }
 
 void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
   if (NULL == pRsp) return;
@@ -8157,23 +8184,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
-  int32_t  meta = 0;
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
+//   int32_t  meta = 0;
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (meta) {
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     if (NULL == pRsp->pMeta) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//   }
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) {
   if (tStartEncode(pEncoder) < 0) return -1;
@@ -8198,22 +8225,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
-  int32_t  meta = 0;
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
+//   int32_t  meta = 0;
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (meta) {
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     if (NULL == pRsp->pMeta) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//   }
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
   if (NULL == pRsp) {
@@ -8249,22 +8276,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
-  int32_t  meta = 0;
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
+//   int32_t  meta = 0;
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (meta) {
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     if (NULL == pRsp->pMeta) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//   }
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) {
   if (NULL == pRsp) {

source/common/src/trow.c

@@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa
   return TSDB_CODE_SUCCESS;
 }
 
+#if 0
 int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     terrno = TSDB_CODE_INVALID_PARA;
@@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal
   }
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
       break;
+#if 0
     case -1:
     case 1:
       tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
       break;
+#endif
     default:
       terrno = TSDB_CODE_INVALID_PARA;
       return TSDB_CODE_FAILED;
@@ -231,7 +235,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe
   return true;
 }
 
-#ifdef BUILD_NO_CALL
+// #ifdef BUILD_NO_CALL
 const uint8_t tdVTypeByte[2][3] = {{
                                        // 2 bits
                                        TD_VTYPE_NORM_BYTE_II,
@@ -439,13 +443,15 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) {
   return true;
 }
 
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) {
   STColumn *pTColumn;
   SColVal  *pColVal;
   int32_t   nColVal = taosArrayGetSize(pArray);
   int32_t   varDataLen = 0;
+  int32_t   nonVarDataLen = 0;
   int32_t   maxVarDataLen = 0;
   int32_t   iColVal = 0;
+  int32_t   nBound = 0;
   void     *varBuf = NULL;
   bool      isAlloc = false;
 
@@ -458,6 +464,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
     } else {
       pColVal = NULL;
     }
+    if (pColVal && !COL_VAL_IS_NONE(pColVal)) {
+      ++nBound;
+    }
 
     if (iColumn == 0) {
       ASSERT(pColVal && pColVal->cid == pTColumn->colId);
@@ -484,16 +493,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
             }
           }
         }
+      } else {
+        if(pColVal && COL_VAL_IS_VALUE(pColVal)) {
+          nonVarDataLen += TYPE_BYTES[pTColumn->type];
+        }
       }
     }
 
     ++iColVal;
   }
 
-  if (!(*ppRow)) {
-    *ppRow = (STSRow *)taosMemoryCalloc(
-        1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
-    isAlloc = true;
+  int32_t rowTotalLen = 0;
+  if (rowType == TD_ROW_TP) {
+    rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1);
+  } else {
+    rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) +
+                  TD_BITMAP_BYTES(nBound - 1);
+  }
+    if (!(*ppRow)) {
+      *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen);
+      isAlloc = true;
   }
 
   if (!(*ppRow)) {
@@ -512,10 +531,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
     }
   }
 
-  SRowBuilder rb = {0};
+  SRowBuilder rb = {.rowType = rowType};
   tdSRowInit(&rb, pTSchema->version);
-  tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen);
+  tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen);
   tdSRowResetBuf(&rb, *ppRow);
+  int32_t iBound = 0;
 
   iColVal = 0;
   for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) {
@@ -529,21 +549,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
         valType = TD_VTYPE_NONE;
       } else if (COL_VAL_IS_NULL(pColVal)) {
         valType = TD_VTYPE_NULL;
+        ++iBound;
       } else if (IS_VAR_DATA_TYPE(pTColumn->type)) {
         varDataSetLen(varBuf, pColVal->value.nData);
         if (pColVal->value.nData != 0) {
           memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
         }
         val = varBuf;
+        ++iBound;
       } else {
         val = (const void *)&pColVal->value.val;
+        ++iBound;
       }
     } else {
       // pColVal = NULL;
       valType = TD_VTYPE_NONE;
     }
 
-    tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+    if (TD_IS_TP_ROW(rb.pBuf)) {
+      tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+    } else {
+      tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1);
+    }
 
     ++iColVal;
   }
@@ -605,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell
 
   return true;
 }
-
+#if 0
 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
   if (!pBitmap || colIdx < 0) {
     terrno = TSDB_CODE_INVALID_PARA;
@@ -655,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
   }
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) {
 #ifdef TD_SUPPORT_BITMAP
@@ -761,11 +789,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp
 
 int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
                               int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) {
-  if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) {
+  if (colIdx < 1) {
     terrno = TSDB_CODE_INVALID_PARA;
+    ASSERTS(0, "colIdx is %" PRIi64, colIdx);
     return terrno;
   }
-  offset -= sizeof(SKvRowIdx);
   --colIdx;
 
 #ifdef TD_SUPPORT_BITMAP
@@ -779,6 +807,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
   SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
   pColIdx->colId = colId;
   pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
+  pBuilder->offset += sizeof(SKvRowIdx);
   if (valType == TD_VTYPE_NORM) {
     char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     if (IS_VAR_DATA_TYPE(colType)) {
@@ -830,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
 
   return 0;
 }
-
+#if 0
 int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,
                               int32_t allNullLen, int32_t boundNullLen) {
   if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) {
@@ -866,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou
 #endif
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
   pBuilder->pBuf = (STSRow *)pBuf;
@@ -904,6 +934,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
       TD_ROW_SET_LEN(pBuilder->pBuf, len);
       TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
       TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols);
+      pBuilder->offset = 0;
       break;
     default:
       terrno = TSDB_CODE_INVALID_PARA;
@@ -1040,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
     case 0:
       tdSetBitmapValTypeII(pBitmap, colIdx, valType);
       break;
+#if 0
     case -1:
     case 1:
       tdSetBitmapValTypeI(pBitmap, colIdx, valType);
       break;
+#endif
     default:
       terrno = TSDB_CODE_INVALID_PARA;
       return TSDB_CODE_FAILED;
@@ -1056,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
 void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) {
   STColumn *pTColumn = &pTSchema->columns[iCol];
   SCellVal  cv = {0};
-  SValue    value = {0};
 
   ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0));
 
@@ -1084,5 +1116,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV
       memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes);
     }
   }
-}
-#endif
+}

source/common/test/CMakeLists.txt

@@ -35,6 +35,10 @@ target_include_directories(
         PUBLIC "${TD_SOURCE_DIR}/include/common"
         PUBLIC "${TD_SOURCE_DIR}/include/util"
 )
+add_test(
+    NAME dataformatTest
+    COMMAND dataformatTest
+)
 
 # tmsg test
 # add_executable(tmsgTest "")

source/common/test/dataformatTest.cpp

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#if 0
 #include <gtest/gtest.h>
 
 #include <taoserror.h>
@@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) {
       case 1: {
         pSchema[i].type = TSDB_DATA_TYPE_INT;
         pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
-        ;
       } break;
       case 2: {
         pSchema[i].type = TSDB_DATA_TYPE_BIGINT;
@@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) {
   return pResult;
 }
 
-// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+// ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
+// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
 static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
   if (!(*pArray)) {
     *pArray = taosArrayInit(nCols, sizeof(SColVal));
@@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
       taosArrayPush(*pArray, &colVal);
       continue;
     }
-
     switch (i) {
       case 0:
-        sscanf(data[i], "%" PRIi64, &colVal.value.ts);
+        colVal.type = TSDB_DATA_TYPE_TIMESTAMP;
+        sscanf(data[i], "%" PRIi64, &colVal.value.val);
         break;
       case 1:
-        sscanf(data[i], "%" PRIi32, &colVal.value.i32);
+        colVal.type = TSDB_DATA_TYPE_INT;
+        sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val);
         break;
       case 2:
-        sscanf(data[i], "%" PRIi64, &colVal.value.i64);
+        colVal.type = TSDB_DATA_TYPE_BIGINT;
+        sscanf(data[i], "%" PRIi64, &colVal.value.val);
         break;
       case 3:
-        sscanf(data[i], "%f", &colVal.value.f);
+        colVal.type = TSDB_DATA_TYPE_FLOAT;
+        sscanf(data[i], "%f", (float *)&colVal.value.val);
         break;
       case 4:
-        sscanf(data[i], "%lf", &colVal.value.d);
+        colVal.type = TSDB_DATA_TYPE_DOUBLE;
+        sscanf(data[i], "%lf", (double *)&colVal.value.val);
         break;
       case 5: {
+        colVal.type = TSDB_DATA_TYPE_BINARY;
         int16_t dataLen = strlen(data[i]) + 1;
         colVal.value.nData = dataLen < 10 ? dataLen : 10;
         colVal.value.pData = (uint8_t *)data[i];
       } break;
       case 6: {
+        colVal.type = TSDB_DATA_TYPE_NCHAR;
         int16_t dataLen = strlen(data[i]) + 1;
         colVal.value.nData = dataLen < 40 ? dataLen : 40;
         colVal.value.pData = (uint8_t *)data[i];  // just for test, not real nchar
       } break;
-      case 7:
-      case 9: {
+      case 7: {
+        colVal.type = TSDB_DATA_TYPE_TINYINT;
         int32_t d8;
         sscanf(data[i], "%" PRId32, &d8);
-        colVal.value.i8 = (int8_t)d8;
-      } break;
+        colVal.value.val = (int8_t)d8;
+      }
       case 8: {
+        colVal.type = TSDB_DATA_TYPE_SMALLINT;
         int32_t d16;
         sscanf(data[i], "%" PRId32, &d16);
-        colVal.value.i16 = (int16_t)d16;
+        colVal.value.val = (int16_t)d16;
+      } break;
+      case 9: {
+        colVal.type = TSDB_DATA_TYPE_BOOL;
+        int32_t d8;
+        sscanf(data[i], "%" PRId32, &d8);
+        colVal.value.val = (int8_t)d8;
       } break;
       case 10: {
+        colVal.type = TSDB_DATA_TYPE_UTINYINT;
         uint32_t u8;
         sscanf(data[i], "%" PRId32, &u8);
-        colVal.value.u8 = (uint8_t)u8;
+        colVal.value.val = (uint8_t)u8;
       } break;
       case 11: {
+        colVal.type = TSDB_DATA_TYPE_USMALLINT;
         uint32_t u16;
         sscanf(data[i], "%" PRId32, &u16);
-        colVal.value.u16 = (uint16_t)u16;
+        colVal.value.val = (uint16_t)u16;
       } break;
       case 12: {
-        sscanf(data[i], "%" PRIu32, &colVal.value.u32);
+        colVal.type = TSDB_DATA_TYPE_UINT;
+        sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val);
       } break;
       case 13: {
-        sscanf(data[i], "%" PRIu64, &colVal.value.u64);
+        colVal.type = TSDB_DATA_TYPE_UBIGINT;
+        sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val);
       } break;
       default:
         ASSERT(0);
@@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
   }
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
-      printf("%s ", cv->value.i8 == 0 ? "false" : "true");
+      printf("%s ", cv->value.val == 0 ? "false" : "true");
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      printf("%" PRIi8 " ", cv->value.i8);
+      printf("%" PRIi8 " ", *(int8_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      printf("%" PRIi16 " ", cv->value.i16);
+      printf("%" PRIi16 " ", *(int16_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_INT:
-      printf("%" PRIi32 " ", cv->value.i32);
+      printf("%" PRIi32 " ", *(int32_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      printf("%" PRIi64 " ", cv->value.i64);
+      printf("%" PRIi64 " ", cv->value.val);
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      printf("%f ", cv->value.f);
+      printf("%f ", *(float *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      printf("%lf ", cv->value.d);
+      printf("%lf ", *(double *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_GEOMETRY: {
@@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
       printf("%s ", tv);
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP:
-      printf("%" PRIi64 " ", cv->value.i64);
+      printf("%" PRIi64 " ", cv->value.val);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       char tv[15] = {0};
@@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
       printf("%s ", tv);
     } break;
     case TSDB_DATA_TYPE_UTINYINT:
-      printf("%" PRIu8 " ", cv->value.u8);
+      printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      printf("%" PRIu16 " ", cv->value.u16);
+      printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_UINT:
-      printf("%" PRIu32 " ", cv->value.u32);
+      printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      printf("%" PRIu64 " ", cv->value.u64);
+      printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_JSON:
       printf("JSON ");
@@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
   return 0;
 }
 
-void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
-  // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) {
+  printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len);
   for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
-    tTSRowGet(row, pTSchema, i, &cv);
+    tTSRowGetVal(row, pTSchema, i, &cv);
     debugPrintSColVal(&cv, pTSchema->columns[i].type);
   }
   printf("\n");
@@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
     case TSDB_DATA_TYPE_TINYINT: {
       int32_t d8;
       sscanf(rawVal, "%" PRId32, &d8);
-      EXPECT_EQ(cv->value.i8, (int8_t)d8);
+      EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8);
     } break;
     case TSDB_DATA_TYPE_SMALLINT: {
       int32_t d16;
       sscanf(rawVal, "%" PRId32, &d16);
-      EXPECT_EQ(cv->value.i16, (int16_t)d16);
+      EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16);
     } break;
     case TSDB_DATA_TYPE_INT: {
-      sscanf(rawVal, "%" PRId32, &rawSVal.i32);
-      EXPECT_EQ(cv->value.i32, rawSVal.i32);
+      sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val);
+      EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_BIGINT: {
-      sscanf(rawVal, "%" PRIi64, &rawSVal.i64);
-      EXPECT_EQ(cv->value.i64, rawSVal.i64);
+      sscanf(rawVal, "%" PRIi64, &rawSVal.val);
+      EXPECT_EQ(cv->value.val, rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_FLOAT: {
-      sscanf(rawVal, "%f", &rawSVal.f);
-      EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f);
+      sscanf(rawVal, "%f", (float *)&rawSVal.val);
+      EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_DOUBLE: {
-      sscanf(rawVal, "%lf", &rawSVal.d);
-      EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d);
+      sscanf(rawVal, "%lf", (double *)&rawSVal.val);
+      EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_GEOMETRY: {
       EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      sscanf(rawVal, "%" PRIi64, &rawSVal.ts);
-      EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts);
+      sscanf(rawVal, "%" PRIi64, &rawSVal.val);
+      EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_NCHAR: {
       EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);  // informal nchar comparsion
@@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
     case TSDB_DATA_TYPE_UTINYINT: {
       uint32_t u8;
       sscanf(rawVal, "%" PRIu32, &u8);
-      EXPECT_EQ(cv->value.u8, (uint8_t)u8);
+      EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8);
     } break;
     case TSDB_DATA_TYPE_USMALLINT: {
       uint32_t u16;
       sscanf(rawVal, "%" PRIu32, &u16);
-      EXPECT_EQ(cv->value.u16, (uint16_t)u16);
+      EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16);
     } break;
     case TSDB_DATA_TYPE_UINT: {
-      sscanf(rawVal, "%" PRIu32, &rawSVal.u32);
-      EXPECT_EQ(cv->value.u32, rawSVal.u32);
+      sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val);
+      EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_UBIGINT: {
-      sscanf(rawVal, "%" PRIu64, &rawSVal.u64);
-      EXPECT_EQ(cv->value.u64, rawSVal.u64);
+      sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val);
+      EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_JSON:
       printf("JSON ");
@@ -395,36 +411,66 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
   return 0;
 }
 
-static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
+static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) {
   for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
-    tTSRowGet(row, pTSchema, i, &cv);
+    tTSRowGetVal(row, pTSchema, i, &cv);
     checkSColVal(data[i], &cv, pTSchema->columns[i].type);
   }
+
+  STSRowIter rowIter = {0};
+  rowIter.pSchema = pTSchema;
+  tdSTSRowIterReset(&rowIter, row);
+  for (int32_t i = 0; i < pTSchema->numOfCols; ++i) {
+    STColumn *pCol = pTSchema->columns + i;
+    SColVal   colVal = {0};
+    SCellVal  cv = {0};
+    if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) {
+      break;
+    }
+
+    colVal.cid = pCol->colId;
+    colVal.type = pCol->type;
+    if (tdValTypeIsNone(cv.valType)) {
+      colVal.flag = CV_FLAG_NONE;
+    } else if (tdValTypeIsNull(cv.valType)) {
+      colVal.flag = CV_FLAG_NULL;
+    } else {
+      colVal.flag = CV_FLAG_VALUE;
+
+      if (IS_VAR_DATA_TYPE(pCol->type)) {
+        colVal.value.nData = varDataLen(cv.val);
+        colVal.value.pData = (uint8_t *)varDataVal(cv.val);
+      } else {
+        memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes);
+      }
+    }
+    checkSColVal(data[i], &colVal, pCol->type);
+  }
 }
 
 TEST(testCase, AllNormTest) {
-  int16_t       nCols = 14;
-  STSRowBuilder rb = {0};
-  STSRow2      *row = nullptr;
-  SArray       *pArray = taosArrayInit(nCols, sizeof(SColVal));
+  int16_t nCols = 14;
+  STSRow *row = nullptr;
+  SArray *pArray = taosArrayInit(nCols, sizeof(SColVal));
   EXPECT_NE(pArray, nullptr);
 
   STSchema *pTSchema = genSTSchema(nCols);
   EXPECT_NE(pTSchema, nullptr);
 
-  // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
-  // c9 bool
-  char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"};
+  // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
+  // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
+  char *data[14] = {
+      "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"};
 
   genTestData((const char **)&data, nCols, &pArray);
 
-  tTSRowNew(&rb, pArray, pTSchema, &row);
-
+  tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP);
   debugPrintTSRow(row, pTSchema, __func__, __LINE__);
+  tdSRowPrint(row, pTSchema, __func__);
   checkTSRow((const char **)&data, row, pTSchema);
 
-  tsRowBuilderClear(&rb);
+  taosMemoryFreeClear(row);
   taosArrayDestroy(pArray);
   taosMemoryFree(pTSchema);
 }
@@ -433,7 +479,7 @@ TEST(testCase, AllNormTest) {
 TEST(testCase, NoneTest) {
   const static int nCols = 14;
   const static int nRows = 20;
-  STSRow2         *row = nullptr;
+  STSRow          *row = nullptr;
   SArray          *pArray = taosArrayInit(nCols, sizeof(SColVal));
   EXPECT_NE(pArray, nullptr);
 
@@ -442,42 +488,46 @@ TEST(testCase, NoneTest) {
 
   // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
   // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
-  const char *data[nRows][nCols] = {
-      {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
-      {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
-      {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
-      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
-      {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
-      {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
-      {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
-      {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
-      {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
-      {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
-      {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
-      {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
-       "18446744073709551615"},
-      {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
-      {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
-      {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
-       "4294967295", "100"},
-      {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
-       "no"}};
+  const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV,
+                                 TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV,
+                                 TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP};
 
-  for (int r = 0; r < nRows; ++r) {
-    genTestData((const char **)&data[r], nCols, &pArray);
-    tTSRowNew(NULL, pArray, pTSchema, &row);
-    debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
-    checkTSRow((const char **)&data[r], row, pTSchema);  // check
-    tTSRowFree(row);
-    taosArrayClear(pArray);
+const char  *data[nRows][nCols] = {
+    {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
+    {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+    {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+    {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+    {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
+    {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+    {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+    {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
+    {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+    {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+    {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
+    {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
+    {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
+    {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
+    {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
+    {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
+      "18446744073709551615"},
+    {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+    {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
+    {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
+      "4294967295", "100"},
+    {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
+      "no"}};
+
+for (int r = 0; r < nRows; ++r) {
+  genTestData((const char **)&data[r], nCols, &pArray);
+  tdSTSRowNew(pArray, pTSchema, &row, rowType[r]);
+  debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
+  tdSRowPrint(row, pTSchema, __func__);
+  checkTSRow((const char **)&data[r], row, pTSchema);  // check
+  taosMemoryFreeClear(row);
+  taosArrayClear(pArray);
   }
 
   taosArrayDestroy(pArray);
   taosMemoryFree(pTSchema);
 }
-#endif
 #endif

source/dnode/mgmt/exe/dmMain.c

@@ -68,7 +68,7 @@ static struct {
   int64_t      startTime;
 } global = {0};
 
-static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); }
+static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); }
 static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; }
 
 static void dmStopDnode(int signum, void *sigInfo, void *context) {

source/dnode/mgmt/mgmt_dnode/inc/dmInt.h

@@ -43,6 +43,7 @@ typedef struct SDnodeMgmt {
   GetMnodeLoadsFp        getMnodeLoadsFp;
   GetQnodeLoadsFp        getQnodeLoadsFp;
   int32_t                statusSeq;
+  SendMonitorReportFp    sendMonitorReportFpBasic;
 } SDnodeMgmt;
 
 // dmHandle.c

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "dmInt.h"
 #include "systable.h"
-#include "tgrant.h"
 
 extern SConfig *tsCfg;
 
@@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   req.memTotal = tsTotalMemoryKB * 1024;
   req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024;
   tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN);
-  char *machine = tGetMachineId();
-  if (machine) {
-    tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1);
-    taosMemoryFreeClear(machine);
-  }
+  tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1);
 
   req.clusterCfg.statusInterval = tsStatusInterval;
   req.clusterCfg.checkTime = 0;
@@ -365,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
-
+#if 0
   if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
     terrno = TSDB_CODE_MND_NO_RIGHTS;
     return -1;
   }
-
+#endif
   if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;

source/dnode/mgmt/mgmt_dnode/src/dmInt.c

@@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   pMgmt->processDropNodeFp = pInput->processDropNodeFp;
   pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
   pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
+  pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic;
   pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
   pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
   pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;

source/dnode/mgmt/mgmt_dnode/src/dmWorker.c

@@ -75,6 +75,7 @@ static void *dmNotifyThreadFp(void *param) {
 static void *dmMonitorThreadFp(void *param) {
   SDnodeMgmt *pMgmt = param;
   int64_t     lastTime = taosGetTimestampMs();
+  int64_t     lastTimeForBasic = taosGetTimestampMs();
   setThreadName("dnode-monitor");
 
   static int32_t TRIM_FREQ = 20;
@@ -85,6 +86,7 @@ static void *dmMonitorThreadFp(void *param) {
     if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
 
     int64_t curTime = taosGetTimestampMs();
+
     if (curTime < lastTime) lastTime = curTime;
     float interval = (curTime - lastTime) / 1000.0f;
     if (interval >= tsMonitorInterval) {
@@ -96,6 +98,15 @@ static void *dmMonitorThreadFp(void *param) {
         taosMemoryTrim(0);
       }
     }
+    
+    if(tsMonitorForceV2){
+      if (curTime < lastTimeForBasic) lastTimeForBasic = curTime;
+      float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f;
+      if (intervalForBasic >= tsMonitorIntervalForBasic) {
+        (*pMgmt->sendMonitorReportFpBasic)();
+        lastTimeForBasic = curTime;
+      }
+    }
   }
 
   return NULL;

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -191,6 +191,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_snode/src/smHandle.c

@@ -87,7 +87,10 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
 
   code = 0;
 _OVER:

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -835,8 +835,11 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -194,26 +194,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   while (pVnode->refCount > 0) taosMsleep(10);
 
   dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
-        pVnode->pWriteW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pWriteW.queue));
   tMultiWorkerCleanup(&pVnode->pWriteW);
 
   dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
-        pVnode->pSyncW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncW.queue));
   tMultiWorkerCleanup(&pVnode->pSyncW);
 
   dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
-        pVnode->pSyncRdW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncRdW.queue));
   tMultiWorkerCleanup(&pVnode->pSyncRdW);
 
   dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
-        pVnode->pApplyW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pApplyW.queue));
   tMultiWorkerCleanup(&pVnode->pApplyW);
 
   dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
 
   dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
-        pVnode->pFetchQ->threadId);
+        taosQueueGetThreadId(pVnode->pFetchQ));
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
 
   tqNotifyClose(pVnode->pImpl->pTq);

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -365,16 +365,16 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   }
 
   dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
-        pVnode->pWriteW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pWriteW.queue));
   dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
-        pVnode->pSyncW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncW.queue));
   dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
-        pVnode->pSyncRdW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncRdW.queue));
   dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
-        pVnode->pApplyW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pApplyW.queue));
   dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
   dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
-        pVnode->pFetchQ->threadId);
+        taosQueueGetThreadId(pVnode->pFetchQ));
   dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
   return 0;
 }

source/dnode/mgmt/node_mgmt/CMakeLists.txt

@@ -1,7 +1,7 @@
 aux_source_directory(src IMPLEMENT_SRC)
 add_library(dnode STATIC ${IMPLEMENT_SRC})
 target_link_libraries(
-    dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode
+    dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
 )
 
 IF (TD_STORAGE)
@@ -14,6 +14,10 @@ IF (TD_STORAGE)
 
 ENDIF ()
 
+IF (DEFINED GRANT_CFG_INCLUDE_DIR)
+  add_definitions(-DGRANTS_CFG)
+ENDIF()
+
 target_include_directories(
     dnode
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -128,6 +128,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 // dmMonitor.c
 void dmSendMonitorReport();
 void dmSendAuditRecords();
+void dmSendMonitorReportBasic();
 void dmGetVnodeLoads(SMonVloadInfo *pInfo);
 void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
 void dmGetMnodeLoads(SMonMloadInfo *pInfo);

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -394,6 +394,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
       .processDropNodeFp = dmProcessDropNodeReq,
       .sendMonitorReportFp = dmSendMonitorReport,
       .sendAuditRecordFp = auditSendRecordsInBatch,
+      .sendMonitorReportFpBasic = dmSendMonitorReportBasic,
       .getVnodeLoadsFp = dmGetVnodeLoads,
       .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
       .getMnodeLoadsFp = dmGetMnodeLoads,

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -22,6 +22,7 @@
 #ifdef TD_TSZ
 #include "tcompression.h"
 #include "tglobal.h"
+#include "tgrant.h"
 #endif
 
 static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
@@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) {
   pData->rebootTime = taosGetTimestampMs();
   pData->dropped = 0;
   pData->stopped = 0;
+  char *machineId = tGetMachineId();
+  if (machineId) {
+    tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1);
+    taosMemoryFreeClear(machineId);
+  } else {
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
+    terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
+    return -1;
+#endif
+  }
 
   pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   if (pData->dnodeHash == NULL) {

source/dnode/mgmt/node_mgmt/src/dmMonitor.c

@@ -25,8 +25,16 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
 }
 
+static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) {
+  pInfo->protocol = 2;
+  pInfo->dnode_id = pDnode->data.dnodeId;
+  pInfo->cluster_id = pDnode->data.clusterId;
+  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+}
+
 static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
-  pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+  //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+  pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f;
   pInfo->has_mnode = pDnode->wrappers[MNODE].required;
   pInfo->has_qnode = pDnode->wrappers[QNODE].required;
   pInfo->has_snode = pDnode->wrappers[SNODE].required;
@@ -44,6 +52,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) {
   monSetDmInfo(&dmInfo);
 }
 
+static void dmGetDmMonitorInfoBasic(SDnode *pDnode) {
+  SMonDmInfo dmInfo = {0};
+  dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic);
+  monSetDmInfo(&dmInfo);
+}
+
 static void dmGetMmMonitorInfo(SDnode *pDnode) {
   SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE];
   if (dmMarkWrapper(pWrapper) == 0) {
@@ -106,7 +120,17 @@ void dmSendMonitorReport() {
   dmGetVmMonitorInfo(pDnode);
   dmGetQmMonitorInfo(pDnode);
   dmGetSmMonitorInfo(pDnode);
-  monSendReport();
+  monGenAndSendReport();
+}
+
+void dmSendMonitorReportBasic() {
+  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+  SDnode *pDnode = dmInstance();
+  dmGetDmMonitorInfoBasic(pDnode);
+  dmGetMmMonitorInfo(pDnode);
+  monGenAndSendReportBasic();
 }
 
 //Todo: put this in seperate file in the future

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   rpcInit.parent = pDnode;
   rpcInit.rfp = rpcRfp;
   rpcInit.compressSize = tsCompressMsgSize;
+  rpcInit.dfp = destroyAhandle;
 
   rpcInit.retryMinInterval = tsRedirectPeriod;
   rpcInit.retryStepFactor = tsRedirectFactor;

source/dnode/mgmt/node_util/CMakeLists.txt

@@ -6,5 +6,5 @@ target_include_directories(
     PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor
+    node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw
 )

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -109,6 +109,7 @@ typedef struct {
   SMsgCb         msgCb;
   bool           validMnodeEps;
   int64_t        ipWhiteVer;
+  char           machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeData;
 
 typedef struct {
@@ -122,6 +123,7 @@ typedef struct {
   ProcessDropNodeFp   processDropNodeFp;
   SendMonitorReportFp sendMonitorReportFp;
   SendAuditRecordsFp  sendAuditRecordFp;
+  SendMonitorReportFp sendMonitorReportFpBasic;
   GetVnodeLoadsFp     getVnodeLoadsFp;
   GetVnodeLoadsFp     getVnodeLoadsLiteFp;
   GetMnodeLoadsFp     getMnodeLoadsFp;

source/dnode/mnode/impl/CMakeLists.txt

@@ -23,9 +23,13 @@ target_include_directories(
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit
+    mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw
 )
 
+IF (DEFINED GRANT_CFG_INCLUDE_DIR)
+  add_definitions(-DGRANTS_CFG)
+ENDIF()
+
 IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(mnode grant)
   ADD_DEFINITIONS(-D_GRANT)

source/dnode/mnode/impl/inc/mndIndex.h

@@ -19,16 +19,16 @@ int32_t  mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool
 int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq);
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MND_IDX_H_*/
+#endif /*_TD_MND_IDX_H_*/

source/dnode/mnode/impl/inc/mndStream.h

@@ -86,6 +86,10 @@ typedef struct SOrphanTask {
   int32_t nodeId;
 } SOrphanTask;
 
+typedef struct {
+  SMsgHead head;
+} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg;
+
 int32_t     mndInitStream(SMnode *pMnode);
 void        mndCleanupStream(SMnode *pMnode);
 SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
@@ -119,6 +123,7 @@ int32_t     mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *
 int32_t     mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 int32_t     mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
 int32_t     mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t     mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
 
 SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
 void             destroyStreamTaskIter(SStreamTaskIter *pIter);

source/dnode/mnode/impl/src/mndCompact.c

@@ -240,7 +240,7 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompac
 
   SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
   if (pVgRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) {
     sdbFreeRaw(pVgRaw);
     return -1;
   }
@@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
 }
 
 static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
   if (pTrans == NULL) {
     mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
     return -1;
   }
   mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
 
+  mndTransSetDbName(pTrans, pCompact->dbname, NULL);
+
   SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
@@ -378,7 +380,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
   }
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
 
-  void   *pIter = NULL;
+  void *pIter = NULL;
   while (1) {
     SCompactDetailObj *pDetail = NULL;
     pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
@@ -452,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-  char obj[MND_COMPACT_ID_LEN] = {0};
+  char obj[TSDB_INT32_ID_LEN] = {0};
   sprintf(obj, "%d", pCompact->compactId);
 
   auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);
@@ -488,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
     sdbRelease(pMnode->pSdb, pDetail);
   }
 
-  return -1;
+  return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST;
 }
 
 int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
   SQueryCompactProgressRsp req = {0};
-  if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
+  int32_t code = 0;
+  code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
+  if (code != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
+    mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", 
+          code, pReq->pCont, pReq->contLen);
     return -1;
   }
 
@@ -502,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
         req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
 
   SMnode        *pMnode = pReq->info.node;
-  int32_t       code = -1;
-  
 
-  if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
+  code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
+  if(code != 0){
+    terrno = code;
     mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", 
         req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
     return -1;
@@ -612,15 +618,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
     return 0;
   }
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
   if (pTrans == NULL) {
     mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
     return -1;
   }
   mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
-
+  
   SCompactObj   *pCompact = mndAcquireCompact(pMnode, compactId);
 
+  mndTransSetDbName(pTrans, pCompact->dbname, NULL);
+
   pIter = NULL;
   while (1) {
     SCompactDetailObj *pDetail = NULL;

source/dnode/mnode/impl/src/mndDb.c

@@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionDelete(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew);
-static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t  mndProcessCreateDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessAlterDbReq(SRpcMsg *pReq);
@@ -256,17 +256,29 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
-  SDbObj *pNewDb = pObj;
+static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  SDbObj  *pNewDb = NULL;
+  int      code = -1;
+
+  pRow = mndDbActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  pNewDb = sdbGetRowObj(pRow);
+  if (pNewDb == NULL) goto _OVER;
 
   SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name);
   if (pOldDb != NULL) {
     mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name);
     sdbRelease(pMnode->pSdb, pOldDb);
-    return -1;
+    goto _OVER;
   }
 
-  return 0;
+  code = 0;
+_OVER:
+  if (pNewDb) mndDbActionDelete(pSdb, pNewDb);
+  taosMemoryFreeClear(pRow);
+  return code;
 }
 
 static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) {
@@ -884,10 +896,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   return terrno;
 }
 
-static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
+static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -943,7 +955,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
   mndTransSetDbName(pTrans, pOld->name, NULL);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
+  if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -1120,10 +1132,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
+static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -1257,7 +1269,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
     goto _OVER;
   }
 
-  if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/

source/dnode/mnode/impl/src/mndDnode.c

@@ -14,6 +14,8 @@
  */
 
 #define _DEFAULT_SOURCE
+#include <stdio.h>
+#include "tjson.h"
 #include "mndDnode.h"
 #include "audit.h"
 #include "mndCluster.h"
@@ -28,6 +30,7 @@
 #include "mndVgroup.h"
 #include "tmisce.h"
 #include "tunit.h"
+#include "taos_monitor.h"
 
 #define TSDB_DNODE_VER_NUMBER   2
 #define TSDB_DNODE_RESERVE_SIZE 64
@@ -74,6 +77,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp);
 static int32_t mndProcessStatusReq(SRpcMsg *pReq);
 static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
 static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
+static int32_t mndProcessStatisReq(SRpcMsg *pReq);
 
 static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
@@ -109,6 +113,7 @@ int32_t mndInitDnode(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq);
   mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
   mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig);
@@ -141,7 +146,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
     memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN);
     taosMemoryFreeClear(machineId);
   } else {
-#ifdef TD_ENTERPRISE
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
     terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
     goto _OVER;
 #endif
@@ -410,9 +415,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
     dInfo.ep.port = pDnode->port;
     dInfo.offlineReason = pDnode->offlineReason;
     tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-    tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1);
     sdbRelease(pSdb, pDnode);
     if (mndIsMnode(pMnode, pDnode->id)) {
       dInfo.isMnode = 1;
@@ -502,6 +504,207 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) {
   return stateChanged;
 }
 
+static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
+  SMnode    *pMnode = pReq->info.node;
+  SStatisReq statisReq = {0};
+  int32_t    code = -1;
+
+  char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
+  sprintf(strClusterId, "%"PRId64, pMnode->clusterId);
+
+  if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return code;
+  }
+
+  if(tsMonitorLogProtocol){
+    mInfo("process statis req,\n %s", statisReq.pCont);
+  }
+
+  SJson* pJson = tjsonParse(statisReq.pCont);
+
+  int32_t ts_size = tjsonGetArraySize(pJson);
+
+  for(int32_t i = 0; i < ts_size; i++){
+    SJson* item = tjsonGetArrayItem(pJson, i);
+
+    SJson* tables = tjsonGetObjectItem(item, "tables");
+
+    int32_t tableSize = tjsonGetArraySize(tables);
+    for(int32_t i = 0; i < tableSize; i++){
+      SJson* table = tjsonGetArrayItem(tables, i);
+
+      char tableName[MONITOR_TABLENAME_LEN] = {0};
+      tjsonGetStringValue(table, "name", tableName);
+
+      SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+      int32_t size = tjsonGetArraySize(metricGroups);
+      for(int32_t i = 0; i < size; i++){
+        SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+        SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+        int32_t tagSize = tjsonGetArraySize(arrayTag);
+        for(int32_t j = 0; j < tagSize; j++){
+          SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+          char tagName[MONITOR_TAG_NAME_LEN] = {0};
+          tjsonGetStringValue(item, "name", tagName);
+
+          if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+            tjsonDeleteItemFromObject(item, "value");
+            tjsonAddStringToObject(item, "value", strClusterId);
+          }
+        }
+      }
+    }
+  }
+
+  char *pCont = tjsonToString(pJson);
+  monSendContent(pCont);
+
+  if(pJson != NULL){
+    tjsonDelete(pJson);
+    pJson = NULL;
+  }
+
+  if(pCont != NULL){
+    taosMemoryFree(pCont);
+    pCont = NULL;
+  }
+
+  tFreeSStatisReq(&statisReq);
+  return 0;
+
+/*
+  SJson* pJson = tjsonParse(statisReq.pCont);
+
+  int32_t ts_size = tjsonGetArraySize(pJson);
+
+  for(int32_t i = 0; i < ts_size; i++){
+    SJson* item = tjsonGetArrayItem(pJson, i);
+
+    SJson* tables = tjsonGetObjectItem(item, "tables");
+
+    int32_t tableSize = tjsonGetArraySize(tables);
+    for(int32_t i = 0; i < tableSize; i++){
+      SJson* table = tjsonGetArrayItem(tables, i);
+
+      char tableName[MONITOR_TABLENAME_LEN] = {0};
+      tjsonGetStringValue(table, "name", tableName);
+
+      SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+      int32_t size = tjsonGetArraySize(metricGroups);
+      for(int32_t i = 0; i < size; i++){
+        SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+        SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+        int32_t tagSize = tjsonGetArraySize(arrayTag);
+
+        char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+        char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+
+        for(int32_t j = 0; j < tagSize; j++){
+          SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+          *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); 
+          tjsonGetStringValue(item, "name", *(labels + j));
+
+          *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
+          tjsonGetStringValue(item, "value", *(sample_labels + j));
+          if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+            strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
+          }
+        }
+
+        SJson* metrics = tjsonGetObjectItem(item, "metrics");
+
+        int32_t metricLen = tjsonGetArraySize(metrics);
+        for(int32_t j = 0; j < metricLen; j++){
+          SJson *item = tjsonGetArrayItem(metrics, j);
+
+          char name[MONITOR_METRIC_NAME_LEN] = {0}; 
+          tjsonGetStringValue(item, "name", name);
+
+          double value = 0;
+          tjsonGetDoubleValue(item, "value", &value);
+
+          double type = 0;
+          tjsonGetDoubleValue(item, "type", &type);
+
+          int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
+          char* metricName = taosMemoryMalloc(metricNameLen); 
+          memset(metricName, 0, metricNameLen);
+          sprintf(metricName, "%s:%s", tableName, name);
+
+          taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
+          if(metric == NULL){
+            if(type == 0){
+              metric = taos_counter_new(metricName, "",  tagSize, (const char**)labels);
+            }
+            if(type == 1){
+              metric = taos_gauge_new(metricName, "",  tagSize, (const char**)labels);
+            }
+            mTrace("fail to get metric from registry, new one metric:%p", metric);
+
+            if(taos_collector_registry_register_metric(metric) == 1){
+              if(type == 0){
+                taos_counter_destroy(metric);
+              }
+              if(type == 1){
+                taos_gauge_destroy(metric);
+              }
+
+              metric = taos_collector_registry_get_metric(metricName);
+
+              mTrace("fail to register metric, get metric from registry:%p", metric);
+            }
+            else{
+              mTrace("succeed to register metric:%p", metric);
+            }
+          }
+          else{
+            mTrace("get metric from registry:%p", metric);
+          }
+          
+          if(type == 0){
+            taos_counter_add(metric, value, (const char**)sample_labels);
+          }
+          if(type == 1){
+            taos_gauge_set(metric, value, (const char**)sample_labels);
+          }
+
+          taosMemoryFreeClear(metricName);
+        }
+
+        for(int32_t j = 0; j < tagSize; j++){
+          taosMemoryFreeClear(*(labels + j));
+          taosMemoryFreeClear(*(sample_labels + j));
+        }
+
+        taosMemoryFreeClear(sample_labels);
+        taosMemoryFreeClear(labels);
+      }
+    }
+    
+  }
+
+  code = 0;
+
+_OVER:
+  if(pJson != NULL){
+    tjsonDelete(pJson);
+    pJson = NULL;
+  }
+
+  tFreeSStatisReq(&statisReq);
+  return code;
+  */
+}
+
 static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
   SMnode    *pMnode = pReq->info.node;
   SStatusReq statusReq = {0};

source/dnode/mnode/impl/src/mndIndex.c

@@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
   return mndAcquireDb(pMnode, db);
 }
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
 
   return 0;
@@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx)
   return 0;
 }
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -482,10 +482,10 @@ _OVER:
   return code;
 }
 
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
 
   mndTransSetSerial(pTrans);
 
-  if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
@@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
   mndTransSetSerial(pTrans);
-  if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndMain.c

@@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
 
   int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
   if (code != 0) {
-    mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
+    mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
+            terrstr(), code);
   }
 
   return code;
@@ -911,11 +912,14 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
     if (pObj->id == pMnode->selfDnodeId) {
       pClusterInfo->first_ep_dnode_id = pObj->id;
       tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
-      pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+      //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+      pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
       // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
       tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));
+      desc.syncState = TAOS_SYNC_STATE_LEADER;
     } else {
       tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role));
+      desc.syncState = pObj->syncState;
     }
     taosArrayPush(pClusterInfo->mnodes, &desc);
     sdbRelease(pSdb, pObj);
@@ -946,6 +950,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
       SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
       pVnDesc->dnode_id = pVgid->dnodeId;
       tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role));
+      pVnDesc->syncState = pVgid->syncState;
       if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
         tstrncpy(desc.status, "ready", sizeof(desc.status));
         pClusterInfo->vgroups_alive++;
@@ -985,8 +990,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
   pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000;
   pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
   if (pMnode->grant.expireTimeMS == 0) {
-    pGrantInfo->expire_time = INT32_MAX;
-    pGrantInfo->timeseries_total = INT32_MAX;
+    pGrantInfo->expire_time = 0;
+    pGrantInfo->timeseries_total = 0;
   }
 
   mndReleaseRpc(pMnode);

source/dnode/mnode/impl/src/mndQnode.c

@@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
   if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
@@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
   if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -44,9 +44,8 @@ static bool hasCountWindowNode(SPhysiNode* pNode) {
   }
 }
 
-static bool countWindowStreamTask(SSubplan* pPlan) {
-  SPhysiNode* pNode = pPlan->pNode;
-  return hasCountWindowNode(pNode);
+static bool isCountWindowStreamTask(SSubplan* pPlan) {
+  return hasCountWindowNode((SPhysiNode*)pPlan->pNode);
 }
 
 int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
@@ -301,15 +300,16 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet*
 }
 
 static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
-  for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+  int32_t size = (int32_t) taosArrayGetSize(pList);
+  for (int32_t i = 0; i < size; ++i) {
     SVgroupVer* pVer = taosArrayGet(pList, i);
     if (pVer->vgId == vgId) {
       return pVer->ver;
     }
   }
 
-  mError("failed to find the vgId:%d for extract last version", vgId);
-  return -1;
+  mDebug("no data in vgId:%d for extract last version, set to be 0, total existed vgs:%d", vgId, size);
+  return 1;
 }
 
 static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) {
@@ -342,13 +342,13 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
   }
 }
 
-static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan) {
-  bool hasCountWindowNode = countWindowStreamTask(pPlan);
-  bool isRelStreamTask = (pTask->hTaskInfo.id.taskId != 0);
-  if (hasCountWindowNode && isRelStreamTask) {
+static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFillhistoryTask) {
+  bool hasCountWindowNode = isCountWindowStreamTask(pPlan);
+
+  if (hasCountWindowNode && (!isFillhistoryTask)) {
     SStreamStatus* pStatus = &pTask->status;
-    mDebug("s-task:0x%x status is set to %s from %s for count window agg task with fill-history option set",
-           pTask->id.taskId, streamTaskGetStatusStr(pStatus->taskStatus), streamTaskGetStatusStr(TASK_STATUS__HALT));
+    mDebug("s-task:0x%x status set %s from %s for count window agg task with fill-history option set",
+           pTask->id.taskId, streamTaskGetStatusStr(TASK_STATUS__HALT), streamTaskGetStatusStr(pStatus->taskStatus));
     pStatus->taskStatus = TASK_STATUS__HALT;
   }
 }
@@ -398,15 +398,17 @@ static void setHTasksId(SStreamObj* pStream) {
 
 static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
                                SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
-  // new stream task
   SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return terrno;
   }
+
   mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
 
-  haltInitialTaskStatus(pTask, plan);
+  if (pStream->conf.fillHistory) {
+    haltInitialTaskStatus(pTask, plan, isFillhistory);
+  }
 
   streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
 
@@ -415,6 +417,7 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
     terrno = code;
     return terrno;
   }
+
   return TDB_CODE_SUCCESS;
 }
 
@@ -470,6 +473,9 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
     int code =
         doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
     if (code != 0) {
+      mError("create stream task, code:%s", tstrerror(code));
+
+      // todo drop the added source tasks.
       sdbRelease(pSdb, pVgroup);
       return code;
     }

source/dnode/mnode/impl/src/mndSnode.c

@@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
 
@@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
   if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndStb.c

@@ -617,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
   return 0;
 }
 
-static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -629,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
-  SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
-  if (pUndoRaw == NULL) return -1;
-  if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
-    sdbFreeRaw(pUndoRaw);
-    return -1;
-  }
-  if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
-
-  return 0;
-}
-
 static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
   if (pCommitRaw == NULL) return -1;
@@ -913,8 +901,6 @@ _OVER:
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1;
-  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
-  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
@@ -1746,10 +1732,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
   return 0;
 }
 
-static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2155,7 +2141,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb
     mndTransSetRpcRsp(pTrans, pCont, contLen);
   }
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -2192,11 +2178,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
     if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) {
       exist = true;
     }
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
-      if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2215,13 +2201,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
       exist = true;
     }
 
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
       memcpy(idxObj.colName, nTagName, strlen(nTagName));
       idxObj.colName[strlen(nTagName)] = 0;
-      if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2354,10 +2340,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
+static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2427,7 +2413,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
+  if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
@@ -3547,7 +3533,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndStream.c

@@ -29,7 +29,7 @@
 
 #define MND_STREAM_MAX_NUM 60
 
-typedef struct SMStreamNodeCheckMsg {
+typedef struct {
   int8_t placeHolder;  // // to fix windows compile error, define place holder
 } SMStreamNodeCheckMsg;
 
@@ -721,6 +721,13 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  // add into buffer firstly
+  // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
+  taosThreadMutexLock(&execInfo.lock);
+  mDebug("stream stream:%s tasks register into node list", createReq.name);
+  saveStreamTasksInfo(&streamObj, &execInfo);
+  taosThreadMutexUnlock(&execInfo.lock);
+
   // execute creation
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -730,12 +737,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   mndTransDrop(pTrans);
 
-  taosThreadMutexLock(&execInfo.lock);
-
-  mDebug("stream tasks register into node list");
-  saveStreamTasksInfo(&streamObj, &execInfo);
-  taosThreadMutexUnlock(&execInfo.lock);
-
   SName dbname = {0};
   tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
@@ -878,7 +879,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
   int64_t ts = taosGetTimestampMs();
   if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) {
     //    mWarn("checkpoint interval less than the threshold, ignore it");
-    return -1;
+    return TSDB_CODE_SUCCESS;
   }
 
   bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
@@ -2180,5 +2181,16 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
   mndReleaseStream(pMnode, pStream);
   taosThreadMutexUnlock(&execInfo.lock);
 
+  {
+    SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
+    rsp.pCont = rpcMallocCont(rsp.contLen);
+    SMsgHead* pHead = rsp.pCont;
+    pHead->vgId = htonl(req.nodeId);
+
+    tmsgSendRsp(&rsp);
+
+    pReq->info.handle = NULL;   // disable auto rsp
+  }
+
   return 0;
 }

source/dnode/mnode/impl/src/mndStreamHb.c

@@ -65,7 +65,7 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI
   taosArrayPush(pList, pInfo);
 }
 
-int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
+int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
   STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint");
   if (pTrans == NULL) {
     return terrno;
@@ -115,7 +115,7 @@ static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, in
     } else {
       mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name,
              pStream->uid, transId);
-      code = createStreamResetStatusTrans(pMnode, pStream);
+      code = mndCreateStreamResetStatusTrans(pMnode, pStream);
     }
   }
 
@@ -222,11 +222,11 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
-  SArray      *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
-  SArray      *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask));
+  SArray      *pFailedTasks = NULL;
+  SArray      *pOrphanTasks = NULL;
 
-  if(grantCheckExpire(TSDB_GRANT_STREAMS) < 0){
-    if(suspendAllStreams(pMnode, &pReq->info) < 0){
+  if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
+    if (suspendAllStreams(pMnode, &pReq->info) < 0) {
       return -1;
     }
   }
@@ -244,6 +244,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
 
   mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
 
+  pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
+  pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask));
+
   taosThreadMutexLock(&execInfo.lock);
 
   // extract stream task list
@@ -349,5 +352,16 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   taosArrayDestroy(pFailedTasks);
   taosArrayDestroy(pOrphanTasks);
 
+  {
+    SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)};
+    rsp.pCont = rpcMallocCont(rsp.contLen);
+    SMsgHead* pHead = rsp.pCont;
+    pHead->vgId = htonl(req.vgId);
+
+    tmsgSendRsp(&rsp);
+
+    pReq->info.handle = NULL;   // disable auto rsp
+  }
+
   return TSDB_CODE_SUCCESS;
 }

source/dnode/mnode/impl/src/mndStreamTrans.c

@@ -261,22 +261,30 @@ int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msg
   return mndTransAppendRedoAction(pTrans, &action);
 }
 
+static bool identicalName(const char* pDb, const char* pParam, int32_t len) {
+  return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0);
+}
+
 int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) {
-  // data in the hash table will be removed automatically, no need to remove it here.
-  SStreamTransInfo *pTransInfo = taosHashGet(execInfo.transMgmt.pDBTrans, pDBName, len);
-  if (pTransInfo == NULL) {
-    return TSDB_CODE_SUCCESS;
-  }
+  void *pIter = NULL;
 
-  // not checkpoint trans, ignore
-  if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) {
-    mDebug("not checkpoint trans, not kill it, name:%s, transId:%d", pTransInfo->name, pTransInfo->transId);
-    return TSDB_CODE_SUCCESS;
-  }
+  while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
+    SStreamTransInfo *pTransInfo = (SStreamTransInfo *)pIter;
+    if (strcmp(pTransInfo->name, MND_STREAM_CHECKPOINT_NAME) != 0) {
+      continue;
+    }
 
-  char *pDupDBName = strndup(pDBName, len);
-  mndKillTransImpl(pMnode, pTransInfo->transId, pDupDBName);
-  taosMemoryFree(pDupDBName);
+    SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId);
+    if (pStream != NULL) {
+      if (identicalName(pStream->sourceDb, pDBName, len)) {
+        mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb);
+      } else if (identicalName(pStream->targetDb, pDBName, len)) {
+        mndKillTransImpl(pMnode, pTransInfo->transId, pStream->targetDb);
+      }
+
+      mndReleaseStream(pMnode, pStream);
+    }
+  }
 
   return TSDB_CODE_SUCCESS;
 }

source/dnode/mnode/impl/src/mndStreamUtil.c

@@ -231,6 +231,8 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
     taosMemoryFree(pReq);
     return -1;
   }
+
+  mDebug("set the resume action for trans:%d", pTrans->id);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndSync.c

@@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
 static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
   SSdbRaw *pRaw = pAction->pRaw;
   SSdb    *pSdb = pMnode->pSdb;
-  SSdbRow *pRow = NULL;
-  void    *pObj = NULL;
-  int      code = -1;
+  int      code = 0;
 
   if (pRaw->status != SDB_STATUS_CREATING) goto _OUT;
 
-  pRow = (pSdb->decodeFps[pRaw->type])(pRaw);
-  if (pRow == NULL) goto _OUT;
-  pObj = sdbGetRowObj(pRow);
-  if (pObj == NULL) goto _OUT;
-
   SdbValidateFp validateFp = pSdb->validateFps[pRaw->type];
-  code = 0;
   if (validateFp) {
-    code = validateFp(pMnode, pTrans, pObj);
+    code = validateFp(pMnode, pTrans, pRaw);
   }
 
 _OUT:
-  if (pRow) {
-    SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
-    if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
-    taosMemoryFreeClear(pRow);
-  }
   return code;
 }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
         pTrans->createdTime);
   int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id);
   if (code != 0) {
-    mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
-           tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId);
+    mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
+           code, pTrans->createdTime, pMnode->syncMgmt.transId);
     sdbFreeRaw(pRaw);
     return -1;
   }
@@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
 
 static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
   if (pAction->rawWritten) return 0;
-  if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+  if (topHalf) {
+    terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH;
+    return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+  }
 
   int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw);
   if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@@ -1214,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
   if (numOfActions == 0) return 0;
 
   if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) {
-    return -1;
+    return code;
   }
 
   int32_t       numOfExecuted = 0;
@@ -1419,7 +1422,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool
     pTrans->stage = TRN_STAGE_COMMIT;
     mInfo("trans:%d, stage from redoAction to commit", pTrans->id);
     continueExec = true;
-  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) {
     mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code));
     continueExec = false;
   } else {

source/dnode/mnode/impl/src/mndUser.c

@@ -2484,7 +2484,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
-      char objName[20] = {0};
+      char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);

source/dnode/mnode/impl/src/mndVgroup.c

@@ -31,10 +31,10 @@
 #define VGROUP_VER_NUMBER   1
 #define VGROUP_RESERVE_SIZE 64
 
-static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
+static int32_t  mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@@ -181,15 +181,28 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
-  SVgObj *pVgroup = pObj;
+static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  SVgObj  *pVgroup = NULL;
+  int      code = -1;
+
+  pRow = mndVgroupActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  pVgroup = sdbGetRowObj(pRow);
+  if (pVgroup == NULL) goto _OVER;
 
   int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
   if (maxVgId > pVgroup->vgId) {
     mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId);
-    return -1;
+    goto _OVER;
   }
-  return 0;
+
+  code = 0;
+_OVER:
+  if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup);
+  taosMemoryFreeClear(pRow);
+  return code;
 }
 
 static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) {

source/dnode/mnode/sdb/inc/sdb.h

@@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
 typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
-typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj);
+typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
 typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3);