Merge remote-tracking branch 'origin/3.0' into feat/TD-27337

docs/zh/07-develop/07-tmq.mdx

@@ -15,11 +15,11 @@ import Node from "./_sub_node.mdx";
 import CSharp from "./_sub_cs.mdx";
 import CDemo from "./_sub_c.mdx";
 
-
 为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine 提供了类似 kafka 的数据订阅功能。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。
 
-# 介绍
+## 数据订阅介绍
-## 主题
+
+### 主题
 与 kafka 一样，你需要定义 topic, TDengine 的 topic 有三种，可以是数据库，超级表，或者一个 `SELECT` 语句，具体的语法参见 [CREATE TOPIC](../../taos-sql/tmq)。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。
 
 如下图，每个 topic 涉及到的数据表可能分布在多个 vnode（相当于 kafka 里的 partition） 上，每个 vnode 上的数据保存在 WAL(Write-Ahead-Log) 文件中，WAL 文件里的数据是顺序写入的（由于 WAL 文件中存储的不只有数据，还有元数据，写入消息等，所以数据的版本号不是连续的）。
@@ -30,11 +30,12 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
 
 对于 `SELECT` 语句形式的 topic，在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。
 
-## 生产者
+### 生产者
 写入 topic 相关联的数据表中数据的都是生产者，生产者实际生产的数据写入到了子表或普通表中，即表所在 vnode 的 WAL 里。
 
-## 消费者
+### 消费者
-### 消费者组
+
+#### 消费者组
 消费者订阅 topic 后，可以消费 topic 里的所有数据（这些数据所在的表可能分布在多个 vnode 上，即 db 所在的所有 vnode）。订阅 topic 时，需要指定一个消费者组 (consumer group)，如果这个消费者组里只有一个消费者，那么这个消费者会顺序的消费这些 vnode 上的数据。
 
 为了提高消费速度，便于多线程、分布式地消费数据，可以在一个消费组里添加多个消费者，这些消费者将均分数据所在的 vnode 进行消费（比如数据分布在 4 个 vnode 上，有 2 个消费者的话，那么每个消费者消费 2 个 vnode；有 3 个消费者的话，2 个消费者各消费 1 个 vnode，1 个消费者消费 2 个 vnode；有 5 个消费者的话，4 个各分配 1 个 vnode 消费，另外 1 个不消费），如下图：
@@ -44,7 +45,8 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
 在一个消费组里添加一个消费者后，在 Mnode 上通过 rebalance 的机制实现消费者的重新分配，该操作对用户是透明的。
 
 一个消费者可以订阅多个 topic。TDengine 的数据订阅在宕机、重启等复杂环境下确保 at least once 消费。
-### 消费进度
+
+#### 消费进度
 在 topic 的一个消费组的一个 vnode 上有消费进度。消费者消费的同时，可以提交消费进度，消费进度即 vnode 上 WAL 的版本号（对于 kafka 里的 offset），消费进度可以手动提交，也可以通过参数（auto.commit.interval.ms）设置为周期性自动提交。
 
 首次消费数据时通过订阅参数（auto.offset.reset）来确定消费位置为最新数据（latest）还是最旧数据（earliest）。
@@ -59,16 +61,16 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
 
 作为一个数据库产品， WAL 文件中存储的不全是数据，也包括其他写入消息，元数据等，所以消费进度不是连续的。
 
-##说明
+### 说明
 从3.2.0.0版本开始，数据订阅支持vnode迁移和分裂。
 
 由于数据订阅依赖wal文件，而在vnode迁移和分裂的过程中，wal并不会同步过去，所以迁移或分裂后，之前没消费完的wal数据后消费不到。所以请保证迁移和分裂之前把数据全部消费完后，再进行vnode迁移或分裂，否则，消费会丢失数据。
 
-# 语法说明
+## 数据订阅语法说明
 
 具体的语法参见 [数据订阅](../../taos-sql/tmq)
 
-# 消费参数
+## 数据订阅相关参数
 
 消费参数主要用于消费者创建时指定，基础配置项如下表所示：
 
@@ -86,9 +88,9 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
 |     `msg.with.table.name`      | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true）               |默认关闭 |
 |     `enable.replay`            | boolean | 是否开启数据回放功能               |默认关闭 |
 
-# 主要数据结构和 API 接口
+## 数据订阅主要 API 接口
 
-不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
+不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节数据订阅部分，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
 
 <Tabs defaultValue="java" groupId="lang">
 <TabItem value="c" label="C">
@@ -310,8 +312,8 @@ void Close()
 </TabItem>
 </Tabs>
 
-# 数据订阅示例
+## 数据订阅示例
-## 写入数据
+### 写入数据
 
 首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
 
@@ -324,7 +326,7 @@ CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
 INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
 INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
 ```
-## 创建 topic
+### 创建 topic
 
 使用 SQL 创建一个 topic：
 
@@ -332,7 +334,7 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
 CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
 ```
 
-## 创建消费者 *consumer*
+### 创建消费者 consumer
 
 对于不同编程语言，其设置方式如下：
 
@@ -499,7 +501,7 @@ var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
     
 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
     
-## 订阅 *topics*
+### 订阅 topics
 
 一个 consumer 支持同时订阅多个 topic。
     
@@ -578,7 +580,7 @@ consumer.Subscribe(topics);
 
 </Tabs>
 
-## 消费
+### 消费
 
 以下代码展示了不同语言下如何对 TMQ 消息进行消费。
 
@@ -714,7 +716,7 @@ while (true)
 
 </Tabs>
 
-## 结束消费
+### 结束消费
 
 消费结束后，应当取消订阅。
 
@@ -795,7 +797,7 @@ consumer.Close();
 
 </Tabs>
 
-## 完整示例代码
+### 完整示例代码
 
 以下是各语言的完整示例代码。
 
@@ -838,8 +840,8 @@ consumer.Close();
 
 </Tabs>
 
-#订阅高级功能
+## 数据订阅高级功能
-##数据回放
+### 数据回放
 - 订阅支持 replay 功能，按照数据写入的时间回放。
   比如，如下时间写入三条数据
   ```sql

docs/zh/07-develop/_sub_java.mdx

@@ -6,3 +6,4 @@
 ```
 ```java
 {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+```

include/common/tdatablock.h

@@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo {
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
+#define BLOCK_VERSION_1          1
+#define BLOCK_VERSION_2          2
+
 #define NBIT                     (3u)
 #define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
 #define CharPos(r_)              ((r_) >> NBIT)
@@ -272,7 +275,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
 
 bool  alreadyAddGroupId(char* ctbName);
 bool  isAutoTableName(char* ctbName);
-void  buildCtbNameAddGruopId(char* ctbName, uint64_t groupId);
+void  buildCtbNameAddGroupId(char* ctbName, uint64_t groupId);
 char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
 int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);
 

include/common/tglobal.h

@@ -106,6 +106,9 @@ extern char     tsMonitorFqdn[];
 extern uint16_t tsMonitorPort;
 extern int32_t  tsMonitorMaxLogs;
 extern bool     tsMonitorComp;
+extern bool     tsMonitorLogProtocol;
+extern int32_t  tsMonitorIntervalForBasic;
+extern bool     tsMonitorForceV2;
 
 // audit
 extern bool     tsEnableAudit;
@@ -216,6 +219,7 @@ extern bool    tsFilterScalarMode;
 extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
 extern int32_t tsResolveFQDNRetryTime;
+extern bool    tsDisableCount;
 
 extern bool tsExperimental;
 // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

include/common/tgrant.h

@@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant);
 
 // #ifndef GRANTS_CFG
 #ifdef TD_ENTERPRISE
-#define GRANTS_SCHEMA                                                                             \
+#define GRANTS_SCHEMA                                                                                              \
-  static const SSysDbTableSchema grantsSchema[] = {                                               \
+  static const SSysDbTableSchema grantsSchema[] = {                                                                \
-      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},  \
+      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},  \
-      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
+      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
-      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},         \
+      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},         \
-      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},   \
+      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},   \
-      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    \
+      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},    \
   }
 #else
-#define GRANTS_SCHEMA                                                                             \
+#define GRANTS_SCHEMA                                                                                              \
-  static const SSysDbTableSchema grantsSchema[] = {                                               \
+  static const SSysDbTableSchema grantsSchema[] = {                                                                \
-      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},  \
+      {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},  \
-      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
+      {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
-      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},         \
+      {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},         \
-      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},   \
+      {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},   \
-      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},       \
+      {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},       \
-      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},    \
+      {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},    \
   }
 #endif
 // #define GRANT_CFG_ADD

include/common/tmsg.h

@@ -207,9 +207,6 @@ typedef enum _mgmt_table {
 #define TD_CHILD_TABLE  TSDB_CHILD_TABLE
 #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
 
-#define TD_REQ_FROM_APP  0
-#define TD_REQ_FROM_TAOX 1
-
 typedef enum ENodeType {
   // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
   // VALUE, OPERATOR, FUNCTION and so on.
@@ -587,7 +584,7 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void    tFreeSSubmitBlkRsp(void* param);
+// void    tFreeSSubmitBlkRsp(void* param);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON     ((int8_t)0x1)
@@ -760,7 +757,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   int8_t   igExists;
-  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
   int8_t   reserved[6];
   tb_uid_t suid;
   int64_t  delay1;
@@ -803,7 +800,7 @@ void    tFreeSMCreateStbRsp(SMCreateStbRsp* pRsp);
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   int8_t   igNotExists;
-  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
   int8_t   reserved[6];
   tb_uid_t suid;
   int32_t  sqlLen;
@@ -1328,14 +1325,13 @@ typedef struct {
 } SDnodeListReq;
 
 int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
-int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
 
 typedef struct {
   int32_t useless;  // useless
 } SServerVerReq;
 
 int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
-int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
+// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
 
 typedef struct {
   char ver[TSDB_VERSION_LEN];
@@ -1586,6 +1582,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
 int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
 void    tFreeSStatusReq(SStatusReq* pReq);
 
+typedef struct {
+  int32_t     contLen;
+  char*       pCont;
+} SStatisReq;
+
+int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+void tFreeSStatisReq(SStatisReq *pReq);
+
 typedef struct {
   int32_t dnodeId;
   int64_t clusterId;
@@ -1614,7 +1619,6 @@ typedef struct {
   SEp     ep;
   char    active[TSDB_ACTIVE_KEY_LEN];
   char    connActive[TSDB_CONN_ACTIVE_KEY_LEN];
-  char    machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeInfo;
 
 typedef struct {
@@ -1634,14 +1638,14 @@ typedef struct {
 } SMTimerReq;
 
 int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
-int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
+// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
 
 typedef struct {
   int64_t tick;
 } SMStreamTickReq;
 
 int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
-int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
+// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
 
 typedef struct {
   int32_t  id;
@@ -1912,7 +1916,7 @@ typedef struct {
 } SShowVariablesReq;
 
 int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
-int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
+// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
 
 typedef struct {
   char name[TSDB_CONFIG_OPTION_LEN + 1];
@@ -1942,7 +1946,7 @@ typedef struct {
 } SShowReq;
 
 int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
-int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
+// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
 void    tFreeSShowReq(SShowReq* pReq);
 
 typedef struct {
@@ -1950,9 +1954,9 @@ typedef struct {
   STableMetaRsp tableMeta;
 } SShowRsp, SVShowTablesRsp;
 
-int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-void    tFreeSShowRsp(SShowRsp* pRsp);
+// void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
@@ -2210,7 +2214,7 @@ typedef struct {
 } SForceBecomeFollowerReq;
 
 int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
-int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
+// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
 
 typedef struct {
   int32_t vgId;
@@ -2227,8 +2231,8 @@ typedef struct {
   char ckey[TSDB_PASSWORD_LEN];
 } SAuthReq, SAuthRsp;
 
-int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
-int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
 
 typedef struct {
   int32_t statusCode;
@@ -2662,6 +2666,7 @@ typedef struct SVCreateStbReq {
   SRSmaParam     rsmaParam;
   int32_t        alterOriDataLen;
   void*          alterOriData;
+  int8_t         source;
 } SVCreateStbReq;
 
 int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@@ -2731,6 +2736,7 @@ typedef struct {
     SVCreateTbReq* pReqs;
     SArray*        pArray;
   };
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
 } SVCreateTbBatchReq;
 
 int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@@ -2759,8 +2765,8 @@ typedef struct {
 int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp);
 int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp);
 
-int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
-int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
 
 // TDMT_VND_DROP_TABLE =================
 typedef struct {
@@ -2823,6 +2829,7 @@ typedef struct {
   int32_t newCommentLen;
   char*   newComment;
   int64_t ctimeMs;  // fill by vnode
+  int8_t  source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@@ -3214,8 +3221,8 @@ typedef struct {
   int32_t hasCheckPoint;
 } SMVStreamGatherInfoReq;
 
-int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
+// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
-int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
+// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
 
 typedef struct {
   int64_t leftForVer;
@@ -3492,7 +3499,7 @@ void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
 int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
 int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
 int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
-int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
+// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
 
 typedef struct {
   int32_t number;
@@ -3920,12 +3927,13 @@ int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp);
 int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 
+#define TD_REQ_FROM_APP               0x0
 #define SUBMIT_REQ_AUTO_CREATE_TABLE  0x1
 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
 #define SUBMIT_REQ_FROM_FILE          0x4
+#define TD_REQ_FROM_TAOX              0x8
 
-#define SOURCE_NULL  0
+#define TD_REQ_FROM_TAOX_OLD          0x1     // for compatibility
-#define SOURCE_TAOSX 1
 
 typedef struct {
   int32_t        flags;
@@ -3938,7 +3946,6 @@ typedef struct {
     SArray* aCol;
   };
   int64_t ctimeMs;
-  int8_t  source;
 } SSubmitTbData;
 
 typedef struct {

include/common/tmsgdef.h

@@ -215,6 +215,7 @@
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)

include/common/trow.h

@@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
  */
 
 static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
-  pBuilder->rowType = TD_ROW_TP;  // default STpRow
+  pBuilder->rowType = pBuilder->rowType;
   pBuilder->sver = sver;
 }
 int32_t                  tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);
@@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow);
 bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
 bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal);
 
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow);
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType);
 bool    tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
 void    tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag);
 

include/dnode/vnode/tqCommon.h

@@ -27,6 +27,8 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
 int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
 int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
                                      bool isLeader, bool restored);
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);

include/libs/monitor/clientMonitor.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_CLIENT_MONITOR_H
+#define TDENGINE_CLIENT_MONITOR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taos_monitor.h"
+#include "thash.h"
+#include "query.h"
+
+typedef enum SQL_RESULT_CODE {
+  SQL_RESULT_SUCCESS = 0,
+  SQL_RESULT_FAILED = 1,
+  SQL_RESULT_CANCEL = 2,
+} SQL_RESULT_CODE;
+
+const char* resultStr(SQL_RESULT_CODE code);
+
+typedef struct {
+  char                       clusterKey[512];
+  SEpSet                     epSet;
+  void*                      pTransporter;
+  taos_collector_registry_t* registry;
+  taos_collector_t*          colector;
+  SHashObj*                  counters;
+} ClientMonitor;
+
+void            clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter);
+void            clusterMonitorClose(const char* clusterKey);
+taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
+                                     const char** label_keys);
+int             taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values);
+void            cluster_monitor_stop();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_CLIENT_MONITOR_H

include/libs/monitor/monitor.h

@@ -76,7 +76,8 @@ typedef struct {
 } SMonBasicInfo;
 
 typedef struct {
-  float        uptime;  // day
+  //float        uptime;  // day
+  int64_t      uptime;  // second
   int8_t       has_mnode;
   int8_t       has_qnode;
   int8_t       has_snode;
@@ -100,13 +101,15 @@ typedef struct {
   int32_t mnode_id;
   char    mnode_ep[TSDB_EP_LEN];
   char    role[MON_ROLE_LEN];
+  int32_t syncState;
 } SMonMnodeDesc;
 
 typedef struct {
   char    first_ep[TSDB_EP_LEN];
   int32_t first_ep_dnode_id;
   char    version[MON_VER_LEN];
-  float   master_uptime;     // day
+  //float   master_uptime;     // day
+  int64_t master_uptime;        //second
   int32_t monitor_interval;  // sec
   int32_t dbs_total;
   int32_t stbs_total;
@@ -125,6 +128,7 @@ typedef struct {
 typedef struct {
   int32_t dnode_id;
   char    vnode_role[MON_ROLE_LEN];
+  int32_t syncState;
 } SMonVnodeDesc;
 
 typedef struct {
@@ -149,7 +153,7 @@ typedef struct {
 } SMonStbInfo;
 
 typedef struct {
-  uint32_t expire_time;
+  int64_t  expire_time;
   int64_t  timeseries_used;
   int64_t  timeseries_total;
 } SMonGrantInfo;
@@ -221,7 +225,9 @@ void    monSetVmInfo(SMonVmInfo *pInfo);
 void    monSetQmInfo(SMonQmInfo *pInfo);
 void    monSetSmInfo(SMonSmInfo *pInfo);
 void    monSetBmInfo(SMonBmInfo *pInfo);
-void    monSendReport();
+void    monGenAndSendReport();
+void    monGenAndSendReportBasic();
+void    monSendContent(char *pCont);
 
 void tFreeSMonMmInfo(SMonMmInfo *pInfo);
 void tFreeSMonVmInfo(SMonVmInfo *pInfo);

include/libs/monitorfw/taos_alloc.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_alloc.h
+ * @brief memory management
+ */
+
+#ifndef TAOS_ALLOC_H
+#define TAOS_ALLOC_H
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is malloc.
+ */
+#define taos_malloc malloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is realloc.
+ */
+#define taos_realloc realloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is strdup.
+ */
+#define taos_strdup strdup
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is free.
+ */
+#define taos_free free
+
+#endif  // TAOS_ALLOC_H

include/libs/monitorfw/taos_collector.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_COLLECTOR_H
+#define TAOS_COLLECTOR_H
+
+#include "taos_map.h"
+#include "taos_metric.h"
+
+/**
+ * @file taos_collector.h
+ * @brief A Prometheus collector returns a collection of metrics
+ */
+
+/**
+ * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is
+ * registered.
+ */
+typedef struct taos_collector taos_collector_t;
+
+/**
+ * @brief The function responsible for preparing metric data and returning metrics for a given collector.
+ *
+ * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may
+ * set this function on your collector to do additional work before returning the contained metrics.
+ *
+ * @param self The target taos_collector_t*
+ * @return The taos_map_t* containing the collected metrics
+ */
+typedef taos_map_t *taos_collect_fn(taos_collector_t *self);
+
+/**
+ * @brief Create a collector
+ * @param name The name of the collector. The name MUST NOT be default or process.
+ * @return The constructed taos_collector_t*
+ */
+taos_collector_t *taos_collector_new(const char *name);
+
+/**
+ * @brief Destroy a collector. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy(taos_collector_t *self);
+
+/**
+ * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ */
+void taos_collector_free_generic(void *gen);
+
+/**
+ * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy_generic(void *gen);
+
+/**
+ * @brief Add a metric to a collector
+ * @param self The target taos_collector_t*
+ * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric);
+
+int taos_collector_remove_metric(taos_collector_t *self, const char* key);
+
+taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name);
+
+/**
+ * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them
+ *        for metric exposition.
+ * @param self The target taos_collector_t*
+ * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before
+ *           returning the collected metrics for exposition.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn);
+
+#endif  // TAOS_COLLECTOR_H

include/libs/monitorfw/taos_collector_registry.h

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_collector_registry.h
+ * @brief The collector registry registers collectors for metric exposition.
+ */
+
+#ifndef TAOS_REGISTRY_H
+#define TAOS_REGISTRY_H
+
+#include "taos_collector.h"
+#include "taos_metric.h"
+
+/**
+ * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format
+ */
+typedef struct taos_collector_registry taos_collector_registry_t;
+
+/**
+ * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT
+ * modify this value.
+ */
+extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT;
+
+/**
+ * @brief Initializes the default collector registry and enables metric collection on the executing process
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_default_init(void);
+
+/**
+ * @brief Constructs a taos_collector_registry_t*
+ * @param name The name of the collector registry. It MUST NOT be default.
+ * @return The constructed taos_collector_registry_t*
+ */
+taos_collector_registry_t *taos_collector_registry_new(const char *name);
+
+/**
+ * @brief Destroy a collector registry. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_destroy(taos_collector_registry_t *self);
+
+/**
+ * @brief Enable process metrics on the given collector registry
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY
+ *
+ * The metric to be registered MUST NOT already be registered with the given . If so, the program will
+ * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore,
+ * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this
+ * function. The metric will be added to the default registry's default collector.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return The registered taos_metric_t*
+ */
+taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer
+ * value on failure.
+ *
+ * See taos_collector_registry_must_register_metric.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_metric(taos_metric_t *metric);
+
+int taos_collector_registry_deregister_metric(const char *key);
+
+taos_metric_t *taos_collector_registry_get_metric(char* metric_name);
+
+/**
+ * @brief Register a collector with the given registry. Returns a non-zero integer value on failure.
+ * @param self The target taos_collector_registry_t*
+ * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector);
+
+/**
+ * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap
+ * memory growth.
+ *
+ * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/
+ *
+ * @param self The target taos_collector_registry_t*
+ * @return The string int he default metric exposition format.
+ */
+const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format);
+
+int taos_collector_registry_clear_batch(taos_collector_registry_t *self);
+
+const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str);
+
+/**
+ *@brief Validates that the given metric name complies with the specification:
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ *
+ * Returns a non-zero integer value on failure.
+ *
+ * @param self The target taos_collector_registry_t*
+ * @param metric_name The metric name to validate
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name);
+
+#endif  // TAOS_H

include/libs/monitorfw/taos_counter.h

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_COUNTER_H
+#define TAOS_COUNTER_H
+
+#include <stdlib.h>
+
+#include "taos_metric.h"
+
+/**
+ * @file taos_counter.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+
+/**
+ * @brief A prometheus counter.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+typedef taos_metric_t taos_counter_t;
+
+/**
+ * @brief Construct a taos_counter_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ *                        require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ *                   no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ *                   literal.
+ * @return The constructed taos_counter_t*
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" });
+ *
+ *     // An example without labels
+ *     taos_counter_new("foo", "foo is a counter without labels", 0, NULL);
+ */
+taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ *        returned on failure.
+ * @param self A taos_counter_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_counter_destroy(taos_counter_t *self);
+
+/**
+ * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure.
+ * @param self The target  taos_counter_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_inc(foo_counter, (const char**) { "bar", "bang" });
+ **
+ *     // An example without labels
+ *     taos_counter_inc(foo_counter, NULL);
+ */
+int taos_counter_inc(taos_counter_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure.
+ * @param self The target  taos_counter_t*
+ * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_counter_add(foo_counter, 22, NULL);
+ */
+int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values);
+
+#endif  // TAOS_COUNTER_H

include/libs/monitorfw/taos_gauge.h

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_gauge.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+
+#ifndef TAOS_GAUGE_H
+#define TAOS_GAUGE_H
+
+#include <stdlib.h>
+
+#include "taos_metric.h"
+
+/**
+ * @brief A prometheus gauge.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+typedef taos_metric_t taos_gauge_t;
+
+/**
+ * @brief Constructs a taos_gauge_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ *                        require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ *                   no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ *                   literal.
+ * @return The constructed taos_guage_t*
+ *
+ *     // An example with labels
+ *     taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" });
+ *
+ *     // An example without labels
+ *     taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL);
+ */
+taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ *        returned on failure.
+ * @param self The target taos_gauge_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_gauge_destroy(taos_gauge_t *self);
+
+/**
+ * @brief Increment the taos_gauge_t* by 1.
+ * @param self The target  taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_inc(foo_gauge, NULL);
+ */
+int taos_gauge_inc(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Decrement the taos_gauge_t* by 1.
+ * @param self The target  taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_dec(foo_gauge, NULL);
+ */
+int taos_gauge_dec(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_gauge_t*.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_add(foo_gauge, 22, NULL);
+ */
+int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_sub(foo_gauge, 22, NULL);
+ */
+int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Set the value for the taos_gauge_t*
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to which the taos_gauge_t* passed as self will be set
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the gauge's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ *     // An example with labels
+ *     taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ *     // An example without labels
+ *     taos_gauge_set(foo_gauge, 22, NULL);
+ */
+int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values);
+
+#endif  // TAOS_GAUGE_H

include/libs/monitorfw/taos_linked_list.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_LIST_H
+#define TAOS_LIST_H
+
+#include <stdlib.h>
+
+struct taos_linked_list;
+/**
+ * @brief Provides a generic linked list
+ */
+typedef struct taos_linked_list taos_linked_list_t;
+
+#endif  // TAOS_LIST_H

include/libs/monitorfw/taos_map.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TAOS_MAP_H
+#define TAOS_MAP_H
+
+struct taos_map;
+typedef struct taos_map taos_map_t;
+
+struct taos_map_node;
+typedef struct taos_map_node taos_map_node_t;
+
+#endif  // TAOS_MAP_H

include/libs/monitorfw/taos_metric.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_metric.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_METRIC_H
+#define TAOS_METRIC_H
+
+#include "taos_metric_sample.h"
+
+struct taos_metric;
+/**
+ * @brief A prometheus metric.
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model
+ */
+typedef struct taos_metric taos_metric_t;
+
+/**
+ * @brief Returns a taos_metric_sample_t*. The order of label_values is significant.
+ *
+ * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map
+ * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be
+ * preferrable in performance-sensitive situations.
+ *
+ * @param self The target taos_metric_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ *                     match the value passed to label_key_count in the counter's constructor. If no label values are
+ *                     necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A taos_metric_sample_t*
+ */
+taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values);
+
+#endif  // TAOS_METRIC_H

include/libs/monitorfw/taos_metric_sample.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_metric_sample.h
+ * @brief Functions for interfacting with metric samples directly
+ */
+
+#ifndef TAOS_METRIC_SAMPLE_H
+#define TAOS_METRIC_SAMPLE_H
+
+struct taos_metric_sample;
+/**
+ * @brief Contains the specific metric and value given the name and label set
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ */
+typedef struct taos_metric_sample taos_metric_sample_t;
+
+/**
+ * @brief Add the r_value to the sample. The value must be greater than or equal to zero.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to add to taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_add(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Subtract the r_value from the sample.
+ *
+ * This operation MUST be called a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to subtract from the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Set the r_value of the sample.
+ *
+ * This operation MUST be called on a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double which will be set to the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_set(taos_metric_sample_t *self, double r_value);
+
+int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value);
+
+#endif  // TAOS_METRIC_SAMPLE_H

include/libs/monitorfw/taos_monitor.h

@@ -0,0 +1,132 @@
+/*
+Copyright 2019 DigitalOcean Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @file taos_monitor.h
+ * @brief Include taos_monitor.h to include the entire public API
+ * @mainpage Welcome to the documentation site for prometheus-client-c!
+ * @tableofcontents
+ * @section Introduction
+ *
+ * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language.
+ * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics
+ * over HTTP.
+ *
+ * @section Creating-and-Registering-Metrics Creating and Registering Metrics
+ *
+ * prometheus-client-c supports the following metric types:
+ *
+ * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter)
+ * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge)
+ * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)
+ *
+ * To get started using one of the metric types, declare the metric at file scope. For example:
+ *
+ * @code{.c}
+ *
+ * #incldue "taos_monitor.h"
+ *
+ * taos_counter_t *my_counter;
+ *
+ * @endcode
+ *
+ * Next, create a metric initialization function. You can create the metric and register it with the default metric
+ * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning
+ * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later...
+ *
+ * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric
+ * constructor into the taos_collector_registry_must_register_metric function. For example:
+ *
+ * @code{.c}
+ *
+ * void foo_metric_init(void) {
+ *   my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL));
+ * }
+ *
+ * @endcode
+ *
+ * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third
+ * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass
+ * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we
+ * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics
+ * for the file we just created to the default prometheus metric collector registery called
+ * TAOS_COLLECTOR_REGISTRY_DEFAULT
+ *
+ * @section Updating-Metric-Sample-Values Updating Metric Sample Values
+ *
+ * Now that we have a metric configured for creation and registration, we can update our metric within any of the
+ * functions of the file in which it was declared. For example:
+ *
+ * @code{.c}
+ *
+ * void my_lib_do_something(void) {
+ *   printf("I did a really important thing!\n");
+ *   taos_counter_inc(my_counter, NULL);
+ * }
+ * @endcode
+ *
+ * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass
+ * NULL as the second argument.
+ *
+ * @section Program-Initialization Program Initialization
+ *
+ * At the start of the program's main function you need to do two things:
+ *
+ * * Initialize the default metric collector registry:
+ *
+ * @code{.c}
+ *
+ * taos_collector_registry_default_init();
+ *
+ * @endcode
+ *
+ * * For each file containing prometheus metrics, call its corresponding metric initialization function
+ *
+ * @code{.c}
+ *
+ * foo_metric_init()
+ *
+ * @endcode
+ *
+ * After initialization is complete, you may proceed to do work and update your metrics.
+ *
+ * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP
+ *
+ * @todo Describe how to use libpromhttp to expose metrics over HTTP
+ *
+ * @section Where-To-Go-From-Here Where to Go From Here?
+ *
+ * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html)
+ * tab in this documentation site for more information about the public API available to you. Also, you can take a look
+ * at the examples directory at the
+ * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration.
+ */
+
+#ifndef TAOS_INCLUDED
+#define TAOS_INCLUDED
+
+#include "taos_alloc.h"
+#include "taos_collector.h"
+#include "taos_collector_registry.h"
+#include "taos_counter.h"
+#include "taos_linked_list.h"
+#include "taos_map.h"
+#include "taos_metric.h"
+#include "taos_metric_sample.h"
+#include "taos_monitor_util.h"
+#include "taos_gauge.h"
+
+#endif //  TAOS_INCLUDED

include/libs/monitorfw/taos_monitor_util.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @file taos_monitor_util.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_MONITOR_UTIL_H
+#define TAOS_MONITOR_UTIL_H
+
+#include "taos_metric.h"
+#include "tjson.h"
+
+void taos_monitor_split_str(char** arr, char* str, const char* del);
+void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf);
+char* taos_monitor_get_metric_name(taos_metric_t* metric);
+
+#endif  // TAOS_MONITOR_UTIL_H

include/os/osAtomic.h

@@ -48,11 +48,13 @@ void    atomic_store_8(int8_t volatile *ptr, int8_t val);
 void    atomic_store_16(int16_t volatile *ptr, int16_t val);
 void    atomic_store_32(int32_t volatile *ptr, int32_t val);
 void    atomic_store_64(int64_t volatile *ptr, int64_t val);
+double  atomic_store_double(double volatile *ptr, double val);
 void    atomic_store_ptr(void *ptr, void *val);
 int8_t  atomic_exchange_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
+double  atomic_exchange_double(double volatile *ptr, int64_t val);
 void   *atomic_exchange_ptr(void *ptr, void *val);
 int8_t  atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
 int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
@@ -68,6 +70,7 @@ int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
+double  atomic_fetch_add_double(double volatile *ptr, double val);
 void   *atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
@@ -78,6 +81,7 @@ int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
+double atomic_fetch_sub_double(double volatile *ptr, double val);
 void   *atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t  atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);

include/util/taoserror.h

@@ -758,8 +758,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_COL_QUERY_MISMATCH        TAOS_DEF_ERROR_CODE(0, 0x266D)
 #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE  TAOS_DEF_ERROR_CODE(0, 0x266E)
 #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS         TAOS_DEF_ERROR_CODE(0, 0x266F)
-#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED    TAOS_DEF_ERROR_CODE(0, 0x2670)
+#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT  TAOS_DEF_ERROR_CODE(0, 0x2670)
-#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2671)
+#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED    TAOS_DEF_ERROR_CODE(0, 0x2671)
+#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2672)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner

include/util/tbase58.h

@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#define TBASE_MAX_ILEN 4096
+#define TBASE_MAX_OLEN 5653
+
 uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen);
 char    *base58_encode(const uint8_t *value, int32_t vlen);
 

include/util/tdef.h

@@ -197,6 +197,8 @@ typedef enum ELogicConditionType {
 // ACCOUNT is a 32 bit positive integer
 // this is the length of its string representation, including the terminator zero
 #define TSDB_ACCT_ID_LEN 11
+#define TSDB_NODE_ID_LEN 11
+#define TSDB_VGROUP_ID_LEN 11
 
 #define TSDB_MAX_COLUMNS 4096
 #define TSDB_MIN_COLUMNS 2  // PRIMARY COLUMN(timestamp) + other columns
@@ -548,6 +550,10 @@ enum {
 #define VNODE_TIMEOUT_SEC 60
 #define MNODE_TIMEOUT_SEC 60
 
+#define MONITOR_TABLENAME_LEN     200
+#define MONITOR_TAG_NAME_LEN      100
+#define MONITOR_TAG_VALUE_LEN     300
+#define MONITOR_METRIC_NAME_LEN   100
 #ifdef __cplusplus
 }
 #endif

include/util/tjson.h

@@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
 SJson*      tjsonParse(const char* pStr);
 bool        tjsonValidateJson(const char* pJson);
 const char* tjsonGetError();
+void        tjsonDeleteItemFromObject(const SJson* pJson, const char* pName);
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -120,6 +120,7 @@ void    taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
 #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
 #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
 #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
+#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }}
 
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }

source/client/CMakeLists.txt

@@ -20,7 +20,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+    PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
 )
 
 if(TD_DARWIN_ARM64)
@@ -61,7 +61,7 @@ target_include_directories(
 target_link_libraries(
     taos_static
     INTERFACE api
-    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+    PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -396,6 +396,8 @@ void       hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr);
 void       destroyAllRequests(SHashObj* pRequests);
 void       stopAllRequests(SHashObj* pRequests);
 
+SAppInstInfo* getAppInstInfo(const char* clusterKey);
+
 // conn level
 int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey);
@@ -432,6 +434,21 @@ void    freeQueryParam(SSyncQueryParam* param);
 int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
 #endif
 
+void clientSlowQueryMonitorInit(const char* clusterKey);
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost);
+
+void clientSQLReqMonitorInit(const char* clusterKey);
+
+enum {
+  MONITORSQLTYPESELECT = 0,
+  MONITORSQLTYPEINSERT = 1,
+  MONITORSQLTYPEDELETE = 2
+};
+
+void sqlReqLog(int64_t rid,  bool killed, int32_t code, int8_t type);
+
+void clientMonitorClose(const char* clusterKey);
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -18,6 +18,7 @@
 #include "clientLog.h"
 #include "functionMgt.h"
 #include "os.h"
+#include "osSleep.h"
 #include "query.h"
 #include "qworker.h"
 #include "scheduler.h"
@@ -105,7 +106,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
 
       atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
       reqType = SLOW_LOG_TYPE_QUERY;
-    }
+    } 
+  }
+
+  if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
+  } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT);
+  } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) {
+    sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE);
   }
 
   if (duration >= (tsSlowLogThreshold * 1000000UL)) {
@@ -114,6 +123,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
       taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
                        taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
                        pRequest->sqlstr);
+      SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
     }
   }
 
@@ -224,6 +234,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) {
 
   taosThreadMutexLock(&appInfo.mutex);
 
+  clientMonitorClose(pAppInfo->instKey);
   hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
   taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
 

source/client/src/clientImpl.c

@@ -15,6 +15,7 @@
 
 #include "cJSON.h"
 #include "clientInt.h"
+#include "clientMonitor.h"
 #include "clientLog.h"
 #include "command.h"
 #include "scheduler.h"
@@ -157,6 +158,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
     tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port);
 
     pInst = &p;
+
+    clientSlowQueryMonitorInit(p->instKey);
+    clientSQLReqMonitorInit(p->instKey);
   } else {
     ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
     // reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
@@ -166,9 +170,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
   taosThreadMutexUnlock(&appInfo.mutex);
 
   taosMemoryFreeClear(key);
+
   return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
 }
 
+SAppInstInfo* getAppInstInfo(const char* clusterKey) {
+  SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
+  if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
+    return *ppAppInstInfo;
+  } else {
+    return NULL;
+  }
+}
+
 void freeQueryParam(SSyncQueryParam* param) {
   if (param == NULL) return;
   tsem_destroy(&param->sem);
@@ -1850,7 +1864,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
 
   char* pStart = p + len;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
 
     if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
       int32_t* offset = (int32_t*)pStart;
@@ -1949,8 +1963,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
   char* pStart = p;
   char* pStart1 = p1;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
-    int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i];
+    int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i];
     if (ASSERT(colLen < dataLen)) {
       tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen);
       return TSDB_CODE_TSC_INTERNAL_ERROR;
@@ -2009,7 +2023,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       }
       colLen1 = len;
       totalLen += colLen1;
-      colLength1[i] = (blockVersion == 1) ? htonl(len) : len;
+      colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len;
     } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       len = numOfRows * sizeof(int32_t);
       memcpy(pStart1, pStart, len);
@@ -2098,7 +2112,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
 
   char* pStart = p;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(blockVersion == 1){
+    if(blockVersion == BLOCK_VERSION_1){
       colLength[i] = htonl(colLength[i]);
     }
     if (colLength[i] >= dataLen) {

source/client/src/clientMain.c

@@ -279,6 +279,7 @@ void taos_close_internal(void *taos) {
 
   STscObj *pTscObj = (STscObj *)taos;
   tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
+  // clientMonitorClose(pTscObj->pAppInfo->instKey);
 
   taosRemoveRef(clientConnRefPool, pTscObj->id);
 }

source/client/src/clientRawBlockWrite.c

@@ -1001,6 +1001,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
 
       tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
       taosArrayPush(tBatch.req.pArray, pCreateReq);
+      tBatch.req.source = TD_REQ_FROM_TAOX;
 
       taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch));
     } else {  // add to the correct vgroup
@@ -1276,7 +1277,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
     return terrno;
   }
   SVAlterTbReq   req = {0};
-  SDecoder       coder = {0};
+  SDecoder       dcoder = {0};
   int32_t        code = TSDB_CODE_SUCCESS;
   SRequestObj*   pRequest = NULL;
   SQuery*        pQuery = NULL;
@@ -1297,8 +1298,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
   // decode and process req
   void*   data = POINTER_SHIFT(meta, sizeof(SMsgHead));
   int32_t len = metaLen - sizeof(SMsgHead);
-  tDecoderInit(&coder, data, len);
+  tDecoderInit(&dcoder, data, len);
-  if (tDecodeSVAlterTbReq(&coder, &req) < 0) {
+  if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) {
     code = TSDB_CODE_INVALID_PARA;
     goto end;
   }
@@ -1340,14 +1341,36 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
   pVgData->vg = pInfo;
-  pVgData->pData = taosMemoryMalloc(metaLen);
+
-  if (NULL == pVgData->pData) {
+  int         tlen = 0;
+  req.source = TD_REQ_FROM_TAOX;
+  tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code);
+  if(code != 0){
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
-  memcpy(pVgData->pData, meta, metaLen);
+  tlen += sizeof(SMsgHead);
-  ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId);
+  void* pMsg = taosMemoryMalloc(tlen);
-  pVgData->size = metaLen;
+  if (NULL == pMsg) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  ((SMsgHead*)pMsg)->vgId = htonl(pInfo.vgId);
+  ((SMsgHead*)pMsg)->contLen = htonl(tlen);
+  void*    pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead));
+  SEncoder coder = {0};
+  tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead));
+  code = tEncodeSVAlterTbReq(&coder, &req);
+  if(code != 0){
+    tEncoderClear(&coder);
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  tEncoderClear(&coder);
+
+  pVgData->pData = pMsg;
+  pVgData->size = tlen;
+
   pVgData->numOfTables = 1;
   taosArrayPush(pArray, &pVgData);
 
@@ -1387,7 +1410,7 @@ end:
   if (pVgData) taosMemoryFreeClear(pVgData->pData);
   taosMemoryFreeClear(pVgData);
   destroyRequest(pRequest);
-  tDecoderClear(&coder);
+  tDecoderClear(&dcoder);
   qDestroyQuery(pQuery);
   terrno = code;
   return code;

source/client/src/clientSqlMonitor.c

@@ -0,0 +1,79 @@
+ /*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* selectMonitorName = "taos_sql_req:count";
+const char* selectMonitorHelp = "count for select sql";
+const int   selectMonitorLabelCount = 4;
+const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"};
+
+static const char* defaultClusterID = "";
+
+void clientSQLReqMonitorInit(const char* clusterKey) {
+  if (!tsEnableMonitor) return;
+  SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+  SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+  clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+  createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels);
+}
+
+void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) {
+  const char* typeStr;
+  switch (type) {
+    case MONITORSQLTYPEDELETE:
+      typeStr = "delete";
+      break;
+    case MONITORSQLTYPEINSERT:
+      typeStr = "insert";
+      break;
+    default:
+      typeStr = "select";
+      break;
+  }
+  const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)};
+  taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues);
+}
+
+void sqlReqLog(int64_t rid,  bool killed, int32_t code, int8_t type) {
+  if (!tsEnableMonitor) return;
+  SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+  if (TSDB_CODE_SUCCESS != code) {
+    result = SQL_RESULT_FAILED;
+  }
+  // to do Distinguish active Kill events
+  // else if (killed) {
+  //   result = SQL_RESULT_CANCEL;
+  // }
+
+  STscObj* pTscObj = acquireTscObj(rid);
+  if (pTscObj != NULL) {
+    if (pTscObj->pAppInfo == NULL) {
+      tscLog("sqlReqLog, not found pAppInfo");
+    } else {
+      clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type);
+    }
+    releaseTscObj(rid);
+  } else {
+    tscLog("sqlReqLog, not found rid");
+  }
+}
+
+void clientMonitorClose(const char* clusterKey) {
+  tscLog("clientMonitorClose, key:%s", clusterKey);
+  clusterMonitorClose(clusterKey);
+}

source/client/src/clientTmq.c

@@ -389,7 +389,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
   if (strcasecmp(key, "msg.consume.excluded") == 0) {
-    conf->sourceExcluded = taosStr2int64(value);
+    conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0;
     return TMQ_CONF_OK;
   }
 
@@ -1611,17 +1611,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
-SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
-  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
-  pRspObj->resType = RES_TYPE__TMQ;
 
+void changeByteEndian(char* pData){
+  char* p = pData;
+
+  // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
+  // version:
+  int32_t blockVersion = *(int32_t*)p;
+  ASSERT(blockVersion == BLOCK_VERSION_1);
+  *(int32_t*)p = BLOCK_VERSION_2;
+
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  int32_t cols = *(int32_t*)p;
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(uint64_t);
+  // check fields
+  p += cols * (sizeof(int8_t) + sizeof(int32_t));
+
+  int32_t* colLength = (int32_t*)p;
+  for (int32_t i = 0; i < cols; ++i) {
+    colLength[i] = htonl(colLength[i]);
+  }
+}
+
+static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
   (*numOfRows) = 0;
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
 
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
-  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
@@ -1633,41 +1655,44 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
   }
   // extract the rows in this data packet
   for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
+    void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t                  rows = htobe64(pRetrieve->numOfRows);
+    void* rawData = NULL;
+    int64_t rows = 0;
+    // deal with compatibility
+    if(*(int64_t*)pRetrieve == 0){
+      rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
+    }else if(*(int64_t*)pRetrieve == 1){
+      rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
+    }
+
     pVg->numOfRows += rows;
     (*numOfRows) += rows;
-
+    changeByteEndian(rawData);
-    if (needTransformSchema) {  // withSchema is false if subscribe subquery, true if subscribe db or stable
+    if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
-      SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
+      SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
-      if (schema) {
+      if(schema){
         taosArrayPush(pRspObj->rsp.blockSchema, &schema);
       }
     }
   }
+}
 
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
+  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
+  pRspObj->resType = RES_TYPE__TMQ;
+  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj);
   return pRspObj;
 }
 
 SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
   SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
   pRspObj->resType = RES_TYPE__TMQ_METADATA;
-  tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
-  tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
-  pRspObj->vgId = pWrapper->vgHandle->vgId;
-  pRspObj->resIter = -1;
   memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
 
-  pRspObj->resInfo.totalRows = 0;
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj);
-  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-
-  // extract the rows in this data packet
-  for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t                  rows = htobe64(pRetrieve->numOfRows);
-    pVg->numOfRows += rows;
-    (*numOfRows) += rows;
-  }
   return pRspObj;
 }
 

source/client/src/slowQueryMonitor.c

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* slowQueryName = "taos_slow_sql:count";
+const char* slowQueryHelp = "slow query log when cost over than config duration";
+const int   slowQueryLabelCount = 4;
+const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"};
+static const char* defaultClusterID = "";
+
+const int64_t usInSeconds = 1000 * 1000;
+const int64_t msInMinutes = 60 * 1000;
+
+static const char* getSlowQueryLableCostDesc(int64_t cost) {
+  if (cost >= 1000 * usInSeconds) {
+    return "1000s-";
+  } else if (cost >= 100 * usInSeconds) {
+    return "100-1000s";
+  } else if (cost >= 10 * usInSeconds) {
+    return "10-100s";
+  } else if (cost >= 3 * usInSeconds) {
+    return "3-10s";
+  }
+  return "0-3s";
+}
+
+void clientSlowQueryMonitorInit(const char* clusterKey) {
+  if (!tsEnableMonitor) return;
+  SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+  SEpSet        epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+  clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+  createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels);
+}
+
+void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) {
+  const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)};
+  taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues);
+}
+
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
+  if (!tsEnableMonitor) return;
+  SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+  if (TSDB_CODE_SUCCESS != code) {
+    result = SQL_RESULT_FAILED;
+  }
+  // to do Distinguish active Kill events
+  // else if (killed) {
+  //   result = SQL_RESULT_CANCEL;
+  // }
+
+  STscObj* pTscObj = acquireTscObj(rid);
+  if (pTscObj != NULL) {
+    if(pTscObj->pAppInfo == NULL) {
+      tscLog("SlowQueryLog, not found pAppInfo");
+    } else {
+      clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost);
+    }
+    releaseTscObj(rid);
+  } else {
+    tscLog("SlowQueryLog, not found rid");
+  }
+}

source/client/test/CMakeLists.txt

@@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES(
         PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
 )
 
+ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
+TARGET_LINK_LIBRARIES(
+        clientMonitorTest
+        PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
+)
+
 TARGET_INCLUDE_DIRECTORIES(
         clientTest
         PUBLIC "${TD_SOURCE_DIR}/include/client/"
@@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES(
         PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
 )
 
+TARGET_INCLUDE_DIRECTORIES(
+        clientMonitorTest
+        PUBLIC "${TD_SOURCE_DIR}/include/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
+)
+
 add_test(
         NAME smlTest
         COMMAND smlTest
 )
+
+# add_test(
+#         NAME clientMonitorTest
+#         COMMAND clientMonitorTest
+# )

source/client/test/clientMonitorTests.cpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "taoserror.h"
+#include "tglobal.h"
+#include "thash.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "executor.h"
+#include "taos.h"
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(clientMonitorTest, monitorTest) {
+  const char* cluster1 = "cluster1";
+  const char* cluster2 = "cluster2";
+  SEpSet epSet;
+  clusterMonitorInit(cluster1, epSet, NULL);
+  const char* counterName1 = "slow_query";
+  const char* counterName2 = "select_count";
+  const char* help1 = "test for slowQuery";
+  const char* help2 = "test for selectSQL";
+  const char* lables[] = {"lable1"};
+  taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables);
+  ASSERT_TRUE(c1 != NULL);
+  taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables);
+  ASSERT_TRUE(c2 != NULL);
+  ASSERT_TRUE(c1 != c2);
+  taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+  ASSERT_TRUE(c21 == NULL);
+  clusterMonitorInit(cluster2, epSet, NULL);
+  c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+  ASSERT_TRUE(c21 != NULL);
+  int i = 0;
+  while (i < 12) {
+      taosMsleep(10);
+      ++i;
+  }
+  clusterMonitorClose(cluster1);
+  clusterMonitorClose(cluster2);
+}
+
+TEST(clientMonitorTest, sendTest) {
+  TAOS*       taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
+  ASSERT_TRUE(taos != NULL);
+  printf("connect taosd sucessfully.\n");
+
+  int64_t rid = *(int64_t *)taos;
+  SlowQueryLog(rid, false, -1, 1000);
+  int i = 0;
+  while (i < 20) {
+    SlowQueryLog(rid, false, 0, i * 1000);
+    taosMsleep(10);
+    ++i;
+  }
+
+  taos_close(taos);
+}

source/common/src/systable.c

@@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = {
 };
 
 static const SSysDbTableSchema userUsersSchema[] = {
-    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
+    {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
-    {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
+    {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
-    {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
+    {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
-    {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 GRANTS_SCHEMA;
@@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = {
 };
 
 static const SSysDbTableSchema variablesSchema[] = {
-    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = {
 static const SSysDbTableSchema vnodesSchema[] = {
     {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true},
     {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
     {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
@@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
 };
 
 static const SSysDbTableSchema userUserPrivilegesSchema[] = {
-    {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema userViewsSchema[] = {
@@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
 };
 
 static const SSysDbTableSchema useGrantsFullSchema[] = {
-    {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema useGrantsLogsSchema[] = {
-    {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysDbTableSchema useMachinesSchema[] = {
-    {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
-    {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
 static const SSysTableMeta infosMeta[] = {
@@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
     {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
     {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
-    {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false},
+    {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true},
-    {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false},
+    {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true},
-    {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false},
+    {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tdatablock.c

@@ -2288,28 +2288,28 @@ _end:
   return TSDB_CODE_SUCCESS;
 }
 
-void  buildCtbNameAddGruopId(char* ctbName, uint64_t groupId){
+void  buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){
   char tmp[TSDB_TABLE_NAME_LEN] = {0};
   snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
   ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0;  // put groupId to the end
   strcat(ctbName, tmp);
 }
 
-bool  isAutoTableName(char* ctbName){
+// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.
-  return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_');
+// the total length is fixed to be 34 bytes.
-}
+bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); }
 
-bool  alreadyAddGroupId(char* ctbName){
+bool alreadyAddGroupId(char* ctbName) {
   size_t len = strlen(ctbName);
   size_t _location = len - 1;
-  while(_location > 0){
+  while (_location > 0) {
-    if(ctbName[_location] < '0' || ctbName[_location] > '9'){
+    if (ctbName[_location] < '0' || ctbName[_location] > '9') {
       break;
     }
     _location--;
   }
 
-  return ctbName[_location] == '_' &&  len - 1 - _location > 15;  //15 means the min length of groupid
+  return ctbName[_location] == '_' && len - 1 - _location >= 15;  // 15 means the min length of groupid
 }
 
 char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
@@ -2366,7 +2366,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
 
   // todo extract method
   int32_t* version = (int32_t*)data;
-  *version = 2;
+  *version = BLOCK_VERSION_1;
   data += sizeof(int32_t);
 
   int32_t* actualLen = (int32_t*)data;
@@ -2447,7 +2447,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
       data += colSizes[col];
     }
 
-//    colSizes[col] = htonl(colSizes[col]);
+    colSizes[col] = htonl(colSizes[col]);
     //    uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
     //    htonl(colSizes[col]), colSizes[col]);
   }
@@ -2512,9 +2512,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
   pStart += sizeof(int32_t) * numOfCols;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(version == 1){
+    colLen[i] = htonl(colLen[i]);
-      colLen[i] = htonl(colLen[i]);
-    }
     ASSERT(colLen[i] >= 0);
 
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);

source/common/src/tglobal.c

@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
-float   tsRatioOfVnodeStreamThreads = 0.5F;
+float   tsRatioOfVnodeStreamThreads = 1.5F;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
@@ -94,6 +94,9 @@ char     tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
 uint16_t tsMonitorPort = 6043;
 int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
+bool     tsMonitorLogProtocol = false;
+int32_t  tsMonitorIntervalForBasic = 30;
+bool     tsMonitorForceV2 = true;
 
 // audit
 bool    tsEnableAudit = true;
@@ -266,6 +269,7 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024;
 bool    tsFilterScalarMode = false;
 int     tsResolveFQDNRetryTime = 100;  // seconds
 int     tsStreamAggCnt = 1000;
+bool    tsDisableCount = true;
 
 char   tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
 char   tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
@@ -535,6 +539,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
+  if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
   return 0;
 }
 
@@ -693,7 +701,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-
+  if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+    return -1;
+  if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+    
   if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -1081,6 +1093,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64;
   tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32;
   tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32;
+  tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
+  tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
   if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
     return -1;
   }
@@ -1095,6 +1109,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
 
   tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
+
+  tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval;
   return 0;
 }
 
@@ -1155,7 +1171,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
   tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
   tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
-
+  tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval;
+  tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32;
+  tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32;
+  
   tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
   tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
   tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32;
@@ -1718,7 +1737,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
                                          {"shellActivityTimer", &tsShellActivityTimer},
                                          {"slowLogThreshold", &tsSlowLogThreshold},
                                          {"useAdapter", &tsUseAdapter},
-                                         {"experimental", &tsExperimental}};
+                                         {"experimental", &tsExperimental},
+                                         {"disableCount", &tsDisableCount}};
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
       taosCfgSetOption(options, tListLen(options), pItem, false);

source/common/src/tmsg.c

@@ -1436,6 +1436,44 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
 
 void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); }
 
+int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1;
+  if (pReq->contLen > 0) {
+    pReq->pCont = taosMemoryMalloc(pReq->contLen + 1);
+    if (pReq->pCont == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSStatisReq(SStatisReq *pReq) {
+  taosMemoryFreeClear(pReq->pCont);
+}
+
 int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -3280,18 +3318,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq)
   return tlen;
 }
 
-int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1;
-  tEndDecode(&decoder);
-
-  tDecoderClear(&decoder);
-  return 0;
-}
-
 int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -3305,17 +3331,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq)
   return tlen;
 }
 
-int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
+// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) {
   SEncoder encoder = {0};
@@ -4233,17 +4259,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq
   return tlen;
 }
 
-int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
+// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) {
   if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1;
@@ -4325,24 +4351,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   return tlen;
 }
 
-int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
+// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
-  if (pReq->payloadLen > 0) {
+//   if (pReq->payloadLen > 0) {
-    pReq->payload = taosMemoryMalloc(pReq->payloadLen);
+//     pReq->payload = taosMemoryMalloc(pReq->payloadLen);
-    if (pReq->payload == NULL) return -1;
+//     if (pReq->payload == NULL) return -1;
-    if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
+//     if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
-  }
+//   }
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); }
 
@@ -4595,34 +4621,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) {
   taosArrayDestroy(pRsp->pIndexRsp);
 }
 
-int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
+//   if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
-  if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
+//   if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
 
-int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
-  if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
+//   if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
-void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
+// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
 
 int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
@@ -4930,17 +4956,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
   return tlen;
 }
 
-int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
+// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
   SEncoder encoder = {0};
@@ -4955,17 +4981,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR
   return tlen;
 }
 
-int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
+// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
   if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
@@ -5830,17 +5856,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom
   return tlen;
 }
 
-int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
+// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -5889,38 +5915,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq
   return 0;
 }
 
-int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
-  if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
+//   if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
-  if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
+//   if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
 
-int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) {
   SEncoder encoder = {0};
@@ -6958,30 +6984,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
 
 void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
 
-int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
-  // SEncoder encoder = {0};
+//   // SEncoder encoder = {0};
-  // tEncoderInit(&encoder, buf, bufLen);
+//   // tEncoderInit(&encoder, buf, bufLen);
 
-  // if (tStartEncode(&encoder) < 0) return -1;
+//   // if (tStartEncode(&encoder) < 0) return -1;
-  // if (pRsp->rspList) {
+//   // if (pRsp->rspList) {
-  //   int32_t num = taosArrayGetSize(pRsp->rspList);
+//   //   int32_t num = taosArrayGetSize(pRsp->rspList);
-  //   if (tEncodeI32(&encoder, num) < 0) return -1;
+//   //   if (tEncodeI32(&encoder, num) < 0) return -1;
-  //   for (int32_t i = 0; i < num; ++i) {
+//   //   for (int32_t i = 0; i < num; ++i) {
-  //     SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
+//   //     SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
-  //     if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
+//   //     if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
-  //   }
+//   //   }
-  // } else {
+//   // } else {
-  //   if (tEncodeI32(&encoder, 0) < 0) return -1;
+//   //   if (tEncodeI32(&encoder, 0) < 0) return -1;
-  // }
+//   // }
-  // tEndEncode(&encoder);
+//   // tEndEncode(&encoder);
 
-  // int32_t tlen = encoder.pos;
+//   // int32_t tlen = encoder.pos;
-  // tEncoderClear(&encoder);
+//   // tEncoderClear(&encoder);
-  // reture tlen;
+//   // reture tlen;
-  return 0;
+//   return 0;
-}
+// }
 
-int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
   // SDecoder  decoder = {0};
   // int32_t num = 0;
   // tDecoderInit(&decoder, buf, bufLen);
@@ -7002,8 +7028,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc
   // tEndDecode(&decoder);
 
   // tDecoderClear(&decoder);
-  return 0;
+//  return 0;
-}
+//}
 
 int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
   int32_t        nRsps = taosArrayGetSize(pRsp->pArray);
@@ -7138,15 +7164,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
   return 0;
 }
 
-int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
+// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
+//   if (tStartDecode(pCoder) < 0) return -1;
 
-  if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
-  if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
+//   if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
 
-  tEndDecode(pCoder);
+//   tEndDecode(pCoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
@@ -7427,34 +7453,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq *
 
 void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); }
 
-int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
+// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
 
-int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
+// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
   if (NULL == pReq) {
@@ -7513,6 +7539,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
   if (pReq->alterOriDataLen > 0) {
     if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1;
   }
+  if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -7535,6 +7562,10 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
     if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+  }
+
   tEndDecode(pCoder);
   return 0;
 }
@@ -7663,6 +7694,8 @@ int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq)
     if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
   }
 
+  if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
+
   tEndEncode(pCoder);
   return 0;
 }
@@ -7677,6 +7710,10 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
     if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+  }
+
   tEndDecode(pCoder);
   return 0;
 }
@@ -7843,37 +7880,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) {
   return 0;
 }
 
-static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
+// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
-  if (tStartEncode(pCoder) < 0) return -1;
+//   if (tStartEncode(pCoder) < 0) return -1;
 
-  if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
+//   if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
-  if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
+//   if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
-  if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
+//   if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
-  if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
+//   if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
 
-  if (flags & TD_AUTO_CREATE_TABLE) {
+//   if (flags & TD_AUTO_CREATE_TABLE) {
-    if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+//     if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
-  }
+//   }
 
-  tEndEncode(pCoder);
+//   tEndEncode(pCoder);
-  return 0;
+//   return 0;
-}
+// }
 
-static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
+// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
-  if (tStartDecode(pCoder) < 0) return -1;
+//   if (tStartDecode(pCoder) < 0) return -1;
 
-  if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
-  if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
+//   if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
-  if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
+//   if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
-  if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
+//   if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
 
-  if (flags & TD_AUTO_CREATE_TABLE) {
+//   if (flags & TD_AUTO_CREATE_TABLE) {
-    if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+//     if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
-  }
+//   }
 
-  tEndDecode(pCoder);
+//   tEndDecode(pCoder);
-  return 0;
+//   return 0;
-}
+// }
 
 static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) {
   if (tStartEncode(pEncoder) < 0) return -1;
@@ -7956,19 +7993,19 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
   return 0;
 }
 
-void tFreeSSubmitBlkRsp(void *param) {
+// void tFreeSSubmitBlkRsp(void *param) {
-  if (NULL == param) {
+//   if (NULL == param) {
-    return;
+//     return;
-  }
+//   }
 
-  SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
+//   SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
 
-  taosMemoryFree(pRsp->tblFName);
+//   taosMemoryFree(pRsp->tblFName);
-  if (pRsp->pMeta) {
+//   if (pRsp->pMeta) {
-    taosMemoryFree(pRsp->pMeta->pSchemas);
+//     taosMemoryFree(pRsp->pMeta->pSchemas);
-    taosMemoryFree(pRsp->pMeta);
+//     taosMemoryFree(pRsp->pMeta);
-  }
+//   }
-}
+// }
 
 void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
   if (NULL == pRsp) return;
@@ -8034,6 +8071,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
       break;
   }
   if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
+  if (tEncodeI8(pEncoder, pReq->source) < 0) return -1;
 
   tEndEncode(pEncoder);
   return 0;
@@ -8094,6 +8132,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
   if (!tDecodeIsEnd(pDecoder)) {
     if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
   }
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI8(pDecoder, &pReq->source) < 0) return -1;
+  }
 
   tEndDecode(pDecoder);
   return 0;
@@ -8138,23 +8179,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
+// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
-  int32_t  meta = 0;
+//   int32_t  meta = 0;
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
+//   if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
+//     if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
+//   }
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) {
   if (tStartEncode(pEncoder) < 0) return -1;
@@ -8179,22 +8220,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
+// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
-  int32_t  meta = 0;
+//   int32_t  meta = 0;
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
+//   if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
+//     if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
+//   }
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
   if (NULL == pRsp) {
@@ -8230,22 +8271,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
+// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
-  int32_t  meta = 0;
+//   int32_t  meta = 0;
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &meta) < 0) return -1;
+//   if (tDecodeI32(&decoder, &meta) < 0) return -1;
-  if (meta) {
+//   if (meta) {
-    pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pRsp->pMeta) return -1;
+//     if (NULL == pRsp->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+//     if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
-  }
+//   }
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) {
   if (NULL == pRsp) {
@@ -8670,7 +8711,6 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
     }
   }
   if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
-  if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -8758,12 +8798,6 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
       goto _exit;
     }
   }
-  if (!tDecodeIsEnd(pCoder)) {
-    if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) {
-      code = TSDB_CODE_INVALID_MSG;
-      goto _exit;
-    }
-  }
 
   tEndDecode(pCoder);
 

source/common/src/trow.c

@@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa
   return TSDB_CODE_SUCCESS;
 }
 
+#if 0
 int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     terrno = TSDB_CODE_INVALID_PARA;
@@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal
   }
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
       break;
+#if 0
     case -1:
     case 1:
       tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
       break;
+#endif
     default:
       terrno = TSDB_CODE_INVALID_PARA;
       return TSDB_CODE_FAILED;
@@ -231,7 +235,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe
   return true;
 }
 
-#ifdef BUILD_NO_CALL
+// #ifdef BUILD_NO_CALL
 const uint8_t tdVTypeByte[2][3] = {{
                                        // 2 bits
                                        TD_VTYPE_NORM_BYTE_II,
@@ -439,13 +443,15 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) {
   return true;
 }
 
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) {
   STColumn *pTColumn;
   SColVal  *pColVal;
   int32_t   nColVal = taosArrayGetSize(pArray);
   int32_t   varDataLen = 0;
+  int32_t   nonVarDataLen = 0;
   int32_t   maxVarDataLen = 0;
   int32_t   iColVal = 0;
+  int32_t   nBound = 0;
   void     *varBuf = NULL;
   bool      isAlloc = false;
 
@@ -458,6 +464,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
     } else {
       pColVal = NULL;
     }
+    if (pColVal && !COL_VAL_IS_NONE(pColVal)) {
+      ++nBound;
+    }
 
     if (iColumn == 0) {
       ASSERT(pColVal && pColVal->cid == pTColumn->colId);
@@ -484,16 +493,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
             }
           }
         }
+      } else {
+        if(pColVal && COL_VAL_IS_VALUE(pColVal)) {
+          nonVarDataLen += TYPE_BYTES[pTColumn->type];
+        }
       }
     }
 
     ++iColVal;
   }
 
-  if (!(*ppRow)) {
+  int32_t rowTotalLen = 0;
-    *ppRow = (STSRow *)taosMemoryCalloc(
+  if (rowType == TD_ROW_TP) {
-        1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
+    rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1);
-    isAlloc = true;
+  } else {
+    rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) +
+                  TD_BITMAP_BYTES(nBound - 1);
+  }
+    if (!(*ppRow)) {
+      *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen);
+      isAlloc = true;
   }
 
   if (!(*ppRow)) {
@@ -512,10 +531,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
     }
   }
 
-  SRowBuilder rb = {0};
+  SRowBuilder rb = {.rowType = rowType};
   tdSRowInit(&rb, pTSchema->version);
-  tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen);
+  tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen);
   tdSRowResetBuf(&rb, *ppRow);
+  int32_t iBound = 0;
 
   iColVal = 0;
   for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) {
@@ -529,21 +549,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
         valType = TD_VTYPE_NONE;
       } else if (COL_VAL_IS_NULL(pColVal)) {
         valType = TD_VTYPE_NULL;
+        ++iBound;
       } else if (IS_VAR_DATA_TYPE(pTColumn->type)) {
         varDataSetLen(varBuf, pColVal->value.nData);
         if (pColVal->value.nData != 0) {
           memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
         }
         val = varBuf;
+        ++iBound;
       } else {
         val = (const void *)&pColVal->value.val;
+        ++iBound;
       }
     } else {
       // pColVal = NULL;
       valType = TD_VTYPE_NONE;
     }
 
-    tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+    if (TD_IS_TP_ROW(rb.pBuf)) {
+      tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+    } else {
+      tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1);
+    }
 
     ++iColVal;
   }
@@ -605,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell
 
   return true;
 }
-
+#if 0
 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
   if (!pBitmap || colIdx < 0) {
     terrno = TSDB_CODE_INVALID_PARA;
@@ -655,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
   }
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) {
 #ifdef TD_SUPPORT_BITMAP
@@ -761,11 +789,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp
 
 int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
                               int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) {
-  if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) {
+  if (colIdx < 1) {
     terrno = TSDB_CODE_INVALID_PARA;
+    ASSERTS(0, "colIdx is %" PRIi64, colIdx);
     return terrno;
   }
-  offset -= sizeof(SKvRowIdx);
   --colIdx;
 
 #ifdef TD_SUPPORT_BITMAP
@@ -779,6 +807,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
   SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
   pColIdx->colId = colId;
   pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
+  pBuilder->offset += sizeof(SKvRowIdx);
   if (valType == TD_VTYPE_NORM) {
     char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     if (IS_VAR_DATA_TYPE(colType)) {
@@ -830,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
 
   return 0;
 }
-
+#if 0
 int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,
                               int32_t allNullLen, int32_t boundNullLen) {
   if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) {
@@ -866,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou
 #endif
   return TSDB_CODE_SUCCESS;
 }
+#endif
 
 int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
   pBuilder->pBuf = (STSRow *)pBuf;
@@ -904,6 +934,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
       TD_ROW_SET_LEN(pBuilder->pBuf, len);
       TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
       TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols);
+      pBuilder->offset = 0;
       break;
     default:
       terrno = TSDB_CODE_INVALID_PARA;
@@ -1040,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
     case 0:
       tdSetBitmapValTypeII(pBitmap, colIdx, valType);
       break;
+#if 0
     case -1:
     case 1:
       tdSetBitmapValTypeI(pBitmap, colIdx, valType);
       break;
+#endif
     default:
       terrno = TSDB_CODE_INVALID_PARA;
       return TSDB_CODE_FAILED;
@@ -1056,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
 void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) {
   STColumn *pTColumn = &pTSchema->columns[iCol];
   SCellVal  cv = {0};
-  SValue    value = {0};
 
   ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0));
 
@@ -1084,5 +1116,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV
       memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes);
     }
   }
-}
+}
-#endif

source/common/test/CMakeLists.txt

@@ -35,6 +35,10 @@ target_include_directories(
         PUBLIC "${TD_SOURCE_DIR}/include/common"
         PUBLIC "${TD_SOURCE_DIR}/include/util"
 )
+add_test(
+    NAME dataformatTest
+    COMMAND dataformatTest
+)
 
 # tmsg test
 # add_executable(tmsgTest "")

source/common/test/dataformatTest.cpp

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#if 0
 #include <gtest/gtest.h>
 
 #include <taoserror.h>
@@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) {
       case 1: {
         pSchema[i].type = TSDB_DATA_TYPE_INT;
         pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
-        ;
       } break;
       case 2: {
         pSchema[i].type = TSDB_DATA_TYPE_BIGINT;
@@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) {
   return pResult;
 }
 
-// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+// ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
+// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
 static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
   if (!(*pArray)) {
     *pArray = taosArrayInit(nCols, sizeof(SColVal));
@@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
       taosArrayPush(*pArray, &colVal);
       continue;
     }
-
     switch (i) {
       case 0:
-        sscanf(data[i], "%" PRIi64, &colVal.value.ts);
+        colVal.type = TSDB_DATA_TYPE_TIMESTAMP;
+        sscanf(data[i], "%" PRIi64, &colVal.value.val);
         break;
       case 1:
-        sscanf(data[i], "%" PRIi32, &colVal.value.i32);
+        colVal.type = TSDB_DATA_TYPE_INT;
+        sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val);
         break;
       case 2:
-        sscanf(data[i], "%" PRIi64, &colVal.value.i64);
+        colVal.type = TSDB_DATA_TYPE_BIGINT;
+        sscanf(data[i], "%" PRIi64, &colVal.value.val);
         break;
       case 3:
-        sscanf(data[i], "%f", &colVal.value.f);
+        colVal.type = TSDB_DATA_TYPE_FLOAT;
+        sscanf(data[i], "%f", (float *)&colVal.value.val);
         break;
       case 4:
-        sscanf(data[i], "%lf", &colVal.value.d);
+        colVal.type = TSDB_DATA_TYPE_DOUBLE;
+        sscanf(data[i], "%lf", (double *)&colVal.value.val);
         break;
       case 5: {
+        colVal.type = TSDB_DATA_TYPE_BINARY;
         int16_t dataLen = strlen(data[i]) + 1;
         colVal.value.nData = dataLen < 10 ? dataLen : 10;
         colVal.value.pData = (uint8_t *)data[i];
       } break;
       case 6: {
+        colVal.type = TSDB_DATA_TYPE_NCHAR;
         int16_t dataLen = strlen(data[i]) + 1;
         colVal.value.nData = dataLen < 40 ? dataLen : 40;
         colVal.value.pData = (uint8_t *)data[i];  // just for test, not real nchar
       } break;
-      case 7:
+      case 7: {
-      case 9: {
+        colVal.type = TSDB_DATA_TYPE_TINYINT;
         int32_t d8;
         sscanf(data[i], "%" PRId32, &d8);
-        colVal.value.i8 = (int8_t)d8;
+        colVal.value.val = (int8_t)d8;
-      } break;
+      }
       case 8: {
+        colVal.type = TSDB_DATA_TYPE_SMALLINT;
         int32_t d16;
         sscanf(data[i], "%" PRId32, &d16);
-        colVal.value.i16 = (int16_t)d16;
+        colVal.value.val = (int16_t)d16;
+      } break;
+      case 9: {
+        colVal.type = TSDB_DATA_TYPE_BOOL;
+        int32_t d8;
+        sscanf(data[i], "%" PRId32, &d8);
+        colVal.value.val = (int8_t)d8;
       } break;
       case 10: {
+        colVal.type = TSDB_DATA_TYPE_UTINYINT;
         uint32_t u8;
         sscanf(data[i], "%" PRId32, &u8);
-        colVal.value.u8 = (uint8_t)u8;
+        colVal.value.val = (uint8_t)u8;
       } break;
       case 11: {
+        colVal.type = TSDB_DATA_TYPE_USMALLINT;
         uint32_t u16;
         sscanf(data[i], "%" PRId32, &u16);
-        colVal.value.u16 = (uint16_t)u16;
+        colVal.value.val = (uint16_t)u16;
       } break;
       case 12: {
-        sscanf(data[i], "%" PRIu32, &colVal.value.u32);
+        colVal.type = TSDB_DATA_TYPE_UINT;
+        sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val);
       } break;
       case 13: {
-        sscanf(data[i], "%" PRIu64, &colVal.value.u64);
+        colVal.type = TSDB_DATA_TYPE_UBIGINT;
+        sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val);
       } break;
       default:
         ASSERT(0);
@@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
   }
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
-      printf("%s ", cv->value.i8 == 0 ? "false" : "true");
+      printf("%s ", cv->value.val == 0 ? "false" : "true");
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      printf("%" PRIi8 " ", cv->value.i8);
+      printf("%" PRIi8 " ", *(int8_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      printf("%" PRIi16 " ", cv->value.i16);
+      printf("%" PRIi16 " ", *(int16_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_INT:
-      printf("%" PRIi32 " ", cv->value.i32);
+      printf("%" PRIi32 " ", *(int32_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      printf("%" PRIi64 " ", cv->value.i64);
+      printf("%" PRIi64 " ", cv->value.val);
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      printf("%f ", cv->value.f);
+      printf("%f ", *(float *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      printf("%lf ", cv->value.d);
+      printf("%lf ", *(double *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_GEOMETRY: {
@@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
       printf("%s ", tv);
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP:
-      printf("%" PRIi64 " ", cv->value.i64);
+      printf("%" PRIi64 " ", cv->value.val);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       char tv[15] = {0};
@@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
       printf("%s ", tv);
     } break;
     case TSDB_DATA_TYPE_UTINYINT:
-      printf("%" PRIu8 " ", cv->value.u8);
+      printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      printf("%" PRIu16 " ", cv->value.u16);
+      printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_UINT:
-      printf("%" PRIu32 " ", cv->value.u32);
+      printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      printf("%" PRIu64 " ", cv->value.u64);
+      printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val);
       break;
     case TSDB_DATA_TYPE_JSON:
       printf("JSON ");
@@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
   return 0;
 }
 
-void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
+void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) {
-  // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+  printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len);
   for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
-    tTSRowGet(row, pTSchema, i, &cv);
+    tTSRowGetVal(row, pTSchema, i, &cv);
     debugPrintSColVal(&cv, pTSchema->columns[i].type);
   }
   printf("\n");
@@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
     case TSDB_DATA_TYPE_TINYINT: {
       int32_t d8;
       sscanf(rawVal, "%" PRId32, &d8);
-      EXPECT_EQ(cv->value.i8, (int8_t)d8);
+      EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8);
     } break;
     case TSDB_DATA_TYPE_SMALLINT: {
       int32_t d16;
       sscanf(rawVal, "%" PRId32, &d16);
-      EXPECT_EQ(cv->value.i16, (int16_t)d16);
+      EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16);
     } break;
     case TSDB_DATA_TYPE_INT: {
-      sscanf(rawVal, "%" PRId32, &rawSVal.i32);
+      sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val);
-      EXPECT_EQ(cv->value.i32, rawSVal.i32);
+      EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_BIGINT: {
-      sscanf(rawVal, "%" PRIi64, &rawSVal.i64);
+      sscanf(rawVal, "%" PRIi64, &rawSVal.val);
-      EXPECT_EQ(cv->value.i64, rawSVal.i64);
+      EXPECT_EQ(cv->value.val, rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_FLOAT: {
-      sscanf(rawVal, "%f", &rawSVal.f);
+      sscanf(rawVal, "%f", (float *)&rawSVal.val);
-      EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f);
+      EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_DOUBLE: {
-      sscanf(rawVal, "%lf", &rawSVal.d);
+      sscanf(rawVal, "%lf", (double *)&rawSVal.val);
-      EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d);
+      EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_GEOMETRY: {
       EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);
     } break;
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      sscanf(rawVal, "%" PRIi64, &rawSVal.ts);
+      sscanf(rawVal, "%" PRIi64, &rawSVal.val);
-      EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts);
+      EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_NCHAR: {
       EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);  // informal nchar comparsion
@@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
     case TSDB_DATA_TYPE_UTINYINT: {
       uint32_t u8;
       sscanf(rawVal, "%" PRIu32, &u8);
-      EXPECT_EQ(cv->value.u8, (uint8_t)u8);
+      EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8);
     } break;
     case TSDB_DATA_TYPE_USMALLINT: {
       uint32_t u16;
       sscanf(rawVal, "%" PRIu32, &u16);
-      EXPECT_EQ(cv->value.u16, (uint16_t)u16);
+      EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16);
     } break;
     case TSDB_DATA_TYPE_UINT: {
-      sscanf(rawVal, "%" PRIu32, &rawSVal.u32);
+      sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val);
-      EXPECT_EQ(cv->value.u32, rawSVal.u32);
+      EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_UBIGINT: {
-      sscanf(rawVal, "%" PRIu64, &rawSVal.u64);
+      sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val);
-      EXPECT_EQ(cv->value.u64, rawSVal.u64);
+      EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val);
     } break;
     case TSDB_DATA_TYPE_JSON:
       printf("JSON ");
@@ -395,36 +411,66 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
   return 0;
 }
 
-static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
+static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) {
   for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
-    tTSRowGet(row, pTSchema, i, &cv);
+    tTSRowGetVal(row, pTSchema, i, &cv);
     checkSColVal(data[i], &cv, pTSchema->columns[i].type);
   }
+
+  STSRowIter rowIter = {0};
+  rowIter.pSchema = pTSchema;
+  tdSTSRowIterReset(&rowIter, row);
+  for (int32_t i = 0; i < pTSchema->numOfCols; ++i) {
+    STColumn *pCol = pTSchema->columns + i;
+    SColVal   colVal = {0};
+    SCellVal  cv = {0};
+    if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) {
+      break;
+    }
+
+    colVal.cid = pCol->colId;
+    colVal.type = pCol->type;
+    if (tdValTypeIsNone(cv.valType)) {
+      colVal.flag = CV_FLAG_NONE;
+    } else if (tdValTypeIsNull(cv.valType)) {
+      colVal.flag = CV_FLAG_NULL;
+    } else {
+      colVal.flag = CV_FLAG_VALUE;
+
+      if (IS_VAR_DATA_TYPE(pCol->type)) {
+        colVal.value.nData = varDataLen(cv.val);
+        colVal.value.pData = (uint8_t *)varDataVal(cv.val);
+      } else {
+        memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes);
+      }
+    }
+    checkSColVal(data[i], &colVal, pCol->type);
+  }
 }
 
 TEST(testCase, AllNormTest) {
-  int16_t       nCols = 14;
+  int16_t nCols = 14;
-  STSRowBuilder rb = {0};
+  STSRow *row = nullptr;
-  STSRow2      *row = nullptr;
+  SArray *pArray = taosArrayInit(nCols, sizeof(SColVal));
-  SArray       *pArray = taosArrayInit(nCols, sizeof(SColVal));
   EXPECT_NE(pArray, nullptr);
 
   STSchema *pTSchema = genSTSchema(nCols);
   EXPECT_NE(pTSchema, nullptr);
 
-  // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
+  // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
-  // c9 bool
+  // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
-  char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"};
+  char *data[14] = {
+      "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"};
 
   genTestData((const char **)&data, nCols, &pArray);
 
-  tTSRowNew(&rb, pArray, pTSchema, &row);
+  tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP);
-
   debugPrintTSRow(row, pTSchema, __func__, __LINE__);
+  tdSRowPrint(row, pTSchema, __func__);
   checkTSRow((const char **)&data, row, pTSchema);
 
-  tsRowBuilderClear(&rb);
+  taosMemoryFreeClear(row);
   taosArrayDestroy(pArray);
   taosMemoryFree(pTSchema);
 }
@@ -433,7 +479,7 @@ TEST(testCase, AllNormTest) {
 TEST(testCase, NoneTest) {
   const static int nCols = 14;
   const static int nRows = 20;
-  STSRow2         *row = nullptr;
+  STSRow          *row = nullptr;
   SArray          *pArray = taosArrayInit(nCols, sizeof(SColVal));
   EXPECT_NE(pArray, nullptr);
 
@@ -442,42 +488,46 @@ TEST(testCase, NoneTest) {
 
   // ts timestamp,  c1 int,  c2 bigint,  c3 float,  c4 double,  c5 binary(10),  c6 nchar(10),  c7 tinyint,  c8 smallint,
   // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
-  const char *data[nRows][nCols] = {
+  const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV,
-      {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
+                                 TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV,
-      {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+                                 TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP};
-      {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
-      {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
-      {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
-      {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
-      {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
-      {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
-      {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
-      {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
-      {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
-      {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
-      {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
-      {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
-       "18446744073709551615"},
-      {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
-      {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
-      {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
-       "4294967295", "100"},
-      {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
-       "no"}};
 
-  for (int r = 0; r < nRows; ++r) {
+const char  *data[nRows][nCols] = {
-    genTestData((const char **)&data[r], nCols, &pArray);
+    {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
-    tTSRowNew(NULL, pArray, pTSchema, &row);
+    {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-    debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
+    {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-    checkTSRow((const char **)&data[r], row, pTSchema);  // check
+    {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
-    tTSRowFree(row);
+    {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
-    taosArrayClear(pArray);
+    {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+    {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+    {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
+    {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+    {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+    {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
+    {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
+    {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
+    {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
+    {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
+    {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
+      "18446744073709551615"},
+    {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+    {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
+    {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
+      "4294967295", "100"},
+    {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
+      "no"}};
+
+for (int r = 0; r < nRows; ++r) {
+  genTestData((const char **)&data[r], nCols, &pArray);
+  tdSTSRowNew(pArray, pTSchema, &row, rowType[r]);
+  debugPrintTSRow(row, pTSchema, __func__, __LINE__);  // debug print
+  tdSRowPrint(row, pTSchema, __func__);
+  checkTSRow((const char **)&data[r], row, pTSchema);  // check
+  taosMemoryFreeClear(row);
+  taosArrayClear(pArray);
   }
 
   taosArrayDestroy(pArray);
   taosMemoryFree(pTSchema);
 }
-#endif
 #endif

source/dnode/mgmt/mgmt_dnode/inc/dmInt.h

@@ -43,6 +43,7 @@ typedef struct SDnodeMgmt {
   GetMnodeLoadsFp        getMnodeLoadsFp;
   GetQnodeLoadsFp        getQnodeLoadsFp;
   int32_t                statusSeq;
+  SendMonitorReportFp    sendMonitorReportFpBasic;
 } SDnodeMgmt;
 
 // dmHandle.c

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "dmInt.h"
 #include "systable.h"
-#include "tgrant.h"
 
 extern SConfig *tsCfg;
 
@@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   req.memTotal = tsTotalMemoryKB * 1024;
   req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024;
   tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN);
-  char *machine = tGetMachineId();
+  tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1);
-  if (machine) {
-    tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1);
-    taosMemoryFreeClear(machine);
-  }
 
   req.clusterCfg.statusInterval = tsStatusInterval;
   req.clusterCfg.checkTime = 0;
@@ -365,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
-
+#if 0
   if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
     terrno = TSDB_CODE_MND_NO_RIGHTS;
     return -1;
   }
-
+#endif
   if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;

source/dnode/mgmt/mgmt_dnode/src/dmInt.c

@@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   pMgmt->processDropNodeFp = pInput->processDropNodeFp;
   pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
   pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
+  pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic;
   pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
   pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
   pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;

source/dnode/mgmt/mgmt_dnode/src/dmWorker.c

@@ -75,6 +75,7 @@ static void *dmNotifyThreadFp(void *param) {
 static void *dmMonitorThreadFp(void *param) {
   SDnodeMgmt *pMgmt = param;
   int64_t     lastTime = taosGetTimestampMs();
+  int64_t     lastTimeForBasic = taosGetTimestampMs();
   setThreadName("dnode-monitor");
 
   static int32_t TRIM_FREQ = 20;
@@ -85,6 +86,7 @@ static void *dmMonitorThreadFp(void *param) {
     if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
 
     int64_t curTime = taosGetTimestampMs();
+
     if (curTime < lastTime) lastTime = curTime;
     float interval = (curTime - lastTime) / 1000.0f;
     if (interval >= tsMonitorInterval) {
@@ -96,6 +98,15 @@ static void *dmMonitorThreadFp(void *param) {
         taosMemoryTrim(0);
       }
     }
+    
+    if(tsMonitorForceV2){
+      if (curTime < lastTimeForBasic) lastTimeForBasic = curTime;
+      float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f;
+      if (intervalForBasic >= tsMonitorIntervalForBasic) {
+        (*pMgmt->sendMonitorReportFpBasic)();
+        lastTimeForBasic = curTime;
+      }
+    }
   }
 
   return NULL;

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -191,6 +191,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_snode/src/smHandle.c

@@ -87,8 +87,10 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
 
   code = 0;
 _OVER:

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -835,9 +835,11 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/node_mgmt/CMakeLists.txt

@@ -1,7 +1,7 @@
 aux_source_directory(src IMPLEMENT_SRC)
 add_library(dnode STATIC ${IMPLEMENT_SRC})
 target_link_libraries(
-    dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode
+    dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
 )
 
 IF (TD_STORAGE)

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -128,6 +128,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 // dmMonitor.c
 void dmSendMonitorReport();
 void dmSendAuditRecords();
+void dmSendMonitorReportBasic();
 void dmGetVnodeLoads(SMonVloadInfo *pInfo);
 void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
 void dmGetMnodeLoads(SMonMloadInfo *pInfo);

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -394,6 +394,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
       .processDropNodeFp = dmProcessDropNodeReq,
       .sendMonitorReportFp = dmSendMonitorReport,
       .sendAuditRecordFp = auditSendRecordsInBatch,
+      .sendMonitorReportFpBasic = dmSendMonitorReportBasic,
       .getVnodeLoadsFp = dmGetVnodeLoads,
       .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
       .getMnodeLoadsFp = dmGetMnodeLoads,

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -22,6 +22,7 @@
 #ifdef TD_TSZ
 #include "tcompression.h"
 #include "tglobal.h"
+#include "tgrant.h"
 #endif
 
 static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
@@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) {
   pData->rebootTime = taosGetTimestampMs();
   pData->dropped = 0;
   pData->stopped = 0;
+  char *machineId = tGetMachineId();
+  if (machineId) {
+    tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1);
+    taosMemoryFreeClear(machineId);
+  } else {
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
+    terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
+    return -1;
+#endif
+  }
 
   pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   if (pData->dnodeHash == NULL) {

source/dnode/mgmt/node_mgmt/src/dmMonitor.c

@@ -25,8 +25,16 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
 }
 
+static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) {
+  pInfo->protocol = 2;
+  pInfo->dnode_id = pDnode->data.dnodeId;
+  pInfo->cluster_id = pDnode->data.clusterId;
+  tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+}
+
 static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
-  pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+  //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+  pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f;
   pInfo->has_mnode = pDnode->wrappers[MNODE].required;
   pInfo->has_qnode = pDnode->wrappers[QNODE].required;
   pInfo->has_snode = pDnode->wrappers[SNODE].required;
@@ -44,6 +52,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) {
   monSetDmInfo(&dmInfo);
 }
 
+static void dmGetDmMonitorInfoBasic(SDnode *pDnode) {
+  SMonDmInfo dmInfo = {0};
+  dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic);
+  monSetDmInfo(&dmInfo);
+}
+
 static void dmGetMmMonitorInfo(SDnode *pDnode) {
   SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE];
   if (dmMarkWrapper(pWrapper) == 0) {
@@ -106,7 +120,17 @@ void dmSendMonitorReport() {
   dmGetVmMonitorInfo(pDnode);
   dmGetQmMonitorInfo(pDnode);
   dmGetSmMonitorInfo(pDnode);
-  monSendReport();
+  monGenAndSendReport();
+}
+
+void dmSendMonitorReportBasic() {
+  if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+  dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+  SDnode *pDnode = dmInstance();
+  dmGetDmMonitorInfoBasic(pDnode);
+  dmGetMmMonitorInfo(pDnode);
+  monGenAndSendReportBasic();
 }
 
 //Todo: put this in seperate file in the future

source/dnode/mgmt/node_util/CMakeLists.txt

@@ -6,5 +6,5 @@ target_include_directories(
     PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor
+    node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw
 )

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -109,6 +109,7 @@ typedef struct {
   SMsgCb         msgCb;
   bool           validMnodeEps;
   int64_t        ipWhiteVer;
+  char           machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeData;
 
 typedef struct {
@@ -122,6 +123,7 @@ typedef struct {
   ProcessDropNodeFp   processDropNodeFp;
   SendMonitorReportFp sendMonitorReportFp;
   SendAuditRecordsFp  sendAuditRecordFp;
+  SendMonitorReportFp sendMonitorReportFpBasic;
   GetVnodeLoadsFp     getVnodeLoadsFp;
   GetVnodeLoadsFp     getVnodeLoadsLiteFp;
   GetMnodeLoadsFp     getMnodeLoadsFp;

source/dnode/mnode/impl/CMakeLists.txt

@@ -23,7 +23,7 @@ target_include_directories(
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit
+    mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw
 )
 
 IF (TD_GRANT)

source/dnode/mnode/impl/inc/mndDef.h

@@ -464,6 +464,7 @@ typedef struct {
   char*    pAst1;
   char*    pAst2;
   SRWLatch lock;
+  int8_t   source;
 } SStbObj;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndIndex.h

@@ -19,16 +19,16 @@ int32_t  mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool
 int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq);
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MND_IDX_H_*/
+#endif /*_TD_MND_IDX_H_*/

source/dnode/mnode/impl/inc/mndStream.h

@@ -86,6 +86,10 @@ typedef struct SOrphanTask {
   int32_t nodeId;
 } SOrphanTask;
 
+typedef struct {
+  SMsgHead head;
+} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg;
+
 int32_t     mndInitStream(SMnode *pMnode);
 void        mndCleanupStream(SMnode *pMnode);
 SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);

source/dnode/mnode/impl/src/mndDb.c

@@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionDelete(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew);
-static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t  mndProcessCreateDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessAlterDbReq(SRpcMsg *pReq);
@@ -256,17 +256,29 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
+static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
-  SDbObj *pNewDb = pObj;
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  SDbObj  *pNewDb = NULL;
+  int      code = -1;
+
+  pRow = mndDbActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  pNewDb = sdbGetRowObj(pRow);
+  if (pNewDb == NULL) goto _OVER;
 
   SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name);
   if (pOldDb != NULL) {
     mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name);
     sdbRelease(pMnode->pSdb, pOldDb);
-    return -1;
+    goto _OVER;
   }
 
-  return 0;
+  code = 0;
+_OVER:
+  if (pNewDb) mndDbActionDelete(pSdb, pNewDb);
+  taosMemoryFreeClear(pRow);
+  return code;
 }
 
 static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) {
@@ -884,10 +896,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   return terrno;
 }
 
-static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
+static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -943,7 +955,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
   mndTransSetDbName(pTrans, pOld->name, NULL);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
+  if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -1120,10 +1132,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
+static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -1257,7 +1269,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
     goto _OVER;
   }
 
-  if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/

source/dnode/mnode/impl/src/mndDnode.c

@@ -14,6 +14,8 @@
  */
 
 #define _DEFAULT_SOURCE
+#include <stdio.h>
+#include "tjson.h"
 #include "mndDnode.h"
 #include "audit.h"
 #include "mndCluster.h"
@@ -28,6 +30,7 @@
 #include "mndVgroup.h"
 #include "tmisce.h"
 #include "tunit.h"
+#include "taos_monitor.h"
 
 #define TSDB_DNODE_VER_NUMBER   2
 #define TSDB_DNODE_RESERVE_SIZE 64
@@ -74,6 +77,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp);
 static int32_t mndProcessStatusReq(SRpcMsg *pReq);
 static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
 static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
+static int32_t mndProcessStatisReq(SRpcMsg *pReq);
 
 static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
@@ -109,6 +113,7 @@ int32_t mndInitDnode(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq);
   mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
   mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig);
@@ -141,7 +146,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
     memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN);
     taosMemoryFreeClear(machineId);
   } else {
-#ifdef TD_ENTERPRISE
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
     terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
     goto _OVER;
 #endif
@@ -410,9 +415,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
     dInfo.ep.port = pDnode->port;
     dInfo.offlineReason = pDnode->offlineReason;
     tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-    tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1);
     sdbRelease(pSdb, pDnode);
     if (mndIsMnode(pMnode, pDnode->id)) {
       dInfo.isMnode = 1;
@@ -502,6 +504,207 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) {
   return stateChanged;
 }
 
+static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
+  SMnode    *pMnode = pReq->info.node;
+  SStatisReq statisReq = {0};
+  int32_t    code = -1;
+
+  char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
+  sprintf(strClusterId, "%"PRId64, pMnode->clusterId);
+
+  if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return code;
+  }
+
+  if(tsMonitorLogProtocol){
+    mInfo("process statis req,\n %s", statisReq.pCont);
+  }
+
+  SJson* pJson = tjsonParse(statisReq.pCont);
+
+  int32_t ts_size = tjsonGetArraySize(pJson);
+
+  for(int32_t i = 0; i < ts_size; i++){
+    SJson* item = tjsonGetArrayItem(pJson, i);
+
+    SJson* tables = tjsonGetObjectItem(item, "tables");
+
+    int32_t tableSize = tjsonGetArraySize(tables);
+    for(int32_t i = 0; i < tableSize; i++){
+      SJson* table = tjsonGetArrayItem(tables, i);
+
+      char tableName[MONITOR_TABLENAME_LEN] = {0};
+      tjsonGetStringValue(table, "name", tableName);
+
+      SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+      int32_t size = tjsonGetArraySize(metricGroups);
+      for(int32_t i = 0; i < size; i++){
+        SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+        SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+        int32_t tagSize = tjsonGetArraySize(arrayTag);
+        for(int32_t j = 0; j < tagSize; j++){
+          SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+          char tagName[MONITOR_TAG_NAME_LEN] = {0};
+          tjsonGetStringValue(item, "name", tagName);
+
+          if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+            tjsonDeleteItemFromObject(item, "value");
+            tjsonAddStringToObject(item, "value", strClusterId);
+          }
+        }
+      }
+    }
+  }
+
+  char *pCont = tjsonToString(pJson);
+  monSendContent(pCont);
+
+  if(pJson != NULL){
+    tjsonDelete(pJson);
+    pJson = NULL;
+  }
+
+  if(pCont != NULL){
+    taosMemoryFree(pCont);
+    pCont = NULL;
+  }
+
+  tFreeSStatisReq(&statisReq);
+  return 0;
+
+/*
+  SJson* pJson = tjsonParse(statisReq.pCont);
+
+  int32_t ts_size = tjsonGetArraySize(pJson);
+
+  for(int32_t i = 0; i < ts_size; i++){
+    SJson* item = tjsonGetArrayItem(pJson, i);
+
+    SJson* tables = tjsonGetObjectItem(item, "tables");
+
+    int32_t tableSize = tjsonGetArraySize(tables);
+    for(int32_t i = 0; i < tableSize; i++){
+      SJson* table = tjsonGetArrayItem(tables, i);
+
+      char tableName[MONITOR_TABLENAME_LEN] = {0};
+      tjsonGetStringValue(table, "name", tableName);
+
+      SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+      int32_t size = tjsonGetArraySize(metricGroups);
+      for(int32_t i = 0; i < size; i++){
+        SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+        SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+        int32_t tagSize = tjsonGetArraySize(arrayTag);
+
+        char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+        char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+
+        for(int32_t j = 0; j < tagSize; j++){
+          SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+          *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); 
+          tjsonGetStringValue(item, "name", *(labels + j));
+
+          *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
+          tjsonGetStringValue(item, "value", *(sample_labels + j));
+          if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+            strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
+          }
+        }
+
+        SJson* metrics = tjsonGetObjectItem(item, "metrics");
+
+        int32_t metricLen = tjsonGetArraySize(metrics);
+        for(int32_t j = 0; j < metricLen; j++){
+          SJson *item = tjsonGetArrayItem(metrics, j);
+
+          char name[MONITOR_METRIC_NAME_LEN] = {0}; 
+          tjsonGetStringValue(item, "name", name);
+
+          double value = 0;
+          tjsonGetDoubleValue(item, "value", &value);
+
+          double type = 0;
+          tjsonGetDoubleValue(item, "type", &type);
+
+          int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
+          char* metricName = taosMemoryMalloc(metricNameLen); 
+          memset(metricName, 0, metricNameLen);
+          sprintf(metricName, "%s:%s", tableName, name);
+
+          taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
+          if(metric == NULL){
+            if(type == 0){
+              metric = taos_counter_new(metricName, "",  tagSize, (const char**)labels);
+            }
+            if(type == 1){
+              metric = taos_gauge_new(metricName, "",  tagSize, (const char**)labels);
+            }
+            mTrace("fail to get metric from registry, new one metric:%p", metric);
+
+            if(taos_collector_registry_register_metric(metric) == 1){
+              if(type == 0){
+                taos_counter_destroy(metric);
+              }
+              if(type == 1){
+                taos_gauge_destroy(metric);
+              }
+
+              metric = taos_collector_registry_get_metric(metricName);
+
+              mTrace("fail to register metric, get metric from registry:%p", metric);
+            }
+            else{
+              mTrace("succeed to register metric:%p", metric);
+            }
+          }
+          else{
+            mTrace("get metric from registry:%p", metric);
+          }
+          
+          if(type == 0){
+            taos_counter_add(metric, value, (const char**)sample_labels);
+          }
+          if(type == 1){
+            taos_gauge_set(metric, value, (const char**)sample_labels);
+          }
+
+          taosMemoryFreeClear(metricName);
+        }
+
+        for(int32_t j = 0; j < tagSize; j++){
+          taosMemoryFreeClear(*(labels + j));
+          taosMemoryFreeClear(*(sample_labels + j));
+        }
+
+        taosMemoryFreeClear(sample_labels);
+        taosMemoryFreeClear(labels);
+      }
+    }
+    
+  }
+
+  code = 0;
+
+_OVER:
+  if(pJson != NULL){
+    tjsonDelete(pJson);
+    pJson = NULL;
+  }
+
+  tFreeSStatisReq(&statisReq);
+  return code;
+  */
+}
+
 static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
   SMnode    *pMnode = pReq->info.node;
   SStatusReq statusReq = {0};

source/dnode/mnode/impl/src/mndIndex.c

@@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
   return mndAcquireDb(pMnode, db);
 }
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
 
   return 0;
@@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx)
   return 0;
 }
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -482,10 +482,10 @@ _OVER:
   return code;
 }
 
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
 
   mndTransSetSerial(pTrans);
 
-  if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
@@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
   mndTransSetSerial(pTrans);
-  if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndMain.c

@@ -911,11 +911,14 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
     if (pObj->id == pMnode->selfDnodeId) {
       pClusterInfo->first_ep_dnode_id = pObj->id;
       tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
-      pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+      //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+      pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
       // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
       tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));
+      desc.syncState = TAOS_SYNC_STATE_LEADER;
     } else {
       tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role));
+      desc.syncState = pObj->syncState;
     }
     taosArrayPush(pClusterInfo->mnodes, &desc);
     sdbRelease(pSdb, pObj);
@@ -946,6 +949,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
       SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
       pVnDesc->dnode_id = pVgid->dnodeId;
       tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role));
+      pVnDesc->syncState = pVgid->syncState;
       if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
         tstrncpy(desc.status, "ready", sizeof(desc.status));
         pClusterInfo->vgroups_alive++;
@@ -985,8 +989,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
   pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000;
   pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
   if (pMnode->grant.expireTimeMS == 0) {
-    pGrantInfo->expire_time = INT32_MAX;
+    pGrantInfo->expire_time = 0;
-    pGrantInfo->timeseries_total = INT32_MAX;
+    pGrantInfo->timeseries_total = 0;
   }
 
   mndReleaseRpc(pMnode);

source/dnode/mnode/impl/src/mndQnode.c

@@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
   if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
@@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
   if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -300,15 +300,16 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet*
 }
 
 static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
-  for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+  int32_t size = (int32_t) taosArrayGetSize(pList);
+  for (int32_t i = 0; i < size; ++i) {
     SVgroupVer* pVer = taosArrayGet(pList, i);
     if (pVer->vgId == vgId) {
       return pVer->ver;
     }
   }
 
-  mError("failed to find the vgId:%d for extract last version", vgId);
+  mDebug("no data in vgId:%d for extract last version, set to be 0, total existed vgs:%d", vgId, size);
-  return -1;
+  return 1;
 }
 
 static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) {
@@ -472,6 +473,9 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
     int code =
         doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
     if (code != 0) {
+      mError("create stream task, code:%s", tstrerror(code));
+
+      // todo drop the added source tasks.
       sdbRelease(pSdb, pVgroup);
       return code;
     }

source/dnode/mnode/impl/src/mndSnode.c

@@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
 
@@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
   if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndStb.c

@@ -458,6 +458,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
   req.rollup = pStb->ast1Len > 0 ? 1 : 0;
   req.alterOriData = alterOriData;
   req.alterOriDataLen = alterOriDataLen;
+  req.source = pStb->source;
   // todo
   req.schemaRow.nCols = pStb->numOfColumns;
   req.schemaRow.version = pStb->colVer;
@@ -616,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
   return 0;
 }
 
-static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -628,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
-  SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
-  if (pUndoRaw == NULL) return -1;
-  if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
-    sdbFreeRaw(pUndoRaw);
-    return -1;
-  }
-  if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
-
-  return 0;
-}
-
 static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
   if (pCommitRaw == NULL) return -1;
@@ -774,7 +763,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   pDst->createdTime = taosGetTimestampMs();
   pDst->updateTime = pDst->createdTime;
   pDst->uid =
-      (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+      (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
+          ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
   pDst->tagVer = 1;
   pDst->colVer = 1;
@@ -790,6 +780,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   pDst->numOfFuncs = pCreate->numOfFuncs;
   pDst->commentLen = pCreate->commentLen;
   pDst->pFuncs = pCreate->pFuncs;
+  pDst->source = pCreate->source;
   pCreate->pFuncs = NULL;
 
   if (pDst->commentLen > 0) {
@@ -910,8 +901,6 @@ _OVER:
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1;
-  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
-  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
@@ -1033,6 +1022,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
   memcpy(pDst, pStb, sizeof(SStbObj));
   taosRUnLockLatch(&pStb->lock);
 
+  pDst->source = createReq->source;
   pDst->updateTime = taosGetTimestampMs();
   pDst->numOfColumns = createReq->numOfColumns;
   pDst->numOfTags = createReq->numOfTags;
@@ -1141,7 +1131,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     }
   } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
     goto _OVER;
-  } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)) {
+  } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) {
     mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
     code = 0;
     goto _OVER;
@@ -1742,10 +1732,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
   return 0;
 }
 
-static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2151,7 +2141,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb
     mndTransSetRpcRsp(pTrans, pCont, contLen);
   }
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -2188,11 +2178,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
     if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) {
       exist = true;
     }
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
-      if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2211,13 +2201,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
       exist = true;
     }
 
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
       memcpy(idxObj.colName, nTagName, strlen(nTagName));
       idxObj.colName[strlen(nTagName)] = 0;
-      if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2350,10 +2340,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
+static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2423,7 +2413,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
+  if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
@@ -2572,7 +2562,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
-  if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) {
+  if ((dropReq.source == TD_REQ_FROM_TAOX_OLD || dropReq.source == TD_REQ_FROM_TAOX) && pStb->uid != dropReq.suid) {
     code = 0;
     goto _OVER;
   }
@@ -3543,7 +3533,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndStream.c

@@ -721,6 +721,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  taosThreadMutexLock(&execInfo.lock);
+  mDebug("stream stream:%s tasks register into node list", createReq.name);
+  saveStreamTasksInfo(&streamObj, &execInfo);
+  taosThreadMutexUnlock(&execInfo.lock);
+
   // execute creation
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -730,12 +735,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   mndTransDrop(pTrans);
 
-  taosThreadMutexLock(&execInfo.lock);
-
-  mDebug("stream tasks register into node list");
-  saveStreamTasksInfo(&streamObj, &execInfo);
-  taosThreadMutexUnlock(&execInfo.lock);
-
   SName dbname = {0};
   tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
@@ -878,7 +877,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
   int64_t ts = taosGetTimestampMs();
   if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) {
     //    mWarn("checkpoint interval less than the threshold, ignore it");
-    return -1;
+    return TSDB_CODE_SUCCESS;
   }
 
   bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
@@ -2180,5 +2179,16 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
   mndReleaseStream(pMnode, pStream);
   taosThreadMutexUnlock(&execInfo.lock);
 
+  {
+    SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
+    rsp.pCont = rpcMallocCont(rsp.contLen);
+    SMsgHead* pHead = rsp.pCont;
+    pHead->vgId = htonl(req.nodeId);
+
+    tmsgSendRsp(&rsp);
+
+    pReq->info.handle = NULL;   // disable auto rsp
+  }
+
   return 0;
 }

source/dnode/mnode/impl/src/mndStreamHb.c

@@ -16,10 +16,6 @@
 #include "mndStream.h"
 #include "mndTrans.h"
 
-typedef struct {
-  SMsgHead head;
-} SMStreamHbRspMsg;
-
 typedef struct SFailedCheckpointInfo {
   int64_t streamUid;
   int64_t checkpointId;

source/dnode/mnode/impl/src/mndSync.c

@@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
 static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
   SSdbRaw *pRaw = pAction->pRaw;
   SSdb    *pSdb = pMnode->pSdb;
-  SSdbRow *pRow = NULL;
+  int      code = 0;
-  void    *pObj = NULL;
-  int      code = -1;
 
   if (pRaw->status != SDB_STATUS_CREATING) goto _OUT;
 
-  pRow = (pSdb->decodeFps[pRaw->type])(pRaw);
-  if (pRow == NULL) goto _OUT;
-  pObj = sdbGetRowObj(pRow);
-  if (pObj == NULL) goto _OUT;
-
   SdbValidateFp validateFp = pSdb->validateFps[pRaw->type];
-  code = 0;
   if (validateFp) {
-    code = validateFp(pMnode, pTrans, pObj);
+    code = validateFp(pMnode, pTrans, pRaw);
   }
 
 _OUT:
-  if (pRow) {
-    SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
-    if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
-    taosMemoryFreeClear(pRow);
-  }
   return code;
 }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
         pTrans->createdTime);
   int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id);
   if (code != 0) {
-    mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
+    mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
-           tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId);
+           code, pTrans->createdTime, pMnode->syncMgmt.transId);
     sdbFreeRaw(pRaw);
     return -1;
   }
@@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
 
 static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
   if (pAction->rawWritten) return 0;
-  if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+  if (topHalf) {
+    terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH;
+    return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+  }
 
   int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw);
   if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@@ -1214,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
   if (numOfActions == 0) return 0;
 
   if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) {
-    return -1;
+    return code;
   }
 
   int32_t       numOfExecuted = 0;
@@ -1419,7 +1422,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool
     pTrans->stage = TRN_STAGE_COMMIT;
     mInfo("trans:%d, stage from redoAction to commit", pTrans->id);
     continueExec = true;
-  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) {
     mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code));
     continueExec = false;
   } else {

source/dnode/mnode/impl/src/mndVgroup.c

@@ -31,10 +31,10 @@
 #define VGROUP_VER_NUMBER   1
 #define VGROUP_RESERVE_SIZE 64
 
-static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
+static int32_t  mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@@ -181,15 +181,28 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
+static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
-  SVgObj *pVgroup = pObj;
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  SVgObj  *pVgroup = NULL;
+  int      code = -1;
+
+  pRow = mndVgroupActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  pVgroup = sdbGetRowObj(pRow);
+  if (pVgroup == NULL) goto _OVER;
 
   int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
   if (maxVgId > pVgroup->vgId) {
     mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId);
-    return -1;
+    goto _OVER;
   }
-  return 0;
+
+  code = 0;
+_OVER:
+  if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup);
+  taosMemoryFreeClear(pRow);
+  return code;
 }
 
 static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) {

source/dnode/mnode/sdb/inc/sdb.h

@@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
 typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
-typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj);
+typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
 typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3);

source/dnode/snode/src/snode.c

@@ -180,6 +180,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
       return tqStreamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg);
     case TDMT_MND_STREAM_HEARTBEAT_RSP:
       return tqStreamProcessStreamHbRsp(pSnode->pMeta, pMsg);
+    case TDMT_MND_STREAM_REQ_CHKPT_RSP:
+      return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg);
+    case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+      return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg);
     default:
       sndError("invalid snode msg:%d", pMsg->msgType);
       ASSERT(0);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -47,6 +47,8 @@
 
 #include "vnode.h"
 
+#include "taos_monitor.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -104,6 +106,20 @@ typedef struct SQueryNode         SQueryNode;
 #define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
+#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count"
+
+#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
+#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
+#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
+#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
+#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
+#define VNODE_METRIC_TAG_NAME_USERNAME "username"
+#define VNODE_METRIC_TAG_NAME_RESULT "result"
+
+#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
+//#define VNODE_METRIC_TAG_VALUE_INSERT "insert"
+//#define VNODE_METRIC_TAG_VALUE_DELETE "delete"
+
 // vnd.h
 typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
 struct SQueryNode {
@@ -243,6 +259,8 @@ int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
@@ -443,6 +461,13 @@ typedef struct SVCommitSched {
   int64_t maxWaitMs;
 } SVCommitSched;
 
+typedef struct SVMonitorObj{
+  char strClusterId[TSDB_CLUSTER_ID_LEN];
+  char strDnodeId[TSDB_NODE_ID_LEN];
+  char strVgId[TSDB_VGROUP_ID_LEN];
+  taos_counter_t *insertCounter;
+}SVMonitorObj;
+
 struct SVnode {
   char*     path;
   SVnodeCfg config;
@@ -481,6 +506,7 @@ struct SVnode {
   int32_t       blockSec;
   int64_t       blockSeq;
   SQHandle*     pQuery;
+  SVMonitorObj  monitor;
 };
 
 #define TD_VID(PVNODE) ((PVNODE)->config.vgId)

source/dnode/vnode/src/meta/metaTable.c

@@ -1008,11 +1008,11 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
       }
       tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t));
     }
-
+    /*
     if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
       tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL);
     }
-
+    */
     metaDebug("batch drop table:%" PRId64, uid);
   }
   metaULock(pMeta);

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
       continue;
     }
 
-    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL};
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
 
     int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
     tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);

source/dnode/vnode/src/tq/tq.c

@@ -1175,7 +1175,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
       return TSDB_CODE_SUCCESS;
     }
   } else {
-    ASSERT(status == TASK_STATUS__HALT);
+//    ASSERT(status == TASK_STATUS__HALT);
+    if (status != TASK_STATUS__HALT) {
+      tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr);
+//      streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
+    }
   }
 
   // check if the checkpoint msg already sent or not.
@@ -1225,3 +1229,11 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
 int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) {
   return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg);
 }
+
+int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) {
+  return tqStreamProcessReqCheckpointRsp(pTq->pStreamMeta, pMsg);
+}
+
+int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) {
+  return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg);
+}

source/dnode/vnode/src/tq/tqRead.c

@@ -392,7 +392,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
               pReader->msg.ver);
 
       SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
-      if ((pSubmitTbData->source & sourceExcluded) != 0) {
+      if ((pSubmitTbData->flags & sourceExcluded) != 0) {
         pReader->nextBlk += 1;
         continue;
       }

source/dnode/vnode/src/tq/tqScan.c

@@ -267,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table;
       }
 
-      if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+      if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
         goto loop_table;
       }
       if (pRsp->withTbName) {
@@ -335,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db;
       }
 
-      if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+      if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
         goto loop_db;
       }
       if (pRsp->withTbName) {

source/dnode/vnode/src/tq/tqSink.c

@@ -72,7 +72,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
       name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
       memcpy(name, varDataVal(varTbName), varDataLen(varTbName));
       if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0) {
-        buildCtbNameAddGruopId(name, groupId);
+        buildCtbNameAddGroupId(name, groupId);
       }
     } else if (stbFullName) {
       name = buildCtbNameByGroupId(stbFullName, groupId);
@@ -185,23 +185,26 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
         !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0) {
       pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
       strcpy(pCreateTableReq->name, pDataBlock->info.parTbName);
-      buildCtbNameAddGruopId(pCreateTableReq->name, gid);
+      buildCtbNameAddGroupId(pCreateTableReq->name, gid);
+//      tqDebug("gen name from:%s", pDataBlock->info.parTbName);
     } else {
       pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
+//      tqDebug("copy name:%s", pDataBlock->info.parTbName);
     }
   } else {
     pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
+//    tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid);
   }
 }
 
 static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
                                             SStreamTask* pTask, int64_t suid) {
-  tqDebug("s-task:%s build create table msg", pTask->id.idStr);
-
   STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
   int32_t   rows = pDataBlock->info.rows;
   SArray*   tagArray = taosArrayInit(4, sizeof(STagVal));
-  ;
+
+  tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows);
+
   int32_t code = 0;
 
   SVCreateTbBatchReq reqs = {0};
@@ -670,7 +673,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
     } else {
       if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 &&
           !isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) {
-        buildCtbNameAddGruopId(dstTableName, groupId);
+        tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName);
+        buildCtbNameAddGroupId(dstTableName, groupId);
       }
     }
 
@@ -815,7 +819,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           return;
         }
 
-        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
         code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
@@ -860,7 +864,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
       pTask->execInfo.sink.numOfBlocks += 1;
       uint64_t groupId = pDataBlock->info.id.groupId;
 
-      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
 
       int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
       if (index == NULL) {  // no data yet, append it

source/dnode/vnode/src/tq/tqUtil.c

@@ -171,6 +171,22 @@ end : {
 }
 }
 
+#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \
+  SDecoder           decoder = {0};\
+  TYPE               req = {0}; \
+  void*   data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
+  int32_t len = pHead->bodyLen - sizeof(SMsgHead); \
+  tDecoderInit(&decoder, data, len); \
+  if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \
+    tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d",  \
+            pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
+    fetchVer++; \
+    tDecoderClear(&decoder); \
+    continue; \
+  } \
+  tDecoderClear(&decoder);
+
+
 static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                   SRpcMsg* pMsg, STqOffsetVal* offset) {
   int        code = 0;
@@ -239,6 +255,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
           goto end;
         }
 
+        if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
+          if (pHead->msgType == TDMT_VND_CREATE_TABLE) {
+            PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq)
+          } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) {
+            PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq)
+          } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
+            PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
+          } else if (pHead->msgType == TDMT_VND_DELETE) {
+            fetchVer++;
+            continue;
+          }
+        }
+
         tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
         tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
         metaRsp.resMsgType = pHead->msgType;

source/dnode/vnode/src/tqCommon/tqCommon.c

@@ -485,6 +485,10 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
   return code;
 }
 
+typedef struct SMStreamCheckpointReadyRspMsg {
+  SMsgHead head;
+}SMStreamCheckpointReadyRspMsg;
+
 int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {
   int32_t vgId = pMeta->vgId;
   char*   msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@@ -513,6 +517,18 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
 
   streamProcessCheckpointReadyMsg(pTask);
   streamMetaReleaseTask(pMeta, pTask);
+
+  {  // send checkpoint ready rsp
+    SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)};
+    rsp.pCont = rpcMallocCont(rsp.contLen);
+    SMsgHead* pHead = rsp.pCont;
+    pHead->vgId = htonl(req.downstreamNodeId);
+
+    tmsgSendRsp(&rsp);
+
+    pMsg->info.handle = NULL;  // disable auto rsp
+  }
+
   return code;
 }
 
@@ -938,9 +954,17 @@ int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta) {
   return taosArrayGetSize(pMeta->pTaskList);
 }
 
-int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {
   rpcFreeCont(pMsg->pCont);
   pMsg->pCont = NULL;
 
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+
+int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+
+int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+  return doProcessDummyRspMsg(pMeta, pMsg);
+}

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -448,8 +448,8 @@ _exit:
     taosMemoryFree(reader[0]);
     reader[0] = NULL;
   } else {
-    tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
+    tsdbInfo("vgId:%d, tsdb snapshot incremental reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d",
-             sver, ever, type);
+             TD_VID(tsdb->pVnode), sver, ever, type);
   }
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c

@@ -69,7 +69,8 @@ _exit:
     taosMemoryFree(reader[0]);
     reader[0] = NULL;
   } else {
-    tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type);
+    tsdbInfo("vgId:%d, tsdb snapshot raw reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever,
+             type);
   }
   return code;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -480,6 +480,28 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
     vnodeRollback(pVnode);
   }
 
+  snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
+  snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
+  snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId);
+
+  if(pVnode->monitor.insertCounter == NULL){
+    int32_t label_count = 7;
+    const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
+                                   VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
+                                   VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
+                                   VNODE_METRIC_TAG_NAME_RESULT};
+    taos_counter_t *counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql",  
+                                                label_count, sample_labels);
+    vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter);
+    if(taos_collector_registry_register_metric(counter) == 1){
+      taos_counter_destroy(counter);
+      counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
+      vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter);
+    }
+    pVnode->monitor.insertCounter = counter;
+    vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter);
+  }
+
   return pVnode;
 
 _err:

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
 #include "tsdb.h"
+#include "vnd.h"
 
 #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags)                                                    \
   do {                                                                                                        \
@@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   // decode req
   if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
     code = TSDB_CODE_INVALID_MSG;
-    goto _exit;
+    goto _exit4;
   }
 
   metaRsp.dbId = pVnode->config.dbId;
@@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
   code = vnodeValidateTableHash(pVnode, tableFName);
   if (code) {
-    goto _exit;
+    goto _exit4;
   }
 
   // query meta
@@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
 
   if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) {
     code = terrno;
-    goto _exit;
+    goto _exit3;
   }
 
   metaRsp.tableType = mer1.me.type;
@@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
     metaRsp.suid = mer1.me.uid;
   } else if (mer1.me.type == TSDB_CHILD_TABLE) {
     metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
-    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
 
     strcpy(metaRsp.stbName, mer2.me.name);
     metaRsp.suid = mer2.me.uid;
@@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
 
 _exit:
+  taosMemoryFree(metaRsp.pSchemas);
+_exit2:
+  metaReaderClear(&mer2);
+_exit3:
+  metaReaderClear(&mer1);
+_exit4:
   rpcMsg.info = pMsg->info;
   rpcMsg.pCont = pRsp;
   rpcMsg.contLen = rspLen;
@@ -141,9 +147,6 @@ _exit:
     *pMsg = rpcMsg;
   }
 
-  taosMemoryFree(metaRsp.pSchemas);
-  metaReaderClear(&mer2);
-  metaReaderClear(&mer1);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) {
 }
 
 int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
-  return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid);
+  return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
 }

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -172,6 +172,23 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
     goto _err;
   }
 
+  // open tsdb snapshot raw reader
+  if (!pReader->tsdbRAWDone) {
+    ASSERT(pReader->sver == 0);
+    code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
+    if (code) goto _err;
+  }
+
+  // check snapshot ever
+  SSnapshot snapshot = {0};
+  vnodeGetSnapshot(pVnode, &snapshot);
+  if (ever != snapshot.lastApplyIndex) {
+    vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
+           TD_VID(pVnode), ever, snapshot.lastApplyIndex);
+    code = TSDB_CODE_SYN_INTERNAL_ERROR;
+    goto _err;
+  }
+
   vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
   *ppReader = pReader;
   return code;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -30,13 +30,15 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
 static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
                                      SRpcMsg *pOriginRpc);
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, 
+                                    SRpcMsg *pOriginalMsg);
 static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, 
+                                      SRpcMsg *pOriginalMsg);
 static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@@ -542,10 +544,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
       break;
     /* TSDB */
     case TDMT_VND_SUBMIT:
-      if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
+      if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp, pMsg) < 0) goto _err;
       break;
     case TDMT_VND_DELETE:
-      if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) goto _err;
       break;
     case TDMT_VND_BATCH_DEL:
       if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
@@ -798,6 +800,10 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
       return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
     case TDMT_MND_STREAM_HEARTBEAT_RSP:
       return tqProcessStreamHbRsp(pVnode->pTq, pMsg);
+    case TDMT_MND_STREAM_REQ_CHKPT_RSP:
+      return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
+    case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+      return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
     default:
       vError("unknown msg type:%d in stream queue", pMsg->msgType);
       return TSDB_CODE_APP_ERROR;
@@ -1482,7 +1488,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
   return code;
 }
 
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, 
+                                    SRpcMsg *pOriginalMsg) {
   int32_t code = 0;
   terrno = 0;
 
@@ -1700,10 +1707,35 @@ _exit:
   atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows);
   atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows);
   atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
+
+  if(pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){
+    const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, 
+                                    pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, 
+                                    pOriginalMsg->info.conn.user, "Success"};
+    taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels);
+  }
+
   if (code == 0) {
     atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
     code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len);
   }
+  /*
+  if (code == 0) {
+    atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
+    code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len);
+
+    const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, 
+                                        pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, 
+                                          pOriginalMsg->info.conn.user, "Success"};
+    taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+  }
+  else{
+    const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, 
+                                        pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, 
+                                        pOriginalMsg->info.conn.user, "Failed"};
+    taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+  }
+  */
 
   // clear
   taosArrayDestroy(newTbUids);
@@ -1959,7 +1991,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe
   return 0;
 }
 
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, 
+                                    SRpcMsg *pOriginalMsg) {
   int32_t     code = 0;
   SDecoder   *pCoder = &(SDecoder){0};
   SDeleteRes *pRes = &(SDeleteRes){0};
@@ -2002,6 +2035,21 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in
   return code;
 
 _err:
+  /*
+  if(code == TSDB_CODE_SUCCESS){
+    const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, 
+                                        pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, 
+                                        pOriginalMsg->info.conn.user, "Success"};
+    taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+  }
+  else{
+    const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, 
+                                        pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, 
+                                        pOriginalMsg->info.conn.user, "Failed"};
+    taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+  }
+  */
+
   return code;
 }
 static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {

source/libs/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(qcom)
 add_subdirectory(nodes)
 add_subdirectory(catalog)
 add_subdirectory(audit)
+add_subdirectory(monitorfw)
 
 add_subdirectory(scalar)
 add_subdirectory(function)

source/libs/executor/src/countwindowoperator.c

@@ -129,7 +129,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
   return code;
 }
 
-static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) {
+static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) {
   SResultRow* pResultRow = NULL;
   for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) {
     SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow);
@@ -143,6 +143,7 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S
     clearWinStateBuff(pBuff);
     clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
   }
+  doFilter(pBlock, pFilterInfo, NULL);
 }
 
 static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
@@ -177,7 +178,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
     if (pInfo->groupId == 0) {
       pInfo->groupId = pBlock->info.id.groupId;
     } else if (pInfo->groupId != pBlock->info.id.groupId) {
-      buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
+      buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes);
       pInfo->groupId = pBlock->info.id.groupId;
     }
 
@@ -187,7 +188,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
     }
   }
 
-  buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
+  buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes);
   return pRes->info.rows == 0 ? NULL : pRes;
 }
 
@@ -246,6 +247,11 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
 
   pInfo->countSup.stateIndex = 0;
 
+  code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
 
   setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo,

source/libs/executor/src/projectoperator.c

@@ -461,7 +461,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
 _error:
   destroyIndefinitOperatorInfo(pInfo);
   taosMemoryFree(pOperator);
-  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  pTaskInfo->code = code;
   return NULL;
 }
 

source/libs/executor/src/streamcountwindowoperator.c

@@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
                                                    (void**)&pCurWin->winInfo.pStatePos, &size);
       }
     }
+    if (ts < pCurWin->winInfo.sessionWin.win.ekey) {
+      pBuffInfo->rebuildWindow = true;
+    }
   } else {
     code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
         pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
@@ -284,6 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
       }
     }
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+      curWin.winInfo.pStatePos->beUpdated = true;
       SSessionKey key = {0};
       getSessionHashKey(&curWin.winInfo.sessionWin, &key);
       tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
@@ -341,7 +345,7 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
   size_t  keyLen = 0;
   int32_t iter = 0;
   while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {
-    void* key = taosHashGetKey(pIte, &keyLen);
+    void* key = tSimpleHashGetKey(pIte, &keyLen);
     tlen += encodeSSessionKey(buf, key);
     tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);
   }
@@ -434,7 +438,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
       pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key);
     }
     while (1) {
-      SSessionKey tmpKey = {0};
+      SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
       int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size);
       if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
         pAggSup->stateStore.streamStateFreeCur(pCur);

source/libs/executor/src/tsort.c

@@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       int64_t firstRowTs = *(int64_t*)tsCol->pData;
       if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
           (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
-            continue;
+        if (bExtractedBlock) {
-          }
+          blockDataDestroy(pBlk);
+	}
+        continue;
+      }
     }
 
     if (pBlk != NULL) {
@@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       tSimpleHashClear(mUidBlk);
       code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
       if (code != TSDB_CODE_SUCCESS) {
-        tSimpleHashCleanup(mUidBlk);
+        for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
-        taosArrayDestroy(aBlkSort);
+          blockDataDestroy(taosArrayGetP(aBlkSort, i));
-        taosArrayDestroy(aExtSrc);
+        }
-        return code;
+        taosArrayClear(aBlkSort);
+        break;	
       }
 
       int64_t el = taosGetTimestampUs() - p;
@@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       szSort = 0;
       qDebug("source %zu created", taosArrayGetSize(aExtSrc));
     }
+
     if (pBlk == NULL) {
       break;
     }
@@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   }
 
   tSimpleHashCleanup(mUidBlk);
+  for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
+    blockDataDestroy(taosArrayGetP(aBlkSort, i));
+  }
   taosArrayDestroy(aBlkSort);
   tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
   if (!tsortIsClosed(pHandle)) {
@@ -1188,7 +1196,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   taosArrayDestroy(aExtSrc);
   tSimpleHashCleanup(mTableNumRows);
   pHandle->type = SORT_SINGLESOURCE_SORT;
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {

source/libs/monitor/CMakeLists.txt

@@ -6,7 +6,7 @@ target_include_directories(
   PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
-target_link_libraries(monitor os util common transport)
+target_link_libraries(monitor os util common qcom transport monitorfw)
 
 if(${BUILD_TEST})
     add_subdirectory(test)