diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx
index e4dc177388..881838f8f0 100644
--- a/docs/zh/07-develop/07-tmq.mdx
+++ b/docs/zh/07-develop/07-tmq.mdx
@@ -15,11 +15,11 @@ import Node from "./_sub_node.mdx";
import CSharp from "./_sub_cs.mdx";
import CDemo from "./_sub_c.mdx";
-
为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine 提供了类似 kafka 的数据订阅功能。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。
-# 介绍
-## 主题
+## 数据订阅介绍
+
+### 主题
与 kafka 一样，你需要定义 topic, TDengine 的 topic 有三种，可以是数据库，超级表，或者一个 `SELECT` 语句，具体的语法参见 [CREATE TOPIC](../../taos-sql/tmq)。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。
如下图，每个 topic 涉及到的数据表可能分布在多个 vnode（相当于 kafka 里的 partition） 上，每个 vnode 上的数据保存在 WAL(Write-Ahead-Log) 文件中，WAL 文件里的数据是顺序写入的（由于 WAL 文件中存储的不只有数据，还有元数据，写入消息等，所以数据的版本号不是连续的）。
@@ -30,11 +30,12 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
对于 `SELECT` 语句形式的 topic，在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。
-## 生产者
+### 生产者
写入 topic 相关联的数据表中数据的都是生产者，生产者实际生产的数据写入到了子表或普通表中，即表所在 vnode 的 WAL 里。
-## 消费者
-### 消费者组
+### 消费者
+
+#### 消费者组
消费者订阅 topic 后，可以消费 topic 里的所有数据（这些数据所在的表可能分布在多个 vnode 上，即 db 所在的所有 vnode）。订阅 topic 时，需要指定一个消费者组 (consumer group)，如果这个消费者组里只有一个消费者，那么这个消费者会顺序的消费这些 vnode 上的数据。
为了提高消费速度，便于多线程、分布式地消费数据，可以在一个消费组里添加多个消费者，这些消费者将均分数据所在的 vnode 进行消费（比如数据分布在 4 个 vnode 上，有 2 个消费者的话，那么每个消费者消费 2 个 vnode；有 3 个消费者的话，2 个消费者各消费 1 个 vnode，1 个消费者消费 2 个 vnode；有 5 个消费者的话，4 个各分配 1 个 vnode 消费，另外 1 个不消费），如下图：
@@ -44,7 +45,8 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
在一个消费组里添加一个消费者后，在 Mnode 上通过 rebalance 的机制实现消费者的重新分配，该操作对用户是透明的。
一个消费者可以订阅多个 topic。TDengine 的数据订阅在宕机、重启等复杂环境下确保 at least once 消费。
-### 消费进度
+
+#### 消费进度
在 topic 的一个消费组的一个 vnode 上有消费进度。消费者消费的同时，可以提交消费进度，消费进度即 vnode 上 WAL 的版本号（对于 kafka 里的 offset），消费进度可以手动提交，也可以通过参数（auto.commit.interval.ms）设置为周期性自动提交。
首次消费数据时通过订阅参数（auto.offset.reset）来确定消费位置为最新数据（latest）还是最旧数据（earliest）。
@@ -59,16 +61,16 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
作为一个数据库产品， WAL 文件中存储的不全是数据，也包括其他写入消息，元数据等，所以消费进度不是连续的。
-##说明
+### 说明
从3.2.0.0版本开始，数据订阅支持vnode迁移和分裂。
由于数据订阅依赖wal文件，而在vnode迁移和分裂的过程中，wal并不会同步过去，所以迁移或分裂后，之前没消费完的wal数据后消费不到。所以请保证迁移和分裂之前把数据全部消费完后，再进行vnode迁移或分裂，否则，消费会丢失数据。
-# 语法说明
+## 数据订阅语法说明
具体的语法参见 [数据订阅](../../taos-sql/tmq)
-# 消费参数
+## 数据订阅相关参数
消费参数主要用于消费者创建时指定，基础配置项如下表所示：
@@ -86,9 +88,9 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提
| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） |默认关闭 |
| `enable.replay` | boolean | 是否开启数据回放功能 |默认关闭 |
-# 主要数据结构和 API 接口
+## 数据订阅主要 API 接口
-不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
+不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节数据订阅部分，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）：
@@ -310,8 +312,8 @@ void Close()
-# 数据订阅示例
-## 写入数据
+## 数据订阅示例
+### 写入数据
首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如：
@@ -324,7 +326,7 @@ CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
```
-## 创建 topic
+### 创建 topic
使用 SQL 创建一个 topic：
@@ -332,7 +334,7 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11');
CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
```
-## 创建消费者 *consumer*
+### 创建消费者 consumer
对于不同编程语言，其设置方式如下：
@@ -499,7 +501,7 @@ var consumer = new ConsumerBuilder>(cfg).Build();
上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
-## 订阅 *topics*
+### 订阅 topics
一个 consumer 支持同时订阅多个 topic。
@@ -578,7 +580,7 @@ consumer.Subscribe(topics);
-## 消费
+### 消费
以下代码展示了不同语言下如何对 TMQ 消息进行消费。
@@ -714,7 +716,7 @@ while (true)
-## 结束消费
+### 结束消费
消费结束后，应当取消订阅。
@@ -795,7 +797,7 @@ consumer.Close();
-## 完整示例代码
+### 完整示例代码
以下是各语言的完整示例代码。
@@ -838,8 +840,8 @@ consumer.Close();
-#订阅高级功能
-##数据回放
+## 数据订阅高级功能
+### 数据回放
- 订阅支持 replay 功能，按照数据写入的时间回放。
比如，如下时间写入三条数据
```sql
diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx
index 60810ec275..c0e9e6c937 100644
--- a/docs/zh/07-develop/_sub_java.mdx
+++ b/docs/zh/07-develop/_sub_java.mdx
@@ -6,3 +6,4 @@
```
```java
{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+```
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 1709efd253..3dcaf078ef 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo {
SColumnInfoData* pColData;
} SBlockOrderInfo;
+#define BLOCK_VERSION_1 1
+#define BLOCK_VERSION_2 2
+
#define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define CharPos(r_) ((r_) >> NBIT)
@@ -272,7 +275,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
bool alreadyAddGroupId(char* ctbName);
bool isAutoTableName(char* ctbName);
-void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId);
+void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId);
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index ef3e61e1a2..469bc6227e 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -106,6 +106,9 @@ extern char tsMonitorFqdn[];
extern uint16_t tsMonitorPort;
extern int32_t tsMonitorMaxLogs;
extern bool tsMonitorComp;
+extern bool tsMonitorLogProtocol;
+extern int32_t tsMonitorIntervalForBasic;
+extern bool tsMonitorForceV2;
// audit
extern bool tsEnableAudit;
@@ -216,6 +219,7 @@ extern bool tsFilterScalarMode;
extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold;
extern int32_t tsResolveFQDNRetryTime;
+extern bool tsDisableCount;
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
diff --git a/include/common/tgrant.h b/include/common/tgrant.h
index 6d11608d21..ca9e030117 100644
--- a/include/common/tgrant.h
+++ b/include/common/tgrant.h
@@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant);
// #ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE
-#define GRANTS_SCHEMA \
- static const SSysDbTableSchema grantsSchema[] = { \
- {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
+#define GRANTS_SCHEMA \
+ static const SSysDbTableSchema grantsSchema[] = { \
+ {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
}
#else
-#define GRANTS_SCHEMA \
- static const SSysDbTableSchema grantsSchema[] = { \
- {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
- {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
+#define GRANTS_SCHEMA \
+ static const SSysDbTableSchema grantsSchema[] = { \
+ {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
+ {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \
}
#endif
// #define GRANT_CFG_ADD
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 1257225319..c5582ff89e 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -207,9 +207,6 @@ typedef enum _mgmt_table {
#define TD_CHILD_TABLE TSDB_CHILD_TABLE
#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
-#define TD_REQ_FROM_APP 0
-#define TD_REQ_FROM_TAOX 1
-
typedef enum ENodeType {
// Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
// VALUE, OPERATOR, FUNCTION and so on.
@@ -587,7 +584,7 @@ typedef struct {
int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitBlkRsp(void* param);
+// void tFreeSSubmitBlkRsp(void* param);
void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SMA_ON ((int8_t)0x1)
@@ -760,7 +757,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
int8_t igExists;
- int8_t source; // 1-taosX or 0-taosClient
+ int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
int8_t reserved[6];
tb_uid_t suid;
int64_t delay1;
@@ -803,7 +800,7 @@ void tFreeSMCreateStbRsp(SMCreateStbRsp* pRsp);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
int8_t igNotExists;
- int8_t source; // 1-taosX or 0-taosClient
+ int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
int8_t reserved[6];
tb_uid_t suid;
int32_t sqlLen;
@@ -1328,14 +1325,13 @@ typedef struct {
} SDnodeListReq;
int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
-int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq);
typedef struct {
int32_t useless; // useless
} SServerVerReq;
int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
-int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
+// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq);
typedef struct {
char ver[TSDB_VERSION_LEN];
@@ -1586,6 +1582,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq);
+typedef struct {
+ int32_t contLen;
+ char* pCont;
+} SStatisReq;
+
+int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
+void tFreeSStatisReq(SStatisReq *pReq);
+
typedef struct {
int32_t dnodeId;
int64_t clusterId;
@@ -1614,7 +1619,6 @@ typedef struct {
SEp ep;
char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN];
- char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeInfo;
typedef struct {
@@ -1634,14 +1638,14 @@ typedef struct {
} SMTimerReq;
int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
-int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
+// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
typedef struct {
int64_t tick;
} SMStreamTickReq;
int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
-int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
+// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
typedef struct {
int32_t id;
@@ -1912,7 +1916,7 @@ typedef struct {
} SShowVariablesReq;
int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
-int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
+// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
typedef struct {
char name[TSDB_CONFIG_OPTION_LEN + 1];
@@ -1942,7 +1946,7 @@ typedef struct {
} SShowReq;
int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
-int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
+// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
void tFreeSShowReq(SShowReq* pReq);
typedef struct {
@@ -1950,9 +1954,9 @@ typedef struct {
STableMetaRsp tableMeta;
} SShowRsp, SVShowTablesRsp;
-int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
-void tFreeSShowRsp(SShowRsp* pRsp);
+// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
+// void tFreeSShowRsp(SShowRsp* pRsp);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
@@ -2210,7 +2214,7 @@ typedef struct {
} SForceBecomeFollowerReq;
int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
-int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
+// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq);
typedef struct {
int32_t vgId;
@@ -2227,8 +2231,8 @@ typedef struct {
char ckey[TSDB_PASSWORD_LEN];
} SAuthReq, SAuthRsp;
-int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
-int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
+// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq);
typedef struct {
int32_t statusCode;
@@ -2662,6 +2666,7 @@ typedef struct SVCreateStbReq {
SRSmaParam rsmaParam;
int32_t alterOriDataLen;
void* alterOriData;
+ int8_t source;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@@ -2731,6 +2736,7 @@ typedef struct {
SVCreateTbReq* pReqs;
SArray* pArray;
};
+ int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@@ -2759,8 +2765,8 @@ typedef struct {
int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp);
int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp);
-int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
-int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
// TDMT_VND_DROP_TABLE =================
typedef struct {
@@ -2823,6 +2829,7 @@ typedef struct {
int32_t newCommentLen;
char* newComment;
int64_t ctimeMs; // fill by vnode
+ int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@@ -3214,8 +3221,8 @@ typedef struct {
int32_t hasCheckPoint;
} SMVStreamGatherInfoReq;
-int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
-int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
+// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq);
+// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq);
typedef struct {
int64_t leftForVer;
@@ -3492,7 +3499,7 @@ void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
-int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
+// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
typedef struct {
int32_t number;
@@ -3920,12 +3927,13 @@ int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp);
int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
+#define TD_REQ_FROM_APP 0x0
#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1
#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
#define SUBMIT_REQ_FROM_FILE 0x4
+#define TD_REQ_FROM_TAOX 0x8
-#define SOURCE_NULL 0
-#define SOURCE_TAOSX 1
+#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility
typedef struct {
int32_t flags;
@@ -3938,7 +3946,6 @@ typedef struct {
SArray* aCol;
};
int64_t ctimeMs;
- int8_t source;
} SSubmitTbData;
typedef struct {
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index dc5cb24564..62392a9caa 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -215,6 +215,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
diff --git a/include/common/trow.h b/include/common/trow.h
index 8332c10ed2..92713e8e63 100644
--- a/include/common/trow.h
+++ b/include/common/trow.h
@@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
*/
static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
- pBuilder->rowType = TD_ROW_TP; // default STpRow
+ pBuilder->rowType = pBuilder->rowType;
pBuilder->sver = sver;
}
int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);
@@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow);
bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal);
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow);
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType);
bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag);
diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h
index 84d0dd4982..22a176f0bb 100644
--- a/include/dnode/vnode/tqCommon.h
+++ b/include/dnode/vnode/tqCommon.h
@@ -27,6 +27,8 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
bool isLeader, bool restored);
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h
new file mode 100644
index 0000000000..9c0302a15f
--- /dev/null
+++ b/include/libs/monitor/clientMonitor.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_CLIENT_MONITOR_H
+#define TDENGINE_CLIENT_MONITOR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taos_monitor.h"
+#include "thash.h"
+#include "query.h"
+
+typedef enum SQL_RESULT_CODE {
+ SQL_RESULT_SUCCESS = 0,
+ SQL_RESULT_FAILED = 1,
+ SQL_RESULT_CANCEL = 2,
+} SQL_RESULT_CODE;
+
+const char* resultStr(SQL_RESULT_CODE code);
+
+typedef struct {
+ char clusterKey[512];
+ SEpSet epSet;
+ void* pTransporter;
+ taos_collector_registry_t* registry;
+ taos_collector_t* colector;
+ SHashObj* counters;
+} ClientMonitor;
+
+void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter);
+void clusterMonitorClose(const char* clusterKey);
+taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
+ const char** label_keys);
+int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values);
+void cluster_monitor_stop();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_CLIENT_MONITOR_H
diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h
index 91b3a54ea1..256be26999 100644
--- a/include/libs/monitor/monitor.h
+++ b/include/libs/monitor/monitor.h
@@ -76,7 +76,8 @@ typedef struct {
} SMonBasicInfo;
typedef struct {
- float uptime; // day
+ //float uptime; // day
+ int64_t uptime; // second
int8_t has_mnode;
int8_t has_qnode;
int8_t has_snode;
@@ -100,13 +101,15 @@ typedef struct {
int32_t mnode_id;
char mnode_ep[TSDB_EP_LEN];
char role[MON_ROLE_LEN];
+ int32_t syncState;
} SMonMnodeDesc;
typedef struct {
char first_ep[TSDB_EP_LEN];
int32_t first_ep_dnode_id;
char version[MON_VER_LEN];
- float master_uptime; // day
+ //float master_uptime; // day
+ int64_t master_uptime; //second
int32_t monitor_interval; // sec
int32_t dbs_total;
int32_t stbs_total;
@@ -125,6 +128,7 @@ typedef struct {
typedef struct {
int32_t dnode_id;
char vnode_role[MON_ROLE_LEN];
+ int32_t syncState;
} SMonVnodeDesc;
typedef struct {
@@ -149,7 +153,7 @@ typedef struct {
} SMonStbInfo;
typedef struct {
- uint32_t expire_time;
+ int64_t expire_time;
int64_t timeseries_used;
int64_t timeseries_total;
} SMonGrantInfo;
@@ -221,7 +225,9 @@ void monSetVmInfo(SMonVmInfo *pInfo);
void monSetQmInfo(SMonQmInfo *pInfo);
void monSetSmInfo(SMonSmInfo *pInfo);
void monSetBmInfo(SMonBmInfo *pInfo);
-void monSendReport();
+void monGenAndSendReport();
+void monGenAndSendReportBasic();
+void monSendContent(char *pCont);
void tFreeSMonMmInfo(SMonMmInfo *pInfo);
void tFreeSMonVmInfo(SMonVmInfo *pInfo);
diff --git a/include/libs/monitorfw/taos_alloc.h b/include/libs/monitorfw/taos_alloc.h
new file mode 100644
index 0000000000..893ffc7e9b
--- /dev/null
+++ b/include/libs/monitorfw/taos_alloc.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_alloc.h
+ * @brief memory management
+ */
+
+#ifndef TAOS_ALLOC_H
+#define TAOS_ALLOC_H
+
+#include
+#include
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is malloc.
+ */
+#define taos_malloc malloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is realloc.
+ */
+#define taos_realloc realloc
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is strdup.
+ */
+#define taos_strdup strdup
+
+/**
+ * @brief Redefine this macro if you wish to override it. The default value is free.
+ */
+#define taos_free free
+
+#endif // TAOS_ALLOC_H
diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h
new file mode 100644
index 0000000000..c37cb612d2
--- /dev/null
+++ b/include/libs/monitorfw/taos_collector.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_COLLECTOR_H
+#define TAOS_COLLECTOR_H
+
+#include "taos_map.h"
+#include "taos_metric.h"
+
+/**
+ * @file taos_collector.h
+ * @brief A Prometheus collector returns a collection of metrics
+ */
+
+/**
+ * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is
+ * registered.
+ */
+typedef struct taos_collector taos_collector_t;
+
+/**
+ * @brief The function responsible for preparing metric data and returning metrics for a given collector.
+ *
+ * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may
+ * set this function on your collector to do additional work before returning the contained metrics.
+ *
+ * @param self The target taos_collector_t*
+ * @return The taos_map_t* containing the collected metrics
+ */
+typedef taos_map_t *taos_collect_fn(taos_collector_t *self);
+
+/**
+ * @brief Create a collector
+ * @param name The name of the collector. The name MUST NOT be default or process.
+ * @return The constructed taos_collector_t*
+ */
+taos_collector_t *taos_collector_new(const char *name);
+
+/**
+ * @brief Destroy a collector. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy(taos_collector_t *self);
+
+/**
+ * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ */
+void taos_collector_free_generic(void *gen);
+
+/**
+ * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction.
+ * @param gen The target taos_collector_t* represented as a void*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_destroy_generic(void *gen);
+
+/**
+ * @brief Add a metric to a collector
+ * @param self The target taos_collector_t*
+ * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric);
+
+int taos_collector_remove_metric(taos_collector_t *self, const char* key);
+
+taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name);
+
+/**
+ * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them
+ * for metric exposition.
+ * @param self The target taos_collector_t*
+ * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before
+ * returning the collected metrics for exposition.
+ * @return A non-zero integer value upon failure.
+ */
+int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn);
+
+#endif // TAOS_COLLECTOR_H
diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h
new file mode 100644
index 0000000000..81d1f8050c
--- /dev/null
+++ b/include/libs/monitorfw/taos_collector_registry.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_collector_registry.h
+ * @brief The collector registry registers collectors for metric exposition.
+ */
+
+#ifndef TAOS_REGISTRY_H
+#define TAOS_REGISTRY_H
+
+#include "taos_collector.h"
+#include "taos_metric.h"
+
+/**
+ * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format
+ */
+typedef struct taos_collector_registry taos_collector_registry_t;
+
+/**
+ * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT
+ * modify this value.
+ */
+extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT;
+
+/**
+ * @brief Initializes the default collector registry and enables metric collection on the executing process
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_default_init(void);
+
+/**
+ * @brief Constructs a taos_collector_registry_t*
+ * @param name The name of the collector registry. It MUST NOT be default.
+ * @return The constructed taos_collector_registry_t*
+ */
+taos_collector_registry_t *taos_collector_registry_new(const char *name);
+
+/**
+ * @brief Destroy a collector registry. You MUST set self to NULL after destruction.
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_destroy(taos_collector_registry_t *self);
+
+/**
+ * @brief Enable process metrics on the given collector registry
+ * @param self The target taos_collector_registry_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY
+ *
+ * The metric to be registered MUST NOT already be registered with the given . If so, the program will
+ * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore,
+ * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this
+ * function. The metric will be added to the default registry's default collector.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return The registered taos_metric_t*
+ */
+taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric);
+
+/**
+ * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer
+ * value on failure.
+ *
+ * See taos_collector_registry_must_register_metric.
+ *
+ * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY*
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_metric(taos_metric_t *metric);
+
+int taos_collector_registry_deregister_metric(const char *key);
+
+taos_metric_t *taos_collector_registry_get_metric(char* metric_name);
+
+/**
+ * @brief Register a collector with the given registry. Returns a non-zero integer value on failure.
+ * @param self The target taos_collector_registry_t*
+ * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector);
+
+/**
+ * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap
+ * memory growth.
+ *
+ * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/
+ *
+ * @param self The target taos_collector_registry_t*
+ * @return The string int he default metric exposition format.
+ */
+const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format);
+
+int taos_collector_registry_clear_batch(taos_collector_registry_t *self);
+
+const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str);
+
+/**
+ *@brief Validates that the given metric name complies with the specification:
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ *
+ * Returns a non-zero integer value on failure.
+ *
+ * @param self The target taos_collector_registry_t*
+ * @param metric_name The metric name to validate
+ * @return A non-zero integer value upon failure
+ */
+int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name);
+
+#endif // TAOS_H
diff --git a/include/libs/monitorfw/taos_counter.h b/include/libs/monitorfw/taos_counter.h
new file mode 100644
index 0000000000..28a9eed41c
--- /dev/null
+++ b/include/libs/monitorfw/taos_counter.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_COUNTER_H
+#define TAOS_COUNTER_H
+
+#include
+
+#include "taos_metric.h"
+
+/**
+ * @file taos_counter.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+
+/**
+ * @brief A prometheus counter.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#counter
+ */
+typedef taos_metric_t taos_counter_t;
+
+/**
+ * @brief Construct a taos_counter_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ * require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ * literal.
+ * @return The constructed taos_counter_t*
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" });
+ *
+ * // An example without labels
+ * taos_counter_new("foo", "foo is a counter without labels", 0, NULL);
+ */
+taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ * returned on failure.
+ * @param self A taos_counter_t*
+ * @return A non-zero integer value upon failure.
+ */
+int taos_counter_destroy(taos_counter_t *self);
+
+/**
+ * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure.
+ * @param self The target taos_counter_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the counter's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_counter_inc(foo_counter, (const char**) { "bar", "bang" });
+ **
+ * // An example without labels
+ * taos_counter_inc(foo_counter, NULL);
+ */
+int taos_counter_inc(taos_counter_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure.
+ * @param self The target taos_counter_t*
+ * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the counter's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_counter_add(foo_counter, 22, NULL);
+ */
+int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values);
+
+#endif // TAOS_COUNTER_H
diff --git a/include/libs/monitorfw/taos_gauge.h b/include/libs/monitorfw/taos_gauge.h
new file mode 100644
index 0000000000..a33661dfe4
--- /dev/null
+++ b/include/libs/monitorfw/taos_gauge.h
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_gauge.h
+ * @brief https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+
+#ifndef TAOS_GAUGE_H
+#define TAOS_GAUGE_H
+
+#include
+
+#include "taos_metric.h"
+
+/**
+ * @brief A prometheus gauge.
+ *
+ * References
+ * * See https://prometheus.io/docs/concepts/metric_types/#gauge
+ */
+typedef taos_metric_t taos_gauge_t;
+
+/**
+ * @brief Constructs a taos_gauge_t*
+ * @param name The name of the metric
+ * @param help The metric description
+ * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not
+ * require labels.
+ * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If
+ * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a
+ * literal.
+ * @return The constructed taos_guage_t*
+ *
+ * // An example with labels
+ * taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" });
+ *
+ * // An example without labels
+ * taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL);
+ */
+taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys);
+
+/**
+ * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be
+ * returned on failure.
+ * @param self The target taos_gauge_t*
+ * @return A non-zero integer value upon failure
+ */
+int taos_gauge_destroy(taos_gauge_t *self);
+
+/**
+ * @brief Increment the taos_gauge_t* by 1.
+ * @param self The target taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the gauge's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure
+ * *Example*
+ *
+ * // An example with labels
+ * taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_gauge_inc(foo_gauge, NULL);
+ */
+int taos_gauge_inc(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Decrement the taos_gauge_t* by 1.
+ * @param self The target taos_gauger_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the gauge's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ * *Example*
+ *
+ * // An example with labels
+ * taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_gauge_dec(foo_gauge, NULL);
+ */
+int taos_gauge_dec(taos_gauge_t *self, const char **label_values);
+
+/**
+ * @brief Add the value to the taos_gauge_t*.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the gauge's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_gauge_add(foo_gauge, 22, NULL);
+ */
+int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure.
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to add to the taos_gauge_t passed as self.
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the gauge's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_gauge_sub(foo_gauge, 22, NULL);
+ */
+int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values);
+
+/**
+ * @brief Set the value for the taos_gauge_t*
+ * @param self The target taos_gauge_t*
+ * @param r_value The double to which the taos_gauge_t* passed as self will be set
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the gauge's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A non-zero integer value upon failure.
+ *
+ * *Example*
+ *
+ * // An example with labels
+ * taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" });
+ *
+ * // An example without labels
+ * taos_gauge_set(foo_gauge, 22, NULL);
+ */
+int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values);
+
+#endif // TAOS_GAUGE_H
diff --git a/include/libs/monitorfw/taos_linked_list.h b/include/libs/monitorfw/taos_linked_list.h
new file mode 100644
index 0000000000..1fff5e65a6
--- /dev/null
+++ b/include/libs/monitorfw/taos_linked_list.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_LIST_H
+#define TAOS_LIST_H
+
+#include
+
+struct taos_linked_list;
+/**
+ * @brief Provides a generic linked list
+ */
+typedef struct taos_linked_list taos_linked_list_t;
+
+#endif // TAOS_LIST_H
diff --git a/include/libs/monitorfw/taos_map.h b/include/libs/monitorfw/taos_map.h
new file mode 100644
index 0000000000..fa554d569c
--- /dev/null
+++ b/include/libs/monitorfw/taos_map.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_MAP_H
+#define TAOS_MAP_H
+
+struct taos_map;
+typedef struct taos_map taos_map_t;
+
+struct taos_map_node;
+typedef struct taos_map_node taos_map_node_t;
+
+#endif // TAOS_MAP_H
diff --git a/include/libs/monitorfw/taos_metric.h b/include/libs/monitorfw/taos_metric.h
new file mode 100644
index 0000000000..a377311fa0
--- /dev/null
+++ b/include/libs/monitorfw/taos_metric.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_metric.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_METRIC_H
+#define TAOS_METRIC_H
+
+#include "taos_metric_sample.h"
+
+struct taos_metric;
+/**
+ * @brief A prometheus metric.
+ *
+ * Reference: https://prometheus.io/docs/concepts/data_model
+ */
+typedef struct taos_metric taos_metric_t;
+
+/**
+ * @brief Returns a taos_metric_sample_t*. The order of label_values is significant.
+ *
+ * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map
+ * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be
+ * preferrable in performance-sensitive situations.
+ *
+ * @param self The target taos_metric_t*
+ * @param label_values The label values associated with the metric sample being updated. The number of labels must
+ * match the value passed to label_key_count in the counter's constructor. If no label values are
+ * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal.
+ * @return A taos_metric_sample_t*
+ */
+taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values);
+
+#endif // TAOS_METRIC_H
diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h
new file mode 100644
index 0000000000..0283d56ad9
--- /dev/null
+++ b/include/libs/monitorfw/taos_metric_sample.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_metric_sample.h
+ * @brief Functions for interfacting with metric samples directly
+ */
+
+#ifndef TAOS_METRIC_SAMPLE_H
+#define TAOS_METRIC_SAMPLE_H
+
+struct taos_metric_sample;
+/**
+ * @brief Contains the specific metric and value given the name and label set
+ * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+ */
+typedef struct taos_metric_sample taos_metric_sample_t;
+
+/**
+ * @brief Add the r_value to the sample. The value must be greater than or equal to zero.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to add to taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_add(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Subtract the r_value from the sample.
+ *
+ * This operation MUST be called a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double to subtract from the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value);
+
+/**
+ * @brief Set the r_value of the sample.
+ *
+ * This operation MUST be called on a sample derived from a gauge metric.
+ * @param self The target taos_metric_sample_t*
+ * @param r_value The double which will be set to the taos_metric_sample_t* provided by self
+ * @return Non-zero integer value upon failure
+ */
+int taos_metric_sample_set(taos_metric_sample_t *self, double r_value);
+
+int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value);
+
+#endif // TAOS_METRIC_SAMPLE_H
diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h
new file mode 100644
index 0000000000..c8569dd0a0
--- /dev/null
+++ b/include/libs/monitorfw/taos_monitor.h
@@ -0,0 +1,132 @@
+/*
+Copyright 2019 DigitalOcean Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @file taos_monitor.h
+ * @brief Include taos_monitor.h to include the entire public API
+ * @mainpage Welcome to the documentation site for prometheus-client-c!
+ * @tableofcontents
+ * @section Introduction
+ *
+ * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language.
+ * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics
+ * over HTTP.
+ *
+ * @section Creating-and-Registering-Metrics Creating and Registering Metrics
+ *
+ * prometheus-client-c supports the following metric types:
+ *
+ * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter)
+ * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge)
+ * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)
+ *
+ * To get started using one of the metric types, declare the metric at file scope. For example:
+ *
+ * @code{.c}
+ *
+ * #incldue "taos_monitor.h"
+ *
+ * taos_counter_t *my_counter;
+ *
+ * @endcode
+ *
+ * Next, create a metric initialization function. You can create the metric and register it with the default metric
+ * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning
+ * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later...
+ *
+ * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric
+ * constructor into the taos_collector_registry_must_register_metric function. For example:
+ *
+ * @code{.c}
+ *
+ * void foo_metric_init(void) {
+ * my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL));
+ * }
+ *
+ * @endcode
+ *
+ * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third
+ * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass
+ * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we
+ * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics
+ * for the file we just created to the default prometheus metric collector registery called
+ * TAOS_COLLECTOR_REGISTRY_DEFAULT
+ *
+ * @section Updating-Metric-Sample-Values Updating Metric Sample Values
+ *
+ * Now that we have a metric configured for creation and registration, we can update our metric within any of the
+ * functions of the file in which it was declared. For example:
+ *
+ * @code{.c}
+ *
+ * void my_lib_do_something(void) {
+ * printf("I did a really important thing!\n");
+ * taos_counter_inc(my_counter, NULL);
+ * }
+ * @endcode
+ *
+ * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass
+ * NULL as the second argument.
+ *
+ * @section Program-Initialization Program Initialization
+ *
+ * At the start of the program's main function you need to do two things:
+ *
+ * * Initialize the default metric collector registry:
+ *
+ * @code{.c}
+ *
+ * taos_collector_registry_default_init();
+ *
+ * @endcode
+ *
+ * * For each file containing prometheus metrics, call its corresponding metric initialization function
+ *
+ * @code{.c}
+ *
+ * foo_metric_init()
+ *
+ * @endcode
+ *
+ * After initialization is complete, you may proceed to do work and update your metrics.
+ *
+ * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP
+ *
+ * @todo Describe how to use libpromhttp to expose metrics over HTTP
+ *
+ * @section Where-To-Go-From-Here Where to Go From Here?
+ *
+ * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html)
+ * tab in this documentation site for more information about the public API available to you. Also, you can take a look
+ * at the examples directory at the
+ * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration.
+ */
+
+#ifndef TAOS_INCLUDED
+#define TAOS_INCLUDED
+
+#include "taos_alloc.h"
+#include "taos_collector.h"
+#include "taos_collector_registry.h"
+#include "taos_counter.h"
+#include "taos_linked_list.h"
+#include "taos_map.h"
+#include "taos_metric.h"
+#include "taos_metric_sample.h"
+#include "taos_monitor_util.h"
+#include "taos_gauge.h"
+
+#endif // TAOS_INCLUDED
\ No newline at end of file
diff --git a/include/libs/monitorfw/taos_monitor_util.h b/include/libs/monitorfw/taos_monitor_util.h
new file mode 100644
index 0000000000..a42d1ddcef
--- /dev/null
+++ b/include/libs/monitorfw/taos_monitor_util.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+/**
+ * @file taos_monitor_util.h
+ * @brief Functions for retrieving metric samples from metrics given an ordered set of labels
+ */
+
+#ifndef TAOS_MONITOR_UTIL_H
+#define TAOS_MONITOR_UTIL_H
+
+#include "taos_metric.h"
+#include "tjson.h"
+
+void taos_monitor_split_str(char** arr, char* str, const char* del);
+void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf);
+char* taos_monitor_get_metric_name(taos_metric_t* metric);
+
+#endif // TAOS_MONITOR_UTIL_H
\ No newline at end of file
diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h
index 9fd00cefb4..48b7b8c56f 100644
--- a/include/os/osAtomic.h
+++ b/include/os/osAtomic.h
@@ -48,11 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val);
void atomic_store_16(int16_t volatile *ptr, int16_t val);
void atomic_store_32(int32_t volatile *ptr, int32_t val);
void atomic_store_64(int64_t volatile *ptr, int64_t val);
+double atomic_store_double(double volatile *ptr, double val);
void atomic_store_ptr(void *ptr, void *val);
int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
+double atomic_exchange_double(double volatile *ptr, int64_t val);
void *atomic_exchange_ptr(void *ptr, void *val);
int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
@@ -68,6 +70,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
+double atomic_fetch_add_double(double volatile *ptr, double val);
void *atomic_fetch_add_ptr(void *ptr, void *val);
int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
@@ -78,6 +81,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
+double atomic_fetch_sub_double(double volatile *ptr, double val);
void *atomic_fetch_sub_ptr(void *ptr, void *val);
int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index ec05fb7783..27202fa51d 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -758,8 +758,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_COL_QUERY_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x266D)
#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E)
#define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F)
-#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2670)
-#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2671)
+#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670)
+#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2671)
+#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner
diff --git a/include/util/tbase58.h b/include/util/tbase58.h
index e1b03f8a8f..ab62e1926e 100644
--- a/include/util/tbase58.h
+++ b/include/util/tbase58.h
@@ -22,6 +22,9 @@
extern "C" {
#endif
+#define TBASE_MAX_ILEN 4096
+#define TBASE_MAX_OLEN 5653
+
uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen);
char *base58_encode(const uint8_t *value, int32_t vlen);
diff --git a/include/util/tdef.h b/include/util/tdef.h
index 0e053d4baa..17254c5405 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -197,6 +197,8 @@ typedef enum ELogicConditionType {
// ACCOUNT is a 32 bit positive integer
// this is the length of its string representation, including the terminator zero
#define TSDB_ACCT_ID_LEN 11
+#define TSDB_NODE_ID_LEN 11
+#define TSDB_VGROUP_ID_LEN 11
#define TSDB_MAX_COLUMNS 4096
#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns
@@ -548,6 +550,10 @@ enum {
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60
+#define MONITOR_TABLENAME_LEN 200
+#define MONITOR_TAG_NAME_LEN 100
+#define MONITOR_TAG_VALUE_LEN 300
+#define MONITOR_METRIC_NAME_LEN 100
#ifdef __cplusplus
}
#endif
diff --git a/include/util/tjson.h b/include/util/tjson.h
index 6922930c13..b9ea72b4bb 100644
--- a/include/util/tjson.h
+++ b/include/util/tjson.h
@@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
SJson* tjsonParse(const char* pStr);
bool tjsonValidateJson(const char* pJson);
const char* tjsonGetError();
+void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName);
#ifdef __cplusplus
}
diff --git a/include/util/tlog.h b/include/util/tlog.h
index 11ac0e1fae..af92e1864e 100644
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@@ -120,6 +120,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
#define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
+#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt
index f089247859..a17c27c297 100644
--- a/source/client/CMakeLists.txt
+++ b/source/client/CMakeLists.txt
@@ -20,7 +20,7 @@ target_include_directories(
target_link_libraries(
taos
INTERFACE api
- PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+ PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
)
if(TD_DARWIN_ARM64)
@@ -61,7 +61,7 @@ target_include_directories(
target_link_libraries(
taos_static
INTERFACE api
- PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry
+ PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
)
if(${BUILD_TEST})
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index f73d121b15..989c6614a6 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -396,6 +396,8 @@ void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr);
void destroyAllRequests(SHashObj* pRequests);
void stopAllRequests(SHashObj* pRequests);
+SAppInstInfo* getAppInstInfo(const char* clusterKey);
+
// conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey);
@@ -432,6 +434,21 @@ void freeQueryParam(SSyncQueryParam* param);
int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
#endif
+void clientSlowQueryMonitorInit(const char* clusterKey);
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost);
+
+void clientSQLReqMonitorInit(const char* clusterKey);
+
+enum {
+ MONITORSQLTYPESELECT = 0,
+ MONITORSQLTYPEINSERT = 1,
+ MONITORSQLTYPEDELETE = 2
+};
+
+void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type);
+
+void clientMonitorClose(const char* clusterKey);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 87f8c6be67..1df50a51da 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -18,6 +18,7 @@
#include "clientLog.h"
#include "functionMgt.h"
#include "os.h"
+#include "osSleep.h"
#include "query.h"
#include "qworker.h"
#include "scheduler.h"
@@ -105,7 +106,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
reqType = SLOW_LOG_TYPE_QUERY;
- }
+ }
+ }
+
+ if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
+ sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
+ } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
+ sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT);
+ } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) {
+ sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE);
}
if (duration >= (tsSlowLogThreshold * 1000000UL)) {
@@ -114,6 +123,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr);
+ SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
}
}
@@ -224,6 +234,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) {
taosThreadMutexLock(&appInfo.mutex);
+ clientMonitorClose(pAppInfo->instKey);
hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 971ed407f9..de4f9f16e4 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -15,6 +15,7 @@
#include "cJSON.h"
#include "clientInt.h"
+#include "clientMonitor.h"
#include "clientLog.h"
#include "command.h"
#include "scheduler.h"
@@ -157,6 +158,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port);
pInst = &p;
+
+ clientSlowQueryMonitorInit(p->instKey);
+ clientSQLReqMonitorInit(p->instKey);
} else {
ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
// reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
@@ -166,9 +170,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
+
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
}
+SAppInstInfo* getAppInstInfo(const char* clusterKey) {
+ SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
+ if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
+ return *ppAppInstInfo;
+ } else {
+ return NULL;
+ }
+}
+
void freeQueryParam(SSyncQueryParam* param) {
if (param == NULL) return;
tsem_destroy(¶m->sem);
@@ -1850,7 +1864,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
char* pStart = p + len;
for (int32_t i = 0; i < numOfCols; ++i) {
- int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
+ int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
int32_t* offset = (int32_t*)pStart;
@@ -1949,8 +1963,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
char* pStart = p;
char* pStart1 = p1;
for (int32_t i = 0; i < numOfCols; ++i) {
- int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
- int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i];
+ int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
+ int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i];
if (ASSERT(colLen < dataLen)) {
tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen);
return TSDB_CODE_TSC_INTERNAL_ERROR;
@@ -2009,7 +2023,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
}
colLen1 = len;
totalLen += colLen1;
- colLength1[i] = (blockVersion == 1) ? htonl(len) : len;
+ colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len;
} else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
len = numOfRows * sizeof(int32_t);
memcpy(pStart1, pStart, len);
@@ -2098,7 +2112,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) {
- if(blockVersion == 1){
+ if(blockVersion == BLOCK_VERSION_1){
colLength[i] = htonl(colLength[i]);
}
if (colLength[i] >= dataLen) {
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index f6aef5aa26..275ca0d2aa 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -279,6 +279,7 @@ void taos_close_internal(void *taos) {
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
+ // clientMonitorClose(pTscObj->pAppInfo->instKey);
taosRemoveRef(clientConnRefPool, pTscObj->id);
}
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 448243cc3d..f143624bab 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1001,6 +1001,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
taosArrayPush(tBatch.req.pArray, pCreateReq);
+ tBatch.req.source = TD_REQ_FROM_TAOX;
taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch));
} else { // add to the correct vgroup
@@ -1276,7 +1277,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
return terrno;
}
SVAlterTbReq req = {0};
- SDecoder coder = {0};
+ SDecoder dcoder = {0};
int32_t code = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = NULL;
SQuery* pQuery = NULL;
@@ -1297,8 +1298,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
// decode and process req
void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
int32_t len = metaLen - sizeof(SMsgHead);
- tDecoderInit(&coder, data, len);
- if (tDecodeSVAlterTbReq(&coder, &req) < 0) {
+ tDecoderInit(&dcoder, data, len);
+ if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
@@ -1340,14 +1341,36 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
goto end;
}
pVgData->vg = pInfo;
- pVgData->pData = taosMemoryMalloc(metaLen);
- if (NULL == pVgData->pData) {
+
+ int tlen = 0;
+ req.source = TD_REQ_FROM_TAOX;
+ tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code);
+ if(code != 0){
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
- memcpy(pVgData->pData, meta, metaLen);
- ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId);
- pVgData->size = metaLen;
+ tlen += sizeof(SMsgHead);
+ void* pMsg = taosMemoryMalloc(tlen);
+ if (NULL == pMsg) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto end;
+ }
+ ((SMsgHead*)pMsg)->vgId = htonl(pInfo.vgId);
+ ((SMsgHead*)pMsg)->contLen = htonl(tlen);
+ void* pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead));
+ SEncoder coder = {0};
+ tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead));
+ code = tEncodeSVAlterTbReq(&coder, &req);
+ if(code != 0){
+ tEncoderClear(&coder);
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto end;
+ }
+ tEncoderClear(&coder);
+
+ pVgData->pData = pMsg;
+ pVgData->size = tlen;
+
pVgData->numOfTables = 1;
taosArrayPush(pArray, &pVgData);
@@ -1387,7 +1410,7 @@ end:
if (pVgData) taosMemoryFreeClear(pVgData->pData);
taosMemoryFreeClear(pVgData);
destroyRequest(pRequest);
- tDecoderClear(&coder);
+ tDecoderClear(&dcoder);
qDestroyQuery(pQuery);
terrno = code;
return code;
diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c
new file mode 100644
index 0000000000..572af7ff55
--- /dev/null
+++ b/source/client/src/clientSqlMonitor.c
@@ -0,0 +1,79 @@
+ /*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* selectMonitorName = "taos_sql_req:count";
+const char* selectMonitorHelp = "count for select sql";
+const int selectMonitorLabelCount = 4;
+const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"};
+
+static const char* defaultClusterID = "";
+
+void clientSQLReqMonitorInit(const char* clusterKey) {
+ if (!tsEnableMonitor) return;
+ SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+ SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+ clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+ createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels);
+}
+
+void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) {
+ const char* typeStr;
+ switch (type) {
+ case MONITORSQLTYPEDELETE:
+ typeStr = "delete";
+ break;
+ case MONITORSQLTYPEINSERT:
+ typeStr = "insert";
+ break;
+ default:
+ typeStr = "select";
+ break;
+ }
+ const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)};
+ taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues);
+}
+
+void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) {
+ if (!tsEnableMonitor) return;
+ SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+ if (TSDB_CODE_SUCCESS != code) {
+ result = SQL_RESULT_FAILED;
+ }
+ // to do Distinguish active Kill events
+ // else if (killed) {
+ // result = SQL_RESULT_CANCEL;
+ // }
+
+ STscObj* pTscObj = acquireTscObj(rid);
+ if (pTscObj != NULL) {
+ if (pTscObj->pAppInfo == NULL) {
+ tscLog("sqlReqLog, not found pAppInfo");
+ } else {
+ clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type);
+ }
+ releaseTscObj(rid);
+ } else {
+ tscLog("sqlReqLog, not found rid");
+ }
+}
+
+void clientMonitorClose(const char* clusterKey) {
+ tscLog("clientMonitorClose, key:%s", clusterKey);
+ clusterMonitorClose(clusterKey);
+}
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 1904874a0b..9b74456da2 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -389,7 +389,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
}
}
if (strcasecmp(key, "msg.consume.excluded") == 0) {
- conf->sourceExcluded = taosStr2int64(value);
+ conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0;
return TMQ_CONF_OK;
}
@@ -1611,17 +1611,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj;
}
-SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
- SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
- pRspObj->resType = RES_TYPE__TMQ;
+void changeByteEndian(char* pData){
+ char* p = pData;
+
+ // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
+ // version:
+ int32_t blockVersion = *(int32_t*)p;
+ ASSERT(blockVersion == BLOCK_VERSION_1);
+ *(int32_t*)p = BLOCK_VERSION_2;
+
+ p += sizeof(int32_t);
+ p += sizeof(int32_t);
+ p += sizeof(int32_t);
+ int32_t cols = *(int32_t*)p;
+ p += sizeof(int32_t);
+ p += sizeof(int32_t);
+ p += sizeof(uint64_t);
+ // check fields
+ p += cols * (sizeof(int8_t) + sizeof(int32_t));
+
+ int32_t* colLength = (int32_t*)p;
+ for (int32_t i = 0; i < cols; ++i) {
+ colLength[i] = htonl(colLength[i]);
+ }
+}
+
+static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
(*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
pRspObj->vgId = pWrapper->vgHandle->vgId;
pRspObj->resIter = -1;
- memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
@@ -1633,41 +1655,44 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
}
// extract the rows in this data packet
for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
- SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
- int64_t rows = htobe64(pRetrieve->numOfRows);
+ void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i);
+ void* rawData = NULL;
+ int64_t rows = 0;
+ // deal with compatibility
+ if(*(int64_t*)pRetrieve == 0){
+ rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
+ rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
+ }else if(*(int64_t*)pRetrieve == 1){
+ rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
+ rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
+ }
+
pVg->numOfRows += rows;
(*numOfRows) += rows;
-
- if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
- SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
- if (schema) {
+ changeByteEndian(rawData);
+ if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
+ SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
+ if(schema){
taosArrayPush(pRspObj->rsp.blockSchema, &schema);
}
}
}
+}
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
+ SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
+ pRspObj->resType = RES_TYPE__TMQ;
+ memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
+ tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj);
return pRspObj;
}
SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
pRspObj->resType = RES_TYPE__TMQ_METADATA;
- tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
- tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
- pRspObj->vgId = pWrapper->vgHandle->vgId;
- pRspObj->resIter = -1;
memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
- pRspObj->resInfo.totalRows = 0;
- pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-
- // extract the rows in this data packet
- for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
- SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
- int64_t rows = htobe64(pRetrieve->numOfRows);
- pVg->numOfRows += rows;
- (*numOfRows) += rows;
- }
+ tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj);
return pRspObj;
}
diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c
new file mode 100644
index 0000000000..b41343443d
--- /dev/null
+++ b/source/client/src/slowQueryMonitor.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "clientLog.h"
+#include "tglobal.h"
+
+const char* slowQueryName = "taos_slow_sql:count";
+const char* slowQueryHelp = "slow query log when cost over than config duration";
+const int slowQueryLabelCount = 4;
+const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"};
+static const char* defaultClusterID = "";
+
+const int64_t usInSeconds = 1000 * 1000;
+const int64_t msInMinutes = 60 * 1000;
+
+static const char* getSlowQueryLableCostDesc(int64_t cost) {
+ if (cost >= 1000 * usInSeconds) {
+ return "1000s-";
+ } else if (cost >= 100 * usInSeconds) {
+ return "100-1000s";
+ } else if (cost >= 10 * usInSeconds) {
+ return "10-100s";
+ } else if (cost >= 3 * usInSeconds) {
+ return "3-10s";
+ }
+ return "0-3s";
+}
+
+void clientSlowQueryMonitorInit(const char* clusterKey) {
+ if (!tsEnableMonitor) return;
+ SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey);
+ SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+ clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter);
+ createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels);
+}
+
+void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) {
+ const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)};
+ taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues);
+}
+
+void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) {
+ if (!tsEnableMonitor) return;
+ SQL_RESULT_CODE result = SQL_RESULT_SUCCESS;
+ if (TSDB_CODE_SUCCESS != code) {
+ result = SQL_RESULT_FAILED;
+ }
+ // to do Distinguish active Kill events
+ // else if (killed) {
+ // result = SQL_RESULT_CANCEL;
+ // }
+
+ STscObj* pTscObj = acquireTscObj(rid);
+ if (pTscObj != NULL) {
+ if(pTscObj->pAppInfo == NULL) {
+ tscLog("SlowQueryLog, not found pAppInfo");
+ } else {
+ clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost);
+ }
+ releaseTscObj(rid);
+ } else {
+ tscLog("SlowQueryLog, not found rid");
+ }
+}
diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt
index 91f0d1eef8..7c3847e4a1 100644
--- a/source/client/test/CMakeLists.txt
+++ b/source/client/test/CMakeLists.txt
@@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES(
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
)
+ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
+TARGET_LINK_LIBRARIES(
+ clientMonitorTest
+ PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
+)
+
TARGET_INCLUDE_DIRECTORIES(
clientTest
PUBLIC "${TD_SOURCE_DIR}/include/client/"
@@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES(
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
+TARGET_INCLUDE_DIRECTORIES(
+ clientMonitorTest
+ PUBLIC "${TD_SOURCE_DIR}/include/client/"
+ PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
+)
+
add_test(
NAME smlTest
COMMAND smlTest
)
+
+# add_test(
+# NAME clientMonitorTest
+# COMMAND clientMonitorTest
+# )
diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp
new file mode 100644
index 0000000000..f7ddc1a5cd
--- /dev/null
+++ b/source/client/test/clientMonitorTests.cpp
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include
+#include "clientInt.h"
+#include "clientMonitor.h"
+#include "taoserror.h"
+#include "tglobal.h"
+#include "thash.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "executor.h"
+#include "taos.h"
+
+int main(int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+TEST(clientMonitorTest, monitorTest) {
+ const char* cluster1 = "cluster1";
+ const char* cluster2 = "cluster2";
+ SEpSet epSet;
+ clusterMonitorInit(cluster1, epSet, NULL);
+ const char* counterName1 = "slow_query";
+ const char* counterName2 = "select_count";
+ const char* help1 = "test for slowQuery";
+ const char* help2 = "test for selectSQL";
+ const char* lables[] = {"lable1"};
+ taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables);
+ ASSERT_TRUE(c1 != NULL);
+ taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables);
+ ASSERT_TRUE(c2 != NULL);
+ ASSERT_TRUE(c1 != c2);
+ taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+ ASSERT_TRUE(c21 == NULL);
+ clusterMonitorInit(cluster2, epSet, NULL);
+ c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables);
+ ASSERT_TRUE(c21 != NULL);
+ int i = 0;
+ while (i < 12) {
+ taosMsleep(10);
+ ++i;
+ }
+ clusterMonitorClose(cluster1);
+ clusterMonitorClose(cluster2);
+}
+
+TEST(clientMonitorTest, sendTest) {
+ TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
+ ASSERT_TRUE(taos != NULL);
+ printf("connect taosd sucessfully.\n");
+
+ int64_t rid = *(int64_t *)taos;
+ SlowQueryLog(rid, false, -1, 1000);
+ int i = 0;
+ while (i < 20) {
+ SlowQueryLog(rid, false, 0, i * 1000);
+ taosMsleep(10);
+ ++i;
+ }
+
+ taos_close(taos);
+}
diff --git a/source/common/src/systable.c b/source/common/src/systable.c
index 47eac317ec..25cc5d7c79 100644
--- a/source/common/src/systable.c
+++ b/source/common/src/systable.c
@@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = {
};
static const SSysDbTableSchema userUsersSchema[] = {
- {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
- {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
- {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
- {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
- {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+ {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+ {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
+ {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
+ {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
GRANTS_SCHEMA;
@@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = {
};
static const SSysDbTableSchema variablesSchema[] = {
- {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+ {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = {
static const SSysDbTableSchema vnodesSchema[] = {
{.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
- {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+ {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true},
{.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
@@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
};
static const SSysDbTableSchema userUserPrivilegesSchema[] = {
- {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema userViewsSchema[] = {
@@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
};
static const SSysDbTableSchema useGrantsFullSchema[] = {
- {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema useGrantsLogsSchema[] = {
- {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysDbTableSchema useMachinesSchema[] = {
- {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
- {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+ {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+ {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
};
static const SSysTableMeta infosMeta[] = {
@@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
- {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false},
- {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false},
- {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false},
+ {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true},
+ {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true},
+ {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true},
};
static const SSysDbTableSchema connectionsSchema[] = {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 2744ddfd59..2412f10db4 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -2288,28 +2288,28 @@ _end:
return TSDB_CODE_SUCCESS;
}
-void buildCtbNameAddGruopId(char* ctbName, uint64_t groupId){
+void buildCtbNameAddGroupId(char* ctbName, uint64_t groupId){
char tmp[TSDB_TABLE_NAME_LEN] = {0};
snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0; // put groupId to the end
strcat(ctbName, tmp);
}
-bool isAutoTableName(char* ctbName){
- return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_');
-}
+// auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.
+// the total length is fixed to be 34 bytes.
+bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); }
-bool alreadyAddGroupId(char* ctbName){
+bool alreadyAddGroupId(char* ctbName) {
size_t len = strlen(ctbName);
size_t _location = len - 1;
- while(_location > 0){
- if(ctbName[_location] < '0' || ctbName[_location] > '9'){
+ while (_location > 0) {
+ if (ctbName[_location] < '0' || ctbName[_location] > '9') {
break;
}
_location--;
}
- return ctbName[_location] == '_' && len - 1 - _location > 15; //15 means the min length of groupid
+ return ctbName[_location] == '_' && len - 1 - _location >= 15; // 15 means the min length of groupid
}
char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
@@ -2366,7 +2366,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
// todo extract method
int32_t* version = (int32_t*)data;
- *version = 2;
+ *version = BLOCK_VERSION_1;
data += sizeof(int32_t);
int32_t* actualLen = (int32_t*)data;
@@ -2447,7 +2447,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
data += colSizes[col];
}
-// colSizes[col] = htonl(colSizes[col]);
+ colSizes[col] = htonl(colSizes[col]);
// uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
// htonl(colSizes[col]), colSizes[col]);
}
@@ -2512,9 +2512,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
pStart += sizeof(int32_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
- if(version == 1){
- colLen[i] = htonl(colLen[i]);
- }
+ colLen[i] = htonl(colLen[i]);
ASSERT(colLen[i] >= 0);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index ee85a909e7..3d8c7c7f8b 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
int32_t tsNumOfMnodeFetchThreads = 1;
int32_t tsNumOfMnodeReadThreads = 1;
int32_t tsNumOfVnodeQueryThreads = 4;
-float tsRatioOfVnodeStreamThreads = 0.5F;
+float tsRatioOfVnodeStreamThreads = 1.5F;
int32_t tsNumOfVnodeFetchThreads = 4;
int32_t tsNumOfVnodeRsmaThreads = 2;
int32_t tsNumOfQnodeQueryThreads = 4;
@@ -94,6 +94,9 @@ char tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
uint16_t tsMonitorPort = 6043;
int32_t tsMonitorMaxLogs = 100;
bool tsMonitorComp = false;
+bool tsMonitorLogProtocol = false;
+int32_t tsMonitorIntervalForBasic = 30;
+bool tsMonitorForceV2 = true;
// audit
bool tsEnableAudit = true;
@@ -266,6 +269,7 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024;
bool tsFilterScalarMode = false;
int tsResolveFQDNRetryTime = 100; // seconds
int tsStreamAggCnt = 1000;
+bool tsDisableCount = true;
char tsS3Endpoint[TSDB_FQDN_LEN] = "";
char tsS3AccessKey[TSDB_FQDN_LEN] = "";
@@ -535,6 +539,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
+ if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+ if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
+ if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
return 0;
}
@@ -693,7 +701,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-
+ if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+ if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+ return -1;
+ if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+
if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -1081,6 +1093,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64;
tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32;
tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32;
+ tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
+ tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
return -1;
}
@@ -1095,6 +1109,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
+
+ tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval;
return 0;
}
@@ -1155,7 +1171,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32;
-
+ tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval;
+ tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32;
+ tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32;
+
tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32;
@@ -1718,7 +1737,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter},
- {"experimental", &tsExperimental}};
+ {"experimental", &tsExperimental},
+ {"disableCount", &tsDisableCount}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 2ffa12f2c1..4640c20e07 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -1436,6 +1436,44 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); }
+int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, buf, bufLen);
+
+ if (tStartEncode(&encoder) < 0) return -1;
+
+ if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1;
+ if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1;
+
+ tEndEncode(&encoder);
+
+ int32_t tlen = encoder.pos;
+ tEncoderClear(&encoder);
+ return tlen;
+}
+
+int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) {
+ SDecoder decoder = {0};
+ tDecoderInit(&decoder, buf, bufLen);
+
+ if (tStartDecode(&decoder) < 0) return -1;
+
+ if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1;
+ if (pReq->contLen > 0) {
+ pReq->pCont = taosMemoryMalloc(pReq->contLen + 1);
+ if (pReq->pCont == NULL) return -1;
+ if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1;
+ }
+
+ tEndDecode(&decoder);
+ tDecoderClear(&decoder);
+ return 0;
+}
+
+void tFreeSStatisReq(SStatisReq *pReq) {
+ taosMemoryFreeClear(pReq->pCont);
+}
+
int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@@ -3280,18 +3318,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq)
return tlen;
}
-int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
-
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1;
- tEndDecode(&decoder);
-
- tDecoderClear(&decoder);
- return 0;
-}
-
int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@@ -3305,17 +3331,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq)
return tlen;
}
-int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) {
SEncoder encoder = {0};
@@ -4233,17 +4259,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq
return tlen;
}
-int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) {
if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1;
@@ -4325,24 +4351,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
return tlen;
}
-int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
- if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
- if (pReq->payloadLen > 0) {
- pReq->payload = taosMemoryMalloc(pReq->payloadLen);
- if (pReq->payload == NULL) return -1;
- if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
- }
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1;
+// if (pReq->payloadLen > 0) {
+// pReq->payload = taosMemoryMalloc(pReq->payloadLen);
+// if (pReq->payload == NULL) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1;
+// }
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); }
@@ -4595,34 +4621,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) {
taosArrayDestroy(pRsp->pIndexRsp);
}
-int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
- SEncoder encoder = {0};
- tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+// SEncoder encoder = {0};
+// tEncoderInit(&encoder, buf, bufLen);
- if (tStartEncode(&encoder) < 0) return -1;
- if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
- if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
- tEndEncode(&encoder);
+// if (tStartEncode(&encoder) < 0) return -1;
+// if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1;
+// if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1;
+// tEndEncode(&encoder);
- int32_t tlen = encoder.pos;
- tEncoderClear(&encoder);
- return tlen;
-}
+// int32_t tlen = encoder.pos;
+// tEncoderClear(&encoder);
+// return tlen;
+// }
-int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
- if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1;
+// if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1;
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
-void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
+// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); }
int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
@@ -4930,17 +4956,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
return tlen;
}
-int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
- tEndDecode(&decoder);
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
+// tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
SEncoder encoder = {0};
@@ -4955,17 +4981,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR
return tlen;
}
-int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
- tEndDecode(&decoder);
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
+// tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
@@ -5830,17 +5856,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom
return tlen;
}
-int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
- tEndDecode(&decoder);
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+// tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) {
SEncoder encoder = {0};
@@ -5889,38 +5915,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq
return 0;
}
-int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
- SEncoder encoder = {0};
- tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+// SEncoder encoder = {0};
+// tEncoderInit(&encoder, buf, bufLen);
- if (tStartEncode(&encoder) < 0) return -1;
- if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
- if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
- if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
- tEndEncode(&encoder);
+// if (tStartEncode(&encoder) < 0) return -1;
+// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+// if (tEncodeI8(&encoder, pReq->spi) < 0) return -1;
+// if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1;
+// if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1;
+// if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1;
+// tEndEncode(&encoder);
- int32_t tlen = encoder.pos;
- tEncoderClear(&encoder);
- return tlen;
-}
+// int32_t tlen = encoder.pos;
+// tEncoderClear(&encoder);
+// return tlen;
+// }
-int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
- if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
- if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
- tEndDecode(&decoder);
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+// if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1;
+// if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1;
+// tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) {
SEncoder encoder = {0};
@@ -6958,30 +6984,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
-int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
- // SEncoder encoder = {0};
- // tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+// // SEncoder encoder = {0};
+// // tEncoderInit(&encoder, buf, bufLen);
- // if (tStartEncode(&encoder) < 0) return -1;
- // if (pRsp->rspList) {
- // int32_t num = taosArrayGetSize(pRsp->rspList);
- // if (tEncodeI32(&encoder, num) < 0) return -1;
- // for (int32_t i = 0; i < num; ++i) {
- // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
- // if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
- // }
- // } else {
- // if (tEncodeI32(&encoder, 0) < 0) return -1;
- // }
- // tEndEncode(&encoder);
+// // if (tStartEncode(&encoder) < 0) return -1;
+// // if (pRsp->rspList) {
+// // int32_t num = taosArrayGetSize(pRsp->rspList);
+// // if (tEncodeI32(&encoder, num) < 0) return -1;
+// // for (int32_t i = 0; i < num; ++i) {
+// // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
+// // if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
+// // }
+// // } else {
+// // if (tEncodeI32(&encoder, 0) < 0) return -1;
+// // }
+// // tEndEncode(&encoder);
- // int32_t tlen = encoder.pos;
- // tEncoderClear(&encoder);
- // reture tlen;
- return 0;
-}
+// // int32_t tlen = encoder.pos;
+// // tEncoderClear(&encoder);
+// // reture tlen;
+// return 0;
+// }
-int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
+//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
// SDecoder decoder = {0};
// int32_t num = 0;
// tDecoderInit(&decoder, buf, bufLen);
@@ -7002,8 +7028,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc
// tEndDecode(&decoder);
// tDecoderClear(&decoder);
- return 0;
-}
+// return 0;
+//}
int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
int32_t nRsps = taosArrayGetSize(pRsp->pArray);
@@ -7138,15 +7164,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
return 0;
}
-int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
- if (tStartDecode(pCoder) < 0) return -1;
+// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
+// if (tStartDecode(pCoder) < 0) return -1;
- if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
- if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
+// if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
+// if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
- tEndDecode(pCoder);
- return 0;
-}
+// tEndDecode(pCoder);
+// return 0;
+// }
int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
@@ -7427,34 +7453,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq *
void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); }
-int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
- SEncoder encoder = {0};
- tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
+// SEncoder encoder = {0};
+// tEncoderInit(&encoder, buf, bufLen);
- if (tStartEncode(&encoder) < 0) return -1;
- if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
- if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+// if (tStartEncode(&encoder) < 0) return -1;
+// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
- tEndEncode(&encoder);
+// tEndEncode(&encoder);
- int32_t tlen = encoder.pos;
- tEncoderClear(&encoder);
- return tlen;
-}
+// int32_t tlen = encoder.pos;
+// tEncoderClear(&encoder);
+// return tlen;
+// }
-int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) {
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
- if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
- tEndDecode(&decoder);
+// tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// tDecoderClear(&decoder);
+// return 0;
+// }
void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
if (NULL == pReq) {
@@ -7513,6 +7539,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
if (pReq->alterOriDataLen > 0) {
if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1;
}
+ if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
tEndEncode(pCoder);
return 0;
@@ -7535,6 +7562,10 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1;
}
+ if (!tDecodeIsEnd(pCoder)) {
+ if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+ }
+
tEndDecode(pCoder);
return 0;
}
@@ -7663,6 +7694,8 @@ int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq)
if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
}
+ if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
+
tEndEncode(pCoder);
return 0;
}
@@ -7677,6 +7710,10 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
}
+ if (!tDecodeIsEnd(pCoder)) {
+ if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+ }
+
tEndDecode(pCoder);
return 0;
}
@@ -7843,37 +7880,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) {
return 0;
}
-static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
- if (tStartEncode(pCoder) < 0) return -1;
+// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) {
+// if (tStartEncode(pCoder) < 0) return -1;
- if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
- if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
- if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
- if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
+// if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1;
+// if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1;
+// if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1;
+// if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1;
- if (flags & TD_AUTO_CREATE_TABLE) {
- if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
- }
+// if (flags & TD_AUTO_CREATE_TABLE) {
+// if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+// }
- tEndEncode(pCoder);
- return 0;
-}
+// tEndEncode(pCoder);
+// return 0;
+// }
-static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
- if (tStartDecode(pCoder) < 0) return -1;
+// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) {
+// if (tStartDecode(pCoder) < 0) return -1;
- if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
- if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
- if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
- if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
+// if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1;
+// if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1;
+// if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1;
+// if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1;
- if (flags & TD_AUTO_CREATE_TABLE) {
- if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
- }
+// if (flags & TD_AUTO_CREATE_TABLE) {
+// if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1;
+// }
- tEndDecode(pCoder);
- return 0;
-}
+// tEndDecode(pCoder);
+// return 0;
+// }
static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) {
if (tStartEncode(pEncoder) < 0) return -1;
@@ -7956,19 +7993,19 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
return 0;
}
-void tFreeSSubmitBlkRsp(void *param) {
- if (NULL == param) {
- return;
- }
+// void tFreeSSubmitBlkRsp(void *param) {
+// if (NULL == param) {
+// return;
+// }
- SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
+// SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
- taosMemoryFree(pRsp->tblFName);
- if (pRsp->pMeta) {
- taosMemoryFree(pRsp->pMeta->pSchemas);
- taosMemoryFree(pRsp->pMeta);
- }
-}
+// taosMemoryFree(pRsp->tblFName);
+// if (pRsp->pMeta) {
+// taosMemoryFree(pRsp->pMeta->pSchemas);
+// taosMemoryFree(pRsp->pMeta);
+// }
+// }
void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
if (NULL == pRsp) return;
@@ -8034,6 +8071,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
break;
}
if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
+ if (tEncodeI8(pEncoder, pReq->source) < 0) return -1;
tEndEncode(pEncoder);
return 0;
@@ -8094,6 +8132,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
}
+ if (!tDecodeIsEnd(pDecoder)) {
+ if (tDecodeI8(pDecoder, &pReq->source) < 0) return -1;
+ }
tEndDecode(pDecoder);
return 0;
@@ -8138,23 +8179,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
return 0;
}
-int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
- int32_t meta = 0;
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
+// int32_t meta = 0;
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
- if (tDecodeI32(&decoder, &meta) < 0) return -1;
- if (meta) {
- pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
- if (NULL == pRsp->pMeta) return -1;
- if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
- }
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1;
+// if (tDecodeI32(&decoder, &meta) < 0) return -1;
+// if (meta) {
+// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+// if (NULL == pRsp->pMeta) return -1;
+// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+// }
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) {
if (tStartEncode(pEncoder) < 0) return -1;
@@ -8179,22 +8220,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) {
return 0;
}
-int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
- int32_t meta = 0;
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
+// int32_t meta = 0;
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &meta) < 0) return -1;
- if (meta) {
- pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
- if (NULL == pRsp->pMeta) return -1;
- if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
- }
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &meta) < 0) return -1;
+// if (meta) {
+// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+// if (NULL == pRsp->pMeta) return -1;
+// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+// }
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
if (NULL == pRsp) {
@@ -8230,22 +8271,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) {
return 0;
}
-int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
- int32_t meta = 0;
- SDecoder decoder = {0};
- tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) {
+// int32_t meta = 0;
+// SDecoder decoder = {0};
+// tDecoderInit(&decoder, buf, bufLen);
- if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &meta) < 0) return -1;
- if (meta) {
- pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
- if (NULL == pRsp->pMeta) return -1;
- if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
- }
- tEndDecode(&decoder);
- tDecoderClear(&decoder);
- return 0;
-}
+// if (tStartDecode(&decoder) < 0) return -1;
+// if (tDecodeI32(&decoder, &meta) < 0) return -1;
+// if (meta) {
+// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+// if (NULL == pRsp->pMeta) return -1;
+// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1;
+// }
+// tEndDecode(&decoder);
+// tDecoderClear(&decoder);
+// return 0;
+// }
void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) {
if (NULL == pRsp) {
@@ -8670,7 +8711,6 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
}
}
if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
- if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1;
tEndEncode(pCoder);
return 0;
@@ -8758,12 +8798,6 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
goto _exit;
}
}
- if (!tDecodeIsEnd(pCoder)) {
- if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) {
- code = TSDB_CODE_INVALID_MSG;
- goto _exit;
- }
- }
tEndDecode(pCoder);
diff --git a/source/common/src/trow.c b/source/common/src/trow.c
index b91562be7a..6e9278c630 100644
--- a/source/common/src/trow.c
+++ b/source/common/src/trow.c
@@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa
return TSDB_CODE_SUCCESS;
}
+#if 0
int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
if (!pBitmap || colIdx < 0) {
terrno = TSDB_CODE_INVALID_PARA;
@@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal
}
return TSDB_CODE_SUCCESS;
}
+#endif
int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
switch (bitmapMode) {
case 0:
tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
break;
+#if 0
case -1:
case 1:
tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
break;
+#endif
default:
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
@@ -231,7 +235,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe
return true;
}
-#ifdef BUILD_NO_CALL
+// #ifdef BUILD_NO_CALL
const uint8_t tdVTypeByte[2][3] = {{
// 2 bits
TD_VTYPE_NORM_BYTE_II,
@@ -439,13 +443,15 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) {
return true;
}
-int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
+int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) {
STColumn *pTColumn;
SColVal *pColVal;
int32_t nColVal = taosArrayGetSize(pArray);
int32_t varDataLen = 0;
+ int32_t nonVarDataLen = 0;
int32_t maxVarDataLen = 0;
int32_t iColVal = 0;
+ int32_t nBound = 0;
void *varBuf = NULL;
bool isAlloc = false;
@@ -458,6 +464,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
} else {
pColVal = NULL;
}
+ if (pColVal && !COL_VAL_IS_NONE(pColVal)) {
+ ++nBound;
+ }
if (iColumn == 0) {
ASSERT(pColVal && pColVal->cid == pTColumn->colId);
@@ -484,16 +493,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
}
}
}
+ } else {
+ if(pColVal && COL_VAL_IS_VALUE(pColVal)) {
+ nonVarDataLen += TYPE_BYTES[pTColumn->type];
+ }
}
}
++iColVal;
}
- if (!(*ppRow)) {
- *ppRow = (STSRow *)taosMemoryCalloc(
- 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
- isAlloc = true;
+ int32_t rowTotalLen = 0;
+ if (rowType == TD_ROW_TP) {
+ rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1);
+ } else {
+ rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) +
+ TD_BITMAP_BYTES(nBound - 1);
+ }
+ if (!(*ppRow)) {
+ *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen);
+ isAlloc = true;
}
if (!(*ppRow)) {
@@ -512,10 +531,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
}
}
- SRowBuilder rb = {0};
+ SRowBuilder rb = {.rowType = rowType};
tdSRowInit(&rb, pTSchema->version);
- tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen);
+ tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen);
tdSRowResetBuf(&rb, *ppRow);
+ int32_t iBound = 0;
iColVal = 0;
for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) {
@@ -529,21 +549,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
valType = TD_VTYPE_NONE;
} else if (COL_VAL_IS_NULL(pColVal)) {
valType = TD_VTYPE_NULL;
+ ++iBound;
} else if (IS_VAR_DATA_TYPE(pTColumn->type)) {
varDataSetLen(varBuf, pColVal->value.nData);
if (pColVal->value.nData != 0) {
memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
}
val = varBuf;
+ ++iBound;
} else {
val = (const void *)&pColVal->value.val;
+ ++iBound;
}
} else {
// pColVal = NULL;
valType = TD_VTYPE_NONE;
}
- tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+ if (TD_IS_TP_ROW(rb.pBuf)) {
+ tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal);
+ } else {
+ tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1);
+ }
++iColVal;
}
@@ -605,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell
return true;
}
-
+#if 0
int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
if (!pBitmap || colIdx < 0) {
terrno = TSDB_CODE_INVALID_PARA;
@@ -655,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
}
return TSDB_CODE_SUCCESS;
}
+#endif
int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) {
#ifdef TD_SUPPORT_BITMAP
@@ -761,11 +789,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp
int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) {
- if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) {
+ if (colIdx < 1) {
terrno = TSDB_CODE_INVALID_PARA;
+ ASSERTS(0, "colIdx is %" PRIi64, colIdx);
return terrno;
}
- offset -= sizeof(SKvRowIdx);
--colIdx;
#ifdef TD_SUPPORT_BITMAP
@@ -779,6 +807,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
pColIdx->colId = colId;
pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN
+ pBuilder->offset += sizeof(SKvRowIdx);
if (valType == TD_VTYPE_NORM) {
char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
if (IS_VAR_DATA_TYPE(colType)) {
@@ -830,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo
return 0;
}
-
+#if 0
int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,
int32_t allNullLen, int32_t boundNullLen) {
if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) {
@@ -866,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou
#endif
return TSDB_CODE_SUCCESS;
}
+#endif
int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
pBuilder->pBuf = (STSRow *)pBuf;
@@ -904,6 +934,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
TD_ROW_SET_LEN(pBuilder->pBuf, len);
TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols);
+ pBuilder->offset = 0;
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
@@ -1040,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
case 0:
tdSetBitmapValTypeII(pBitmap, colIdx, valType);
break;
+#if 0
case -1:
case 1:
tdSetBitmapValTypeI(pBitmap, colIdx, valType);
break;
+#endif
default:
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
@@ -1056,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int
void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) {
STColumn *pTColumn = &pTSchema->columns[iCol];
SCellVal cv = {0};
- SValue value = {0};
ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0));
@@ -1084,5 +1116,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV
memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes);
}
}
-}
-#endif
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt
index eb79e79afa..51c733e796 100644
--- a/source/common/test/CMakeLists.txt
+++ b/source/common/test/CMakeLists.txt
@@ -35,6 +35,10 @@ target_include_directories(
PUBLIC "${TD_SOURCE_DIR}/include/common"
PUBLIC "${TD_SOURCE_DIR}/include/util"
)
+add_test(
+ NAME dataformatTest
+ COMMAND dataformatTest
+)
# tmsg test
# add_executable(tmsgTest "")
diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
index b1083e4590..8bc7d47ebf 100644
--- a/source/common/test/dataformatTest.cpp
+++ b/source/common/test/dataformatTest.cpp
@@ -13,7 +13,6 @@
* along with this program. If not, see .
*/
-#if 0
#include
#include
@@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) {
case 1: {
pSchema[i].type = TSDB_DATA_TYPE_INT;
pSchema[i].bytes = TYPE_BYTES[pSchema[i].type];
- ;
} break;
case 2: {
pSchema[i].type = TSDB_DATA_TYPE_BIGINT;
@@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) {
return pResult;
}
-// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool
+// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
+// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
if (!(*pArray)) {
*pArray = taosArrayInit(nCols, sizeof(SColVal));
@@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) {
taosArrayPush(*pArray, &colVal);
continue;
}
-
switch (i) {
case 0:
- sscanf(data[i], "%" PRIi64, &colVal.value.ts);
+ colVal.type = TSDB_DATA_TYPE_TIMESTAMP;
+ sscanf(data[i], "%" PRIi64, &colVal.value.val);
break;
case 1:
- sscanf(data[i], "%" PRIi32, &colVal.value.i32);
+ colVal.type = TSDB_DATA_TYPE_INT;
+ sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val);
break;
case 2:
- sscanf(data[i], "%" PRIi64, &colVal.value.i64);
+ colVal.type = TSDB_DATA_TYPE_BIGINT;
+ sscanf(data[i], "%" PRIi64, &colVal.value.val);
break;
case 3:
- sscanf(data[i], "%f", &colVal.value.f);
+ colVal.type = TSDB_DATA_TYPE_FLOAT;
+ sscanf(data[i], "%f", (float *)&colVal.value.val);
break;
case 4:
- sscanf(data[i], "%lf", &colVal.value.d);
+ colVal.type = TSDB_DATA_TYPE_DOUBLE;
+ sscanf(data[i], "%lf", (double *)&colVal.value.val);
break;
case 5: {
+ colVal.type = TSDB_DATA_TYPE_BINARY;
int16_t dataLen = strlen(data[i]) + 1;
colVal.value.nData = dataLen < 10 ? dataLen : 10;
colVal.value.pData = (uint8_t *)data[i];
} break;
case 6: {
+ colVal.type = TSDB_DATA_TYPE_NCHAR;
int16_t dataLen = strlen(data[i]) + 1;
colVal.value.nData = dataLen < 40 ? dataLen : 40;
colVal.value.pData = (uint8_t *)data[i]; // just for test, not real nchar
} break;
- case 7:
- case 9: {
+ case 7: {
+ colVal.type = TSDB_DATA_TYPE_TINYINT;
int32_t d8;
sscanf(data[i], "%" PRId32, &d8);
- colVal.value.i8 = (int8_t)d8;
- } break;
+ colVal.value.val = (int8_t)d8;
+ }
case 8: {
+ colVal.type = TSDB_DATA_TYPE_SMALLINT;
int32_t d16;
sscanf(data[i], "%" PRId32, &d16);
- colVal.value.i16 = (int16_t)d16;
+ colVal.value.val = (int16_t)d16;
+ } break;
+ case 9: {
+ colVal.type = TSDB_DATA_TYPE_BOOL;
+ int32_t d8;
+ sscanf(data[i], "%" PRId32, &d8);
+ colVal.value.val = (int8_t)d8;
} break;
case 10: {
+ colVal.type = TSDB_DATA_TYPE_UTINYINT;
uint32_t u8;
sscanf(data[i], "%" PRId32, &u8);
- colVal.value.u8 = (uint8_t)u8;
+ colVal.value.val = (uint8_t)u8;
} break;
case 11: {
+ colVal.type = TSDB_DATA_TYPE_USMALLINT;
uint32_t u16;
sscanf(data[i], "%" PRId32, &u16);
- colVal.value.u16 = (uint16_t)u16;
+ colVal.value.val = (uint16_t)u16;
} break;
case 12: {
- sscanf(data[i], "%" PRIu32, &colVal.value.u32);
+ colVal.type = TSDB_DATA_TYPE_UINT;
+ sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val);
} break;
case 13: {
- sscanf(data[i], "%" PRIu64, &colVal.value.u64);
+ colVal.type = TSDB_DATA_TYPE_UBIGINT;
+ sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val);
} break;
default:
ASSERT(0);
@@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
}
switch (type) {
case TSDB_DATA_TYPE_BOOL:
- printf("%s ", cv->value.i8 == 0 ? "false" : "true");
+ printf("%s ", cv->value.val == 0 ? "false" : "true");
break;
case TSDB_DATA_TYPE_TINYINT:
- printf("%" PRIi8 " ", cv->value.i8);
+ printf("%" PRIi8 " ", *(int8_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_SMALLINT:
- printf("%" PRIi16 " ", cv->value.i16);
+ printf("%" PRIi16 " ", *(int16_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_INT:
- printf("%" PRIi32 " ", cv->value.i32);
+ printf("%" PRIi32 " ", *(int32_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_BIGINT:
- printf("%" PRIi64 " ", cv->value.i64);
+ printf("%" PRIi64 " ", cv->value.val);
break;
case TSDB_DATA_TYPE_FLOAT:
- printf("%f ", cv->value.f);
+ printf("%f ", *(float *)&cv->value.val);
break;
case TSDB_DATA_TYPE_DOUBLE:
- printf("%lf ", cv->value.d);
+ printf("%lf ", *(double *)&cv->value.val);
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
@@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
printf("%s ", tv);
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
- printf("%" PRIi64 " ", cv->value.i64);
+ printf("%" PRIi64 " ", cv->value.val);
break;
case TSDB_DATA_TYPE_NCHAR: {
char tv[15] = {0};
@@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
printf("%s ", tv);
} break;
case TSDB_DATA_TYPE_UTINYINT:
- printf("%" PRIu8 " ", cv->value.u8);
+ printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_USMALLINT:
- printf("%" PRIu16 " ", cv->value.u16);
+ printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_UINT:
- printf("%" PRIu32 " ", cv->value.u32);
+ printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_UBIGINT:
- printf("%" PRIu64 " ", cv->value.u64);
+ printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val);
break;
case TSDB_DATA_TYPE_JSON:
printf("JSON ");
@@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
return 0;
}
-void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
- // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) {
+ printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len);
for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
SColVal cv = {0};
- tTSRowGet(row, pTSchema, i, &cv);
+ tTSRowGetVal(row, pTSchema, i, &cv);
debugPrintSColVal(&cv, pTSchema->columns[i].type);
}
printf("\n");
@@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
case TSDB_DATA_TYPE_TINYINT: {
int32_t d8;
sscanf(rawVal, "%" PRId32, &d8);
- EXPECT_EQ(cv->value.i8, (int8_t)d8);
+ EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8);
} break;
case TSDB_DATA_TYPE_SMALLINT: {
int32_t d16;
sscanf(rawVal, "%" PRId32, &d16);
- EXPECT_EQ(cv->value.i16, (int16_t)d16);
+ EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16);
} break;
case TSDB_DATA_TYPE_INT: {
- sscanf(rawVal, "%" PRId32, &rawSVal.i32);
- EXPECT_EQ(cv->value.i32, rawSVal.i32);
+ sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val);
+ EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val);
} break;
case TSDB_DATA_TYPE_BIGINT: {
- sscanf(rawVal, "%" PRIi64, &rawSVal.i64);
- EXPECT_EQ(cv->value.i64, rawSVal.i64);
+ sscanf(rawVal, "%" PRIi64, &rawSVal.val);
+ EXPECT_EQ(cv->value.val, rawSVal.val);
} break;
case TSDB_DATA_TYPE_FLOAT: {
- sscanf(rawVal, "%f", &rawSVal.f);
- EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f);
+ sscanf(rawVal, "%f", (float *)&rawSVal.val);
+ EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val);
} break;
case TSDB_DATA_TYPE_DOUBLE: {
- sscanf(rawVal, "%lf", &rawSVal.d);
- EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d);
+ sscanf(rawVal, "%lf", (double *)&rawSVal.val);
+ EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val);
} break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData);
} break;
case TSDB_DATA_TYPE_TIMESTAMP: {
- sscanf(rawVal, "%" PRIi64, &rawSVal.ts);
- EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts);
+ sscanf(rawVal, "%" PRIi64, &rawSVal.val);
+ EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val);
} break;
case TSDB_DATA_TYPE_NCHAR: {
EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); // informal nchar comparsion
@@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
case TSDB_DATA_TYPE_UTINYINT: {
uint32_t u8;
sscanf(rawVal, "%" PRIu32, &u8);
- EXPECT_EQ(cv->value.u8, (uint8_t)u8);
+ EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8);
} break;
case TSDB_DATA_TYPE_USMALLINT: {
uint32_t u16;
sscanf(rawVal, "%" PRIu32, &u16);
- EXPECT_EQ(cv->value.u16, (uint16_t)u16);
+ EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16);
} break;
case TSDB_DATA_TYPE_UINT: {
- sscanf(rawVal, "%" PRIu32, &rawSVal.u32);
- EXPECT_EQ(cv->value.u32, rawSVal.u32);
+ sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val);
+ EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val);
} break;
case TSDB_DATA_TYPE_UBIGINT: {
- sscanf(rawVal, "%" PRIu64, &rawSVal.u64);
- EXPECT_EQ(cv->value.u64, rawSVal.u64);
+ sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val);
+ EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val);
} break;
case TSDB_DATA_TYPE_JSON:
printf("JSON ");
@@ -395,36 +411,66 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
return 0;
}
-static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
+static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) {
for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
SColVal cv = {0};
- tTSRowGet(row, pTSchema, i, &cv);
+ tTSRowGetVal(row, pTSchema, i, &cv);
checkSColVal(data[i], &cv, pTSchema->columns[i].type);
}
+
+ STSRowIter rowIter = {0};
+ rowIter.pSchema = pTSchema;
+ tdSTSRowIterReset(&rowIter, row);
+ for (int32_t i = 0; i < pTSchema->numOfCols; ++i) {
+ STColumn *pCol = pTSchema->columns + i;
+ SColVal colVal = {0};
+ SCellVal cv = {0};
+ if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) {
+ break;
+ }
+
+ colVal.cid = pCol->colId;
+ colVal.type = pCol->type;
+ if (tdValTypeIsNone(cv.valType)) {
+ colVal.flag = CV_FLAG_NONE;
+ } else if (tdValTypeIsNull(cv.valType)) {
+ colVal.flag = CV_FLAG_NULL;
+ } else {
+ colVal.flag = CV_FLAG_VALUE;
+
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ colVal.value.nData = varDataLen(cv.val);
+ colVal.value.pData = (uint8_t *)varDataVal(cv.val);
+ } else {
+ memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes);
+ }
+ }
+ checkSColVal(data[i], &colVal, pCol->type);
+ }
}
TEST(testCase, AllNormTest) {
- int16_t nCols = 14;
- STSRowBuilder rb = {0};
- STSRow2 *row = nullptr;
- SArray *pArray = taosArrayInit(nCols, sizeof(SColVal));
+ int16_t nCols = 14;
+ STSRow *row = nullptr;
+ SArray *pArray = taosArrayInit(nCols, sizeof(SColVal));
EXPECT_NE(pArray, nullptr);
STSchema *pTSchema = genSTSchema(nCols);
EXPECT_NE(pTSchema, nullptr);
- // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
- // c9 bool
- char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"};
+ // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
+ // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
+ char *data[14] = {
+ "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"};
genTestData((const char **)&data, nCols, &pArray);
- tTSRowNew(&rb, pArray, pTSchema, &row);
-
+ tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP);
debugPrintTSRow(row, pTSchema, __func__, __LINE__);
+ tdSRowPrint(row, pTSchema, __func__);
checkTSRow((const char **)&data, row, pTSchema);
- tsRowBuilderClear(&rb);
+ taosMemoryFreeClear(row);
taosArrayDestroy(pArray);
taosMemoryFree(pTSchema);
}
@@ -433,7 +479,7 @@ TEST(testCase, AllNormTest) {
TEST(testCase, NoneTest) {
const static int nCols = 14;
const static int nRows = 20;
- STSRow2 *row = nullptr;
+ STSRow *row = nullptr;
SArray *pArray = taosArrayInit(nCols, sizeof(SColVal));
EXPECT_NE(pArray, nullptr);
@@ -442,42 +488,46 @@ TEST(testCase, NoneTest) {
// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint,
// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned
- const char *data[nRows][nCols] = {
- {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
- {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
- {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
- {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
- {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
- {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
- {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
- {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
- {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
- {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
- {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
- {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
- {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
- {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
- {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
- {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
- "18446744073709551615"},
- {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
- {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
- {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
- "4294967295", "100"},
- {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
- "no"}};
+ const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV,
+ TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV,
+ TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP};
- for (int r = 0; r < nRows; ++r) {
- genTestData((const char **)&data[r], nCols, &pArray);
- tTSRowNew(NULL, pArray, pTSchema, &row);
- debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print
- checkTSRow((const char **)&data[r], row, pTSchema); // check
- tTSRowFree(row);
- taosArrayClear(pArray);
+const char *data[nRows][nCols] = {
+ {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"},
+ {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+ {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+ {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"},
+ {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"},
+ {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+ {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"},
+ {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"},
+ {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"},
+ {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+ {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"},
+ {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"},
+ {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"},
+ {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"},
+ {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"},
+ {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu",
+ "18446744073709551615"},
+ {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"},
+ {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"},
+ {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu",
+ "4294967295", "100"},
+ {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu",
+ "no"}};
+
+for (int r = 0; r < nRows; ++r) {
+ genTestData((const char **)&data[r], nCols, &pArray);
+ tdSTSRowNew(pArray, pTSchema, &row, rowType[r]);
+ debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print
+ tdSRowPrint(row, pTSchema, __func__);
+ checkTSRow((const char **)&data[r], row, pTSchema); // check
+ taosMemoryFreeClear(row);
+ taosArrayClear(pArray);
}
taosArrayDestroy(pArray);
taosMemoryFree(pTSchema);
}
-#endif
#endif
\ No newline at end of file
diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
index 80502e2662..975246a10f 100644
--- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
+++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
@@ -43,6 +43,7 @@ typedef struct SDnodeMgmt {
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
int32_t statusSeq;
+ SendMonitorReportFp sendMonitorReportFpBasic;
} SDnodeMgmt;
// dmHandle.c
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index a3a29f6f77..aa5e87e8b1 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -16,7 +16,6 @@
#define _DEFAULT_SOURCE
#include "dmInt.h"
#include "systable.h"
-#include "tgrant.h"
extern SConfig *tsCfg;
@@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.memTotal = tsTotalMemoryKB * 1024;
req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024;
tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN);
- char *machine = tGetMachineId();
- if (machine) {
- tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1);
- taosMemoryFreeClear(machine);
- }
+ tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1);
req.clusterCfg.statusInterval = tsStatusInterval;
req.clusterCfg.checkTime = 0;
@@ -365,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
-
+#if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
}
-
+#endif
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
index b9dd45f1c0..a651fbf060 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c
@@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->processDropNodeFp = pInput->processDropNodeFp;
pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
+ pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic;
pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
index 7ca19d7725..cecb3d1581 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
@@ -75,6 +75,7 @@ static void *dmNotifyThreadFp(void *param) {
static void *dmMonitorThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
+ int64_t lastTimeForBasic = taosGetTimestampMs();
setThreadName("dnode-monitor");
static int32_t TRIM_FREQ = 20;
@@ -85,6 +86,7 @@ static void *dmMonitorThreadFp(void *param) {
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
int64_t curTime = taosGetTimestampMs();
+
if (curTime < lastTime) lastTime = curTime;
float interval = (curTime - lastTime) / 1000.0f;
if (interval >= tsMonitorInterval) {
@@ -96,6 +98,15 @@ static void *dmMonitorThreadFp(void *param) {
taosMemoryTrim(0);
}
}
+
+ if(tsMonitorForceV2){
+ if (curTime < lastTimeForBasic) lastTimeForBasic = curTime;
+ float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f;
+ if (intervalForBasic >= tsMonitorIntervalForBasic) {
+ (*pMgmt->sendMonitorReportFpBasic)();
+ lastTimeForBasic = curTime;
+ }
+ }
}
return NULL;
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index 531e6f4b3d..a3a5f1d971 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -191,6 +191,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
index 8c726f9f03..1b1dcc9b54 100644
--- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c
+++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c
@@ -87,8 +87,10 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 4213541351..bfac0bab9d 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -835,9 +835,11 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt
index 6b875db860..15c1d2fa4d 100644
--- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt
+++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt
@@ -1,7 +1,7 @@
aux_source_directory(src IMPLEMENT_SRC)
add_library(dnode STATIC ${IMPLEMENT_SRC})
target_link_libraries(
- dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode
+ dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
)
IF (TD_STORAGE)
diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
index bc6a4652e7..90e44e5acc 100644
--- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
+++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
@@ -128,6 +128,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c
void dmSendMonitorReport();
void dmSendAuditRecords();
+void dmSendMonitorReportBasic();
void dmGetVnodeLoads(SMonVloadInfo *pInfo);
void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
void dmGetMnodeLoads(SMonMloadInfo *pInfo);
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index d173c83e97..12712d6585 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -394,6 +394,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
.processDropNodeFp = dmProcessDropNodeReq,
.sendMonitorReportFp = dmSendMonitorReport,
.sendAuditRecordFp = auditSendRecordsInBatch,
+ .sendMonitorReportFpBasic = dmSendMonitorReportBasic,
.getVnodeLoadsFp = dmGetVnodeLoads,
.getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
.getMnodeLoadsFp = dmGetMnodeLoads,
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
index 6cbf31b15f..049b1bdf84 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
@@ -22,6 +22,7 @@
#ifdef TD_TSZ
#include "tcompression.h"
#include "tglobal.h"
+#include "tgrant.h"
#endif
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
@@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) {
pData->rebootTime = taosGetTimestampMs();
pData->dropped = 0;
pData->stopped = 0;
+ char *machineId = tGetMachineId();
+ if (machineId) {
+ tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1);
+ taosMemoryFreeClear(machineId);
+ } else {
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
+ terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
+ return -1;
+#endif
+ }
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) {
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
index c42aa6a1ae..21e25f5535 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c
@@ -25,8 +25,16 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
}
+static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) {
+ pInfo->protocol = 2;
+ pInfo->dnode_id = pDnode->data.dnodeId;
+ pInfo->cluster_id = pDnode->data.clusterId;
+ tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
+}
+
static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
- pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+ //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f);
+ pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f;
pInfo->has_mnode = pDnode->wrappers[MNODE].required;
pInfo->has_qnode = pDnode->wrappers[QNODE].required;
pInfo->has_snode = pDnode->wrappers[SNODE].required;
@@ -44,6 +52,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) {
monSetDmInfo(&dmInfo);
}
+static void dmGetDmMonitorInfoBasic(SDnode *pDnode) {
+ SMonDmInfo dmInfo = {0};
+ dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic);
+ monSetDmInfo(&dmInfo);
+}
+
static void dmGetMmMonitorInfo(SDnode *pDnode) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE];
if (dmMarkWrapper(pWrapper) == 0) {
@@ -106,7 +120,17 @@ void dmSendMonitorReport() {
dmGetVmMonitorInfo(pDnode);
dmGetQmMonitorInfo(pDnode);
dmGetSmMonitorInfo(pDnode);
- monSendReport();
+ monGenAndSendReport();
+}
+
+void dmSendMonitorReportBasic() {
+ if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+ dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
+
+ SDnode *pDnode = dmInstance();
+ dmGetDmMonitorInfoBasic(pDnode);
+ dmGetMmMonitorInfo(pDnode);
+ monGenAndSendReportBasic();
}
//Todo: put this in seperate file in the future
diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt
index 5c670cbdd3..d882d784de 100644
--- a/source/dnode/mgmt/node_util/CMakeLists.txt
+++ b/source/dnode/mgmt/node_util/CMakeLists.txt
@@ -6,5 +6,5 @@ target_include_directories(
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
- node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor
+ node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw
)
\ No newline at end of file
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 4769ef8538..2175824acd 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -109,6 +109,7 @@ typedef struct {
SMsgCb msgCb;
bool validMnodeEps;
int64_t ipWhiteVer;
+ char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeData;
typedef struct {
@@ -122,6 +123,7 @@ typedef struct {
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
+ SendMonitorReportFp sendMonitorReportFpBasic;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt
index 0e77ae9743..a172756aad 100644
--- a/source/dnode/mnode/impl/CMakeLists.txt
+++ b/source/dnode/mnode/impl/CMakeLists.txt
@@ -23,7 +23,7 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
- mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit
+ mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw
)
IF (TD_GRANT)
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index fc58fabad5..4a092057ce 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -464,6 +464,7 @@ typedef struct {
char* pAst1;
char* pAst2;
SRWLatch lock;
+ int8_t source;
} SStbObj;
typedef struct {
diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h
index 2d5479bc9b..3b9b68a1b5 100644
--- a/source/dnode/mnode/impl/inc/mndIndex.h
+++ b/source/dnode/mnode/impl/inc/mndIndex.h
@@ -19,16 +19,16 @@ int32_t mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool
int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq);
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
#ifdef __cplusplus
}
#endif
-#endif /*_TD_MND_IDX_H_*/
\ No newline at end of file
+#endif /*_TD_MND_IDX_H_*/
diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index aed49809dd..57fd187da3 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -86,6 +86,10 @@ typedef struct SOrphanTask {
int32_t nodeId;
} SOrphanTask;
+typedef struct {
+ SMsgHead head;
+} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg;
+
int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 077c0a9c2a..37c2d19bd4 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb);
static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew);
-static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndProcessCreateDbReq(SRpcMsg *pReq);
static int32_t mndProcessAlterDbReq(SRpcMsg *pReq);
@@ -256,17 +256,29 @@ _OVER:
return pRow;
}
-static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
- SDbObj *pNewDb = pObj;
+static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+ SSdb *pSdb = pMnode->pSdb;
+ SSdbRow *pRow = NULL;
+ SDbObj *pNewDb = NULL;
+ int code = -1;
+
+ pRow = mndDbActionDecode(pRaw);
+ if (pRow == NULL) goto _OVER;
+ pNewDb = sdbGetRowObj(pRow);
+ if (pNewDb == NULL) goto _OVER;
SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name);
if (pOldDb != NULL) {
mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name);
sdbRelease(pMnode->pSdb, pOldDb);
- return -1;
+ goto _OVER;
}
- return 0;
+ code = 0;
+_OVER:
+ if (pNewDb) mndDbActionDelete(pSdb, pNewDb);
+ taosMemoryFreeClear(pRow);
+ return code;
}
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) {
@@ -884,10 +896,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
return terrno;
}
-static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
+static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
@@ -943,7 +955,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
mndTransSetDbName(pTrans, pOld->name, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
- if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
+ if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -1120,10 +1132,10 @@ _OVER:
return code;
}
-static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
+static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
@@ -1257,7 +1269,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
goto _OVER;
}
- if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
+ if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
/*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
/*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index cb8a24e675..79a5f5fd83 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -14,6 +14,8 @@
*/
#define _DEFAULT_SOURCE
+#include
+#include "tjson.h"
#include "mndDnode.h"
#include "audit.h"
#include "mndCluster.h"
@@ -28,6 +30,7 @@
#include "mndVgroup.h"
#include "tmisce.h"
#include "tunit.h"
+#include "taos_monitor.h"
#define TSDB_DNODE_VER_NUMBER 2
#define TSDB_DNODE_RESERVE_SIZE 64
@@ -74,6 +77,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp);
static int32_t mndProcessStatusReq(SRpcMsg *pReq);
static int32_t mndProcessNotifyReq(SRpcMsg *pReq);
static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq);
+static int32_t mndProcessStatisReq(SRpcMsg *pReq);
static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
@@ -109,6 +113,7 @@ int32_t mndInitDnode(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq);
mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq);
mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq);
+ mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig);
@@ -141,7 +146,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN);
taosMemoryFreeClear(machineId);
} else {
-#ifdef TD_ENTERPRISE
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
goto _OVER;
#endif
@@ -410,9 +415,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
dInfo.ep.port = pDnode->port;
dInfo.offlineReason = pDnode->offlineReason;
tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
- tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
- tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
- tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1);
sdbRelease(pSdb, pDnode);
if (mndIsMnode(pMnode, pDnode->id)) {
dInfo.isMnode = 1;
@@ -502,6 +504,207 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) {
return stateChanged;
}
+static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
+ SMnode *pMnode = pReq->info.node;
+ SStatisReq statisReq = {0};
+ int32_t code = -1;
+
+ char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
+ sprintf(strClusterId, "%"PRId64, pMnode->clusterId);
+
+ if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
+ terrno = TSDB_CODE_INVALID_MSG;
+ return code;
+ }
+
+ if(tsMonitorLogProtocol){
+ mInfo("process statis req,\n %s", statisReq.pCont);
+ }
+
+ SJson* pJson = tjsonParse(statisReq.pCont);
+
+ int32_t ts_size = tjsonGetArraySize(pJson);
+
+ for(int32_t i = 0; i < ts_size; i++){
+ SJson* item = tjsonGetArrayItem(pJson, i);
+
+ SJson* tables = tjsonGetObjectItem(item, "tables");
+
+ int32_t tableSize = tjsonGetArraySize(tables);
+ for(int32_t i = 0; i < tableSize; i++){
+ SJson* table = tjsonGetArrayItem(tables, i);
+
+ char tableName[MONITOR_TABLENAME_LEN] = {0};
+ tjsonGetStringValue(table, "name", tableName);
+
+ SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+ int32_t size = tjsonGetArraySize(metricGroups);
+ for(int32_t i = 0; i < size; i++){
+ SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+ SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+ int32_t tagSize = tjsonGetArraySize(arrayTag);
+ for(int32_t j = 0; j < tagSize; j++){
+ SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+ char tagName[MONITOR_TAG_NAME_LEN] = {0};
+ tjsonGetStringValue(item, "name", tagName);
+
+ if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+ tjsonDeleteItemFromObject(item, "value");
+ tjsonAddStringToObject(item, "value", strClusterId);
+ }
+ }
+ }
+ }
+ }
+
+ char *pCont = tjsonToString(pJson);
+ monSendContent(pCont);
+
+ if(pJson != NULL){
+ tjsonDelete(pJson);
+ pJson = NULL;
+ }
+
+ if(pCont != NULL){
+ taosMemoryFree(pCont);
+ pCont = NULL;
+ }
+
+ tFreeSStatisReq(&statisReq);
+ return 0;
+
+/*
+ SJson* pJson = tjsonParse(statisReq.pCont);
+
+ int32_t ts_size = tjsonGetArraySize(pJson);
+
+ for(int32_t i = 0; i < ts_size; i++){
+ SJson* item = tjsonGetArrayItem(pJson, i);
+
+ SJson* tables = tjsonGetObjectItem(item, "tables");
+
+ int32_t tableSize = tjsonGetArraySize(tables);
+ for(int32_t i = 0; i < tableSize; i++){
+ SJson* table = tjsonGetArrayItem(tables, i);
+
+ char tableName[MONITOR_TABLENAME_LEN] = {0};
+ tjsonGetStringValue(table, "name", tableName);
+
+ SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
+
+ int32_t size = tjsonGetArraySize(metricGroups);
+ for(int32_t i = 0; i < size; i++){
+ SJson* item = tjsonGetArrayItem(metricGroups, i);
+
+ SJson* arrayTag = tjsonGetObjectItem(item, "tags");
+
+ int32_t tagSize = tjsonGetArraySize(arrayTag);
+
+ char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+ char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
+
+ for(int32_t j = 0; j < tagSize; j++){
+ SJson* item = tjsonGetArrayItem(arrayTag, j);
+
+ *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
+ tjsonGetStringValue(item, "name", *(labels + j));
+
+ *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
+ tjsonGetStringValue(item, "value", *(sample_labels + j));
+ if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
+ strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
+ }
+ }
+
+ SJson* metrics = tjsonGetObjectItem(item, "metrics");
+
+ int32_t metricLen = tjsonGetArraySize(metrics);
+ for(int32_t j = 0; j < metricLen; j++){
+ SJson *item = tjsonGetArrayItem(metrics, j);
+
+ char name[MONITOR_METRIC_NAME_LEN] = {0};
+ tjsonGetStringValue(item, "name", name);
+
+ double value = 0;
+ tjsonGetDoubleValue(item, "value", &value);
+
+ double type = 0;
+ tjsonGetDoubleValue(item, "type", &type);
+
+ int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
+ char* metricName = taosMemoryMalloc(metricNameLen);
+ memset(metricName, 0, metricNameLen);
+ sprintf(metricName, "%s:%s", tableName, name);
+
+ taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
+ if(metric == NULL){
+ if(type == 0){
+ metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
+ }
+ if(type == 1){
+ metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
+ }
+ mTrace("fail to get metric from registry, new one metric:%p", metric);
+
+ if(taos_collector_registry_register_metric(metric) == 1){
+ if(type == 0){
+ taos_counter_destroy(metric);
+ }
+ if(type == 1){
+ taos_gauge_destroy(metric);
+ }
+
+ metric = taos_collector_registry_get_metric(metricName);
+
+ mTrace("fail to register metric, get metric from registry:%p", metric);
+ }
+ else{
+ mTrace("succeed to register metric:%p", metric);
+ }
+ }
+ else{
+ mTrace("get metric from registry:%p", metric);
+ }
+
+ if(type == 0){
+ taos_counter_add(metric, value, (const char**)sample_labels);
+ }
+ if(type == 1){
+ taos_gauge_set(metric, value, (const char**)sample_labels);
+ }
+
+ taosMemoryFreeClear(metricName);
+ }
+
+ for(int32_t j = 0; j < tagSize; j++){
+ taosMemoryFreeClear(*(labels + j));
+ taosMemoryFreeClear(*(sample_labels + j));
+ }
+
+ taosMemoryFreeClear(sample_labels);
+ taosMemoryFreeClear(labels);
+ }
+ }
+
+ }
+
+ code = 0;
+
+_OVER:
+ if(pJson != NULL){
+ tjsonDelete(pJson);
+ pJson = NULL;
+ }
+
+ tFreeSStatisReq(&statisReq);
+ return code;
+ */
+}
+
static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStatusReq statusReq = {0};
diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c
index 041cc664e5..622ed0080f 100644
--- a/source/dnode/mnode/impl/src/mndIndex.c
+++ b/source/dnode/mnode/impl/src/mndIndex.c
@@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
return mndAcquireDb(pMnode, db);
}
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
@@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx)
return 0;
}
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
@@ -482,10 +482,10 @@ _OVER:
return code;
}
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
@@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
mndTransSetSerial(pTrans);
- if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+ if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
@@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
mndTransSetSerial(pTrans);
- if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+ if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index a39e9e93c6..b188d314d9 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -911,11 +911,14 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pObj->id == pMnode->selfDnodeId) {
pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
- pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+ //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
+ pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
// pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));
+ desc.syncState = TAOS_SYNC_STATE_LEADER;
} else {
tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role));
+ desc.syncState = pObj->syncState;
}
taosArrayPush(pClusterInfo->mnodes, &desc);
sdbRelease(pSdb, pObj);
@@ -946,6 +949,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
pVnDesc->dnode_id = pVgid->dnodeId;
tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role));
+ pVnDesc->syncState = pVgid->syncState;
if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++;
@@ -985,8 +989,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
if (pMnode->grant.expireTimeMS == 0) {
- pGrantInfo->expire_time = INT32_MAX;
- pGrantInfo->timeseries_total = INT32_MAX;
+ pGrantInfo->expire_time = 0;
+ pGrantInfo->timeseries_total = 0;
}
mndReleaseRpc(pMnode);
diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c
index a7d8c81d3f..2f4ad69e2a 100644
--- a/source/dnode/mnode/impl/src/mndQnode.c
+++ b/source/dnode/mnode/impl/src/mndQnode.c
@@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
if (pTrans == NULL) goto _OVER;
+ mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
@@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
if (pTrans == NULL) goto _OVER;
+ mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index ef9a7205e1..9aba428ff6 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -300,15 +300,16 @@ static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet*
}
static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
- for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+ int32_t size = (int32_t) taosArrayGetSize(pList);
+ for (int32_t i = 0; i < size; ++i) {
SVgroupVer* pVer = taosArrayGet(pList, i);
if (pVer->vgId == vgId) {
return pVer->ver;
}
}
- mError("failed to find the vgId:%d for extract last version", vgId);
- return -1;
+ mDebug("no data in vgId:%d for extract last version, set to be 0, total existed vgs:%d", vgId, size);
+ return 1;
}
static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) {
@@ -472,6 +473,9 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
int code =
doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
if (code != 0) {
+ mError("create stream task, code:%s", tstrerror(code));
+
+ // todo drop the added source tasks.
sdbRelease(pSdb, pVgroup);
return code;
}
diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c
index 1275ba7962..4243ccb77c 100644
--- a/source/dnode/mnode/impl/src/mndSnode.c
+++ b/source/dnode/mnode/impl/src/mndSnode.c
@@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
if (pTrans == NULL) goto _OVER;
+ mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
@@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
if (pTrans == NULL) goto _OVER;
+ mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 1ddd2f34e6..7ee1b36916 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -458,6 +458,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
req.rollup = pStb->ast1Len > 0 ? 1 : 0;
req.alterOriData = alterOriData;
req.alterOriDataLen = alterOriDataLen;
+ req.source = pStb->source;
// todo
req.schemaRow.nCols = pStb->numOfColumns;
req.schemaRow.version = pStb->colVer;
@@ -616,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
return 0;
}
-static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
@@ -628,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
return 0;
}
-static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
- SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
- if (pUndoRaw == NULL) return -1;
- if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
- sdbFreeRaw(pUndoRaw);
- return -1;
- }
- if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
-
- return 0;
-}
-
static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
if (pCommitRaw == NULL) return -1;
@@ -774,7 +763,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pDst->createdTime = taosGetTimestampMs();
pDst->updateTime = pDst->createdTime;
pDst->uid =
- (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+ (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
+ ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
pDst->dbUid = pDb->uid;
pDst->tagVer = 1;
pDst->colVer = 1;
@@ -790,6 +780,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pDst->numOfFuncs = pCreate->numOfFuncs;
pDst->commentLen = pCreate->commentLen;
pDst->pFuncs = pCreate->pFuncs;
+ pDst->source = pCreate->source;
pCreate->pFuncs = NULL;
if (pDst->commentLen > 0) {
@@ -910,8 +901,6 @@ _OVER:
int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1;
- if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
- if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
@@ -1033,6 +1022,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
memcpy(pDst, pStb, sizeof(SStbObj));
taosRUnLockLatch(&pStb->lock);
+ pDst->source = createReq->source;
pDst->updateTime = taosGetTimestampMs();
pDst->numOfColumns = createReq->numOfColumns;
pDst->numOfTags = createReq->numOfTags;
@@ -1141,7 +1131,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
}
} else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
goto _OVER;
- } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)) {
+ } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) {
mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
code = 0;
goto _OVER;
@@ -1742,10 +1732,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
return 0;
}
-static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
@@ -2151,7 +2141,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb
mndTransSetRpcRsp(pTrans, pCont, contLen);
}
- if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+ if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -2188,11 +2178,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) {
exist = true;
}
- if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+ if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (exist == true) {
- if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+ if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
}
@@ -2211,13 +2201,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
exist = true;
}
- if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+ if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (exist == true) {
memcpy(idxObj.colName, nTagName, strlen(nTagName));
idxObj.colName[strlen(nTagName)] = 0;
- if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+ if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
}
@@ -2350,10 +2340,10 @@ _OVER:
return code;
}
-static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
+static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL) return -1;
- if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+ if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
@@ -2423,7 +2413,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p
mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
- if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
+ if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
@@ -2572,7 +2562,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
}
}
- if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) {
+ if ((dropReq.source == TD_REQ_FROM_TAOX_OLD || dropReq.source == TD_REQ_FROM_TAOX) && pStb->uid != dropReq.suid) {
code = 0;
goto _OVER;
}
@@ -3543,7 +3533,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
- if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+ if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 56adfa9072..3ef2f64df7 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -721,6 +721,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
+ taosThreadMutexLock(&execInfo.lock);
+ mDebug("stream stream:%s tasks register into node list", createReq.name);
+ saveStreamTasksInfo(&streamObj, &execInfo);
+ taosThreadMutexUnlock(&execInfo.lock);
+
// execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -730,12 +735,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
mndTransDrop(pTrans);
- taosThreadMutexLock(&execInfo.lock);
-
- mDebug("stream tasks register into node list");
- saveStreamTasksInfo(&streamObj, &execInfo);
- taosThreadMutexUnlock(&execInfo.lock);
-
SName dbname = {0};
tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -878,7 +877,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
int64_t ts = taosGetTimestampMs();
if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) {
// mWarn("checkpoint interval less than the threshold, ignore it");
- return -1;
+ return TSDB_CODE_SUCCESS;
}
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock);
@@ -2180,5 +2179,16 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream);
taosThreadMutexUnlock(&execInfo.lock);
+ {
+ SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
+ rsp.pCont = rpcMallocCont(rsp.contLen);
+ SMsgHead* pHead = rsp.pCont;
+ pHead->vgId = htonl(req.nodeId);
+
+ tmsgSendRsp(&rsp);
+
+ pReq->info.handle = NULL; // disable auto rsp
+ }
+
return 0;
}
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index 1d296a1c6e..14f3c533e3 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -16,10 +16,6 @@
#include "mndStream.h"
#include "mndTrans.h"
-typedef struct {
- SMsgHead head;
-} SMStreamHbRspMsg;
-
typedef struct SFailedCheckpointInfo {
int64_t streamUid;
int64_t checkpointId;
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 0fc8dad420..a1490a3813 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
SSdbRaw *pRaw = pAction->pRaw;
SSdb *pSdb = pMnode->pSdb;
- SSdbRow *pRow = NULL;
- void *pObj = NULL;
- int code = -1;
+ int code = 0;
if (pRaw->status != SDB_STATUS_CREATING) goto _OUT;
- pRow = (pSdb->decodeFps[pRaw->type])(pRaw);
- if (pRow == NULL) goto _OUT;
- pObj = sdbGetRowObj(pRow);
- if (pObj == NULL) goto _OUT;
-
SdbValidateFp validateFp = pSdb->validateFps[pRaw->type];
- code = 0;
if (validateFp) {
- code = validateFp(pMnode, pTrans, pObj);
+ code = validateFp(pMnode, pTrans, pRaw);
}
_OUT:
- if (pRow) {
- SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
- if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
- taosMemoryFreeClear(pRow);
- }
return code;
}
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 1adc4ed4bf..04d74112d4 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
pTrans->createdTime);
int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id);
if (code != 0) {
- mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
- tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId);
+ mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
+ code, pTrans->createdTime, pMnode->syncMgmt.transId);
sdbFreeRaw(pRaw);
return -1;
}
@@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) {
if (pAction->rawWritten) return 0;
- if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+ if (topHalf) {
+ terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH;
+ return TSDB_CODE_MND_TRANS_CTX_SWITCH;
+ }
int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw);
if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@@ -1214,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
if (numOfActions == 0) return 0;
if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) {
- return -1;
+ return code;
}
int32_t numOfExecuted = 0;
@@ -1419,7 +1422,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool
pTrans->stage = TRN_STAGE_COMMIT;
mInfo("trans:%d, stage from redoAction to commit", pTrans->id);
continueExec = true;
- } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+ } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) {
mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code));
continueExec = false;
} else {
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index a5df9ad820..b0290191bc 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -31,10 +31,10 @@
#define VGROUP_VER_NUMBER 1
#define VGROUP_RESERVE_SIZE 64
-static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
+static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@@ -181,15 +181,28 @@ _OVER:
return pRow;
}
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
- SVgObj *pVgroup = pObj;
+static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+ SSdb *pSdb = pMnode->pSdb;
+ SSdbRow *pRow = NULL;
+ SVgObj *pVgroup = NULL;
+ int code = -1;
+
+ pRow = mndVgroupActionDecode(pRaw);
+ if (pRow == NULL) goto _OVER;
+ pVgroup = sdbGetRowObj(pRow);
+ if (pVgroup == NULL) goto _OVER;
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
if (maxVgId > pVgroup->vgId) {
mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId);
- return -1;
+ goto _OVER;
}
- return 0;
+
+ code = 0;
+_OVER:
+ if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup);
+ taosMemoryFreeClear(pRow);
+ return code;
}
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) {
diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h
index 9d1125dad2..e2d4a1f6fc 100644
--- a/source/dnode/mnode/sdb/inc/sdb.h
+++ b/source/dnode/mnode/sdb/inc/sdb.h
@@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
-typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj);
+typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw);
typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3);
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index 34623c021b..3bef5b595b 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -180,6 +180,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
return tqStreamTaskProcessCheckpointReadyMsg(pSnode->pMeta, pMsg);
case TDMT_MND_STREAM_HEARTBEAT_RSP:
return tqStreamProcessStreamHbRsp(pSnode->pMeta, pMsg);
+ case TDMT_MND_STREAM_REQ_CHKPT_RSP:
+ return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg);
+ case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+ return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg);
default:
sndError("invalid snode msg:%d", pMsg->msgType);
ASSERT(0);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index b83af84086..e610161544 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -47,6 +47,8 @@
#include "vnode.h"
+#include "taos_monitor.h"
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -104,6 +106,20 @@ typedef struct SQueryNode SQueryNode;
#define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json"
+#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count"
+
+#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
+#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
+#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
+#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
+#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
+#define VNODE_METRIC_TAG_NAME_USERNAME "username"
+#define VNODE_METRIC_TAG_NAME_RESULT "result"
+
+#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
+//#define VNODE_METRIC_TAG_VALUE_INSERT "insert"
+//#define VNODE_METRIC_TAG_VALUE_DELETE "delete"
+
// vnd.h
typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
struct SQueryNode {
@@ -243,6 +259,8 @@ int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq);
@@ -443,6 +461,13 @@ typedef struct SVCommitSched {
int64_t maxWaitMs;
} SVCommitSched;
+typedef struct SVMonitorObj{
+ char strClusterId[TSDB_CLUSTER_ID_LEN];
+ char strDnodeId[TSDB_NODE_ID_LEN];
+ char strVgId[TSDB_VGROUP_ID_LEN];
+ taos_counter_t *insertCounter;
+}SVMonitorObj;
+
struct SVnode {
char* path;
SVnodeCfg config;
@@ -481,6 +506,7 @@ struct SVnode {
int32_t blockSec;
int64_t blockSeq;
SQHandle* pQuery;
+ SVMonitorObj monitor;
};
#define TD_VID(PVNODE) ((PVNODE)->config.vgId)
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index a2c7a04ab3..4a978c8f41 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -1008,11 +1008,11 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
}
tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t));
}
-
+ /*
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL);
}
-
+ */
metaDebug("batch drop table:%" PRId64, uid);
}
metaULock(pMeta);
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
index 54d859992c..fb898c02f8 100644
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
continue;
}
- SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL};
+ SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index a3fd19c7f8..011e62cb89 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1175,7 +1175,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS;
}
} else {
- ASSERT(status == TASK_STATUS__HALT);
+// ASSERT(status == TASK_STATUS__HALT);
+ if (status != TASK_STATUS__HALT) {
+ tqError("s-task:%s should in halt status, let's halt it directly", pTask->id.idStr);
+// streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT);
+ }
}
// check if the checkpoint msg already sent or not.
@@ -1225,3 +1229,11 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg);
}
+
+int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) {
+ return tqStreamProcessReqCheckpointRsp(pTq->pStreamMeta, pMsg);
+}
+
+int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) {
+ return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg);
+}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 727157a2f8..7a737fbb5d 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -392,7 +392,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
pReader->msg.ver);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
- if ((pSubmitTbData->source & sourceExcluded) != 0) {
+ if ((pSubmitTbData->flags & sourceExcluded) != 0) {
pReader->nextBlk += 1;
continue;
}
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index f108cedcf5..103007eb57 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -267,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table;
}
- if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+ if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
goto loop_table;
}
if (pRsp->withTbName) {
@@ -335,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db;
}
- if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+ if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
goto loop_db;
}
if (pRsp->withTbName) {
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index b56bf3e0fe..9ff4e3ba62 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -72,7 +72,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
memcpy(name, varDataVal(varTbName), varDataLen(varTbName));
if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0) {
- buildCtbNameAddGruopId(name, groupId);
+ buildCtbNameAddGroupId(name, groupId);
}
} else if (stbFullName) {
name = buildCtbNameByGroupId(stbFullName, groupId);
@@ -185,23 +185,26 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
!alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0) {
pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
strcpy(pCreateTableReq->name, pDataBlock->info.parTbName);
- buildCtbNameAddGruopId(pCreateTableReq->name, gid);
+ buildCtbNameAddGroupId(pCreateTableReq->name, gid);
+// tqDebug("gen name from:%s", pDataBlock->info.parTbName);
} else {
pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
+// tqDebug("copy name:%s", pDataBlock->info.parTbName);
}
} else {
pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
+// tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid);
}
}
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
SStreamTask* pTask, int64_t suid) {
- tqDebug("s-task:%s build create table msg", pTask->id.idStr);
-
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
- ;
+
+ tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows);
+
int32_t code = 0;
SVCreateTbBatchReq reqs = {0};
@@ -670,7 +673,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} else {
if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(dstTableName) && !alreadyAddGroupId(dstTableName) && groupId != 0) {
- buildCtbNameAddGruopId(dstTableName, groupId);
+ tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName);
+ buildCtbNameAddGroupId(dstTableName, groupId);
}
}
@@ -815,7 +819,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
return;
}
- SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+ SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
if (code != TSDB_CODE_SUCCESS) {
continue;
@@ -860,7 +864,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
pTask->execInfo.sink.numOfBlocks += 1;
uint64_t groupId = pDataBlock->info.id.groupId;
- SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+ SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
if (index == NULL) { // no data yet, append it
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 31267dbf52..dad1211326 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -171,6 +171,22 @@ end : {
}
}
+#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \
+ SDecoder decoder = {0};\
+ TYPE req = {0}; \
+ void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
+ int32_t len = pHead->bodyLen - sizeof(SMsgHead); \
+ tDecoderInit(&decoder, data, len); \
+ if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \
+ tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \
+ pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
+ fetchVer++; \
+ tDecoderClear(&decoder); \
+ continue; \
+ } \
+ tDecoderClear(&decoder);
+
+
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int code = 0;
@@ -239,6 +255,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
goto end;
}
+ if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
+ if (pHead->msgType == TDMT_VND_CREATE_TABLE) {
+ PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq)
+ } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) {
+ PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq)
+ } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
+ PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
+ } else if (pHead->msgType == TDMT_VND_DELETE) {
+ fetchVer++;
+ continue;
+ }
+ }
+
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
metaRsp.resMsgType = pHead->msgType;
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 4e49091da4..a2d45062b9 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -485,6 +485,10 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
return code;
}
+typedef struct SMStreamCheckpointReadyRspMsg {
+ SMsgHead head;
+}SMStreamCheckpointReadyRspMsg;
+
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {
int32_t vgId = pMeta->vgId;
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@@ -513,6 +517,18 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
streamProcessCheckpointReadyMsg(pTask);
streamMetaReleaseTask(pMeta, pTask);
+
+ { // send checkpoint ready rsp
+ SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)};
+ rsp.pCont = rpcMallocCont(rsp.contLen);
+ SMsgHead* pHead = rsp.pCont;
+ pHead->vgId = htonl(req.downstreamNodeId);
+
+ tmsgSendRsp(&rsp);
+
+ pMsg->info.handle = NULL; // disable auto rsp
+ }
+
return code;
}
@@ -938,9 +954,17 @@ int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta) {
return taosArrayGetSize(pMeta->pTaskList);
}
-int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return TSDB_CODE_SUCCESS;
}
+
+int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+
+int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
+
+int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
+ return doProcessDummyRspMsg(pMeta, pMsg);
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 6aff1c2930..81a136a0e3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -448,8 +448,8 @@ _exit:
taosMemoryFree(reader[0]);
reader[0] = NULL;
} else {
- tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
- sver, ever, type);
+ tsdbInfo("vgId:%d, tsdb snapshot incremental reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d",
+ TD_VID(tsdb->pVnode), sver, ever, type);
}
return code;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
index b7c22aa0e9..ae840eb6d2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
@@ -69,7 +69,8 @@ _exit:
taosMemoryFree(reader[0]);
reader[0] = NULL;
} else {
- tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type);
+ tsdbInfo("vgId:%d, tsdb snapshot raw reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever,
+ type);
}
return code;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 946ce9d278..e1f76b3a25 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -480,6 +480,28 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
vnodeRollback(pVnode);
}
+ snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId);
+ snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
+ snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId);
+
+ if(pVnode->monitor.insertCounter == NULL){
+ int32_t label_count = 7;
+ const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
+ VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
+ VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
+ VNODE_METRIC_TAG_NAME_RESULT};
+ taos_counter_t *counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql",
+ label_count, sample_labels);
+ vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter);
+ if(taos_collector_registry_register_metric(counter) == 1){
+ taos_counter_destroy(counter);
+ counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
+ vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter);
+ }
+ pVnode->monitor.insertCounter = counter;
+ vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter);
+ }
+
return pVnode;
_err:
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 4fc7a88494..d1c811858a 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -13,8 +13,8 @@
* along with this program. If not, see .
*/
-#include "vnd.h"
#include "tsdb.h"
+#include "vnd.h"
#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \
do { \
@@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
- goto _exit;
+ goto _exit4;
}
metaRsp.dbId = pVnode->config.dbId;
@@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName);
if (code) {
- goto _exit;
+ goto _exit4;
}
// query meta
@@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) {
code = terrno;
- goto _exit;
+ goto _exit3;
}
metaRsp.tableType = mer1.me.type;
@@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.suid = mer1.me.uid;
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
- if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+ if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid;
@@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
_exit:
+ taosMemoryFree(metaRsp.pSchemas);
+_exit2:
+ metaReaderClear(&mer2);
+_exit3:
+ metaReaderClear(&mer1);
+_exit4:
rpcMsg.info = pMsg->info;
rpcMsg.pCont = pRsp;
rpcMsg.contLen = rspLen;
@@ -141,9 +147,6 @@ _exit:
*pMsg = rpcMsg;
}
- taosMemoryFree(metaRsp.pSchemas);
- metaReaderClear(&mer2);
- metaReaderClear(&mer1);
return TSDB_CODE_SUCCESS;
}
@@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) {
}
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
- return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid);
+ return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index ed1dcc64c9..b4c8a26f5c 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -172,6 +172,23 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
goto _err;
}
+ // open tsdb snapshot raw reader
+ if (!pReader->tsdbRAWDone) {
+ ASSERT(pReader->sver == 0);
+ code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
+ if (code) goto _err;
+ }
+
+ // check snapshot ever
+ SSnapshot snapshot = {0};
+ vnodeGetSnapshot(pVnode, &snapshot);
+ if (ever != snapshot.lastApplyIndex) {
+ vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
+ TD_VID(pVnode), ever, snapshot.lastApplyIndex);
+ code = TSDB_CODE_SYN_INTERNAL_ERROR;
+ goto _err;
+ }
+
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = pReader;
return code;
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 86355cdbb0..52d75f9ab5 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -30,13 +30,15 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
SRpcMsg *pOriginRpc);
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
+ SRpcMsg *pOriginalMsg);
static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
+ SRpcMsg *pOriginalMsg);
static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@@ -542,10 +544,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
break;
/* TSDB */
case TDMT_VND_SUBMIT:
- if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
+ if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp, pMsg) < 0) goto _err;
break;
case TDMT_VND_DELETE:
- if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) goto _err;
break;
case TDMT_VND_BATCH_DEL:
if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
@@ -798,6 +800,10 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg);
case TDMT_MND_STREAM_HEARTBEAT_RSP:
return tqProcessStreamHbRsp(pVnode->pTq, pMsg);
+ case TDMT_MND_STREAM_REQ_CHKPT_RSP:
+ return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
+ case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
+ return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@@ -1482,7 +1488,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
return code;
}
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
+ SRpcMsg *pOriginalMsg) {
int32_t code = 0;
terrno = 0;
@@ -1700,10 +1707,35 @@ _exit:
atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows);
atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows);
atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
+
+ if(pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){
+ const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId,
+ pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
+ pOriginalMsg->info.conn.user, "Success"};
+ taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels);
+ }
+
if (code == 0) {
atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len);
}
+ /*
+ if (code == 0) {
+ atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
+ code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len);
+
+ const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId,
+ pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
+ pOriginalMsg->info.conn.user, "Success"};
+ taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+ }
+ else{
+ const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId,
+ pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
+ pOriginalMsg->info.conn.user, "Failed"};
+ taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+ }
+ */
// clear
taosArrayDestroy(newTbUids);
@@ -1959,7 +1991,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe
return 0;
}
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp,
+ SRpcMsg *pOriginalMsg) {
int32_t code = 0;
SDecoder *pCoder = &(SDecoder){0};
SDeleteRes *pRes = &(SDeleteRes){0};
@@ -2002,6 +2035,21 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in
return code;
_err:
+ /*
+ if(code == TSDB_CODE_SUCCESS){
+ const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId,
+ pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
+ pOriginalMsg->info.conn.user, "Success"};
+ taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+ }
+ else{
+ const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId,
+ pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId,
+ pOriginalMsg->info.conn.user, "Failed"};
+ taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels);
+ }
+ */
+
return code;
}
static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt
index 9f812517c1..6196d30a58 100644
--- a/source/libs/CMakeLists.txt
+++ b/source/libs/CMakeLists.txt
@@ -8,6 +8,7 @@ add_subdirectory(qcom)
add_subdirectory(nodes)
add_subdirectory(catalog)
add_subdirectory(audit)
+add_subdirectory(monitorfw)
add_subdirectory(scalar)
add_subdirectory(function)
diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c
index 3980e5ae4d..1d2a55fac8 100644
--- a/source/libs/executor/src/countwindowoperator.c
+++ b/source/libs/executor/src/countwindowoperator.c
@@ -129,7 +129,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
return code;
}
-static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) {
+static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) {
SResultRow* pResultRow = NULL;
for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) {
SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow);
@@ -143,6 +143,7 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S
clearWinStateBuff(pBuff);
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
}
+ doFilter(pBlock, pFilterInfo, NULL);
}
static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
@@ -177,7 +178,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
if (pInfo->groupId == 0) {
pInfo->groupId = pBlock->info.id.groupId;
} else if (pInfo->groupId != pBlock->info.id.groupId) {
- buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
+ buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes);
pInfo->groupId = pBlock->info.id.groupId;
}
@@ -187,7 +188,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) {
}
}
- buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes);
+ buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes);
return pRes->info.rows == 0 ? NULL : pRes;
}
@@ -246,6 +247,11 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
pInfo->countSup.stateIndex = 0;
+ code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo,
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index cc83ecd84e..691210e5ba 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -461,7 +461,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
_error:
destroyIndefinitOperatorInfo(pInfo);
taosMemoryFree(pOperator);
- pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+ pTaskInfo->code = code;
return NULL;
}
diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index 689ba54ca6..f9c7b51316 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
}
+ if (ts < pCurWin->winInfo.sessionWin.win.ekey) {
+ pBuffInfo->rebuildWindow = true;
+ }
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
@@ -284,6 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+ curWin.winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
@@ -341,7 +345,7 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper
size_t keyLen = 0;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {
- void* key = taosHashGetKey(pIte, &keyLen);
+ void* key = tSimpleHashGetKey(pIte, &keyLen);
tlen += encodeSSessionKey(buf, key);
tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);
}
@@ -434,7 +438,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key);
}
while (1) {
- SSessionKey tmpKey = {0};
+ SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index fde5f22aaa..9ff903cdb9 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
int64_t firstRowTs = *(int64_t*)tsCol->pData;
if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) ||
(pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
- continue;
- }
+ if (bExtractedBlock) {
+ blockDataDestroy(pBlk);
+ }
+ continue;
+ }
}
if (pBlk != NULL) {
@@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
tSimpleHashClear(mUidBlk);
code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
if (code != TSDB_CODE_SUCCESS) {
- tSimpleHashCleanup(mUidBlk);
- taosArrayDestroy(aBlkSort);
- taosArrayDestroy(aExtSrc);
- return code;
+ for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
+ blockDataDestroy(taosArrayGetP(aBlkSort, i));
+ }
+ taosArrayClear(aBlkSort);
+ break;
}
int64_t el = taosGetTimestampUs() - p;
@@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
szSort = 0;
qDebug("source %zu created", taosArrayGetSize(aExtSrc));
}
+
if (pBlk == NULL) {
break;
}
@@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
}
tSimpleHashCleanup(mUidBlk);
+ for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
+ blockDataDestroy(taosArrayGetP(aBlkSort, i));
+ }
taosArrayDestroy(aBlkSort);
tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
if (!tsortIsClosed(pHandle)) {
@@ -1188,7 +1196,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
taosArrayDestroy(aExtSrc);
tSimpleHashCleanup(mTableNumRows);
pHandle->type = SORT_SINGLESOURCE_SORT;
- return TSDB_CODE_SUCCESS;
+ return code;
}
static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt
index 30dce7aaef..cc8f40fa4c 100644
--- a/source/libs/monitor/CMakeLists.txt
+++ b/source/libs/monitor/CMakeLists.txt
@@ -6,7 +6,7 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
-target_link_libraries(monitor os util common transport)
+target_link_libraries(monitor os util common qcom transport monitorfw)
if(${BUILD_TEST})
add_subdirectory(test)
diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h
index ae1af4ba62..7fc718393b 100644
--- a/source/libs/monitor/inc/monInt.h
+++ b/source/libs/monitor/inc/monInt.h
@@ -17,8 +17,10 @@
#define _TD_MONITOR_INT_H_
#include "monitor.h"
+#include "query.h"
#include "tjson.h"
+#include "thash.h"
typedef struct {
int64_t curTime;
@@ -44,8 +46,22 @@ typedef struct {
SMonSmInfo smInfo;
SMonQmInfo qmInfo;
SMonBmInfo bmInfo;
+ SHashObj *metrics;
} SMonitor;
+void monGenClusterInfoTable(SMonInfo *pMonitor);
+void monGenVgroupInfoTable(SMonInfo *pMonitor);
+void monGenDnodeInfoTable(SMonInfo *pMonitor);
+void monGenDnodeStatusInfoTable(SMonInfo *pMonitor);
+void monGenDataDiskTable(SMonInfo *pMonitor);
+void monGenLogDiskTable(SMonInfo *pMonitor);
+void monGenMnodeRoleTable(SMonInfo *pMonitor);
+void monGenVnodeRoleTable(SMonInfo *pMonitor);
+
+void monSendPromReport();
+void monInitMonitorFW();
+void monCleanupMonitorFW();
+
#ifdef __cplusplus
}
#endif
diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c
new file mode 100644
index 0000000000..0891932583
--- /dev/null
+++ b/source/libs/monitor/src/clientMonitor.c
@@ -0,0 +1,210 @@
+#include "clientMonitor.h"
+#include "os.h"
+#include "tmisce.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+
+SRWLatch monitorLock;
+void* tmrClientMonitor;
+tmr_h tmrStartHandle;
+SHashObj* clusterMonitorInfoTable;
+
+static int interval = 30 * 1000;
+static int sendBathchSize = 1;
+
+int32_t sendReport(ClientMonitor* pMonitor, char* pCont);
+void generateClusterReport(ClientMonitor* pMonitor, bool send) {
+ char ts[50];
+ sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
+ char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL);
+ if(NULL == pCont) {
+ uError("generateClusterReport failed, get null content.");
+ return;
+ }
+ if (send && strlen(pCont) != 0) {
+ if (sendReport(pMonitor, pCont) == 0) {
+ taos_collector_registry_clear_batch(pMonitor->registry);
+ }
+ }
+ taosMemoryFreeClear(pCont);
+}
+
+void reportSendProcess(void* param, void* tmrId) {
+ taosTmrReset(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor, &tmrStartHandle);
+ taosRLockLatch(&monitorLock);
+
+ static int index = 0;
+ index++;
+ ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL);
+ while (ppMonitor != NULL && *ppMonitor != NULL) {
+ ClientMonitor* pMonitor = *ppMonitor;
+ generateClusterReport(*ppMonitor, index == sendBathchSize);
+ ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor);
+ }
+
+ if (index == sendBathchSize) index = 0;
+ taosRUnLockLatch(&monitorLock);
+}
+
+void monitorClientInitOnce() {
+ static int8_t init = 0;
+ if (atomic_exchange_8(&init, 1) == 0) {
+ uInfo("tscMonitorInit once.");
+ clusterMonitorInfoTable =
+ (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+
+ tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR");
+ tmrStartHandle = taosTmrStart(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor);
+ if(tsMonitorInterval < 1){
+ interval = 30 * 1000;
+ } else {
+ interval = tsMonitorInterval * 1000;
+ }
+ if (tsMonitorInterval < 10) {
+ sendBathchSize = (10 / sendBathchSize) + 1;
+ }
+ taosInitRWLatch(&monitorLock);
+ }
+}
+
+void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) {
+ if (clusterKey == NULL || strlen(clusterKey) == 0) {
+ uError("createMonitorClient failed, clusterKey is NULL");
+ return;
+ }
+ taosWLockLatch(&monitorLock);
+ if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) {
+ uInfo("createMonitorClient for %s.", clusterKey);
+ ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor));
+ snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey);
+ pMonitor->registry = taos_collector_registry_new(clusterKey);
+ pMonitor->colector = taos_collector_new(clusterKey);
+ epsetAssign(&pMonitor->epSet, &epSet);
+ pMonitor->pTransporter = pTransporter;
+
+ taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector);
+ pMonitor->counters =
+ (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+
+ taosHashPut(clusterMonitorInfoTable, clusterKey, strlen(clusterKey), &pMonitor, sizeof(ClientMonitor*));
+ uInfo("createMonitorClient for %s finished %p.", clusterKey, pMonitor);
+ }
+ taosWUnLockLatch(&monitorLock);
+}
+
+static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) {
+ static int32_t emptyRspNum = 0;
+ if (TSDB_CODE_SUCCESS != code) {
+ uError("found error in monitorReport send callback, code:%d, please check the network.", code);
+ }
+ if (pMsg) {
+ taosMemoryFree(pMsg->pData);
+ taosMemoryFree(pMsg->pEpSet);
+ }
+ return code;
+}
+
+int32_t sendReport(ClientMonitor* pMonitor, char* pCont) {
+ SStatisReq sStatisReq;
+ sStatisReq.pCont = pCont;
+ sStatisReq.contLen = strlen(pCont);
+
+ int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq);
+ if (tlen < 0) return 0;
+ void* buf = taosMemoryMalloc(tlen);
+ if (buf == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ tSerializeSStatisReq(buf, tlen, &sStatisReq);
+
+ SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
+ if (pInfo == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ pInfo->fp = monitorReportAsyncCB;
+ pInfo->msgInfo.pData = buf;
+ pInfo->msgInfo.len = tlen;
+ pInfo->msgType = TDMT_MND_STATIS;
+ // pInfo->param = taosMemoryMalloc(sizeof(int32_t));
+ // *(int32_t*)pInfo->param = i;
+ pInfo->paramFreeFp = taosMemoryFree;
+ pInfo->requestId = tGenIdPI64();
+ pInfo->requestObjRefId = 0;
+
+ int64_t transporterId = 0;
+ return asyncSendMsgToServer(pMonitor->pTransporter, &pMonitor->epSet, &transporterId, pInfo);
+}
+
+void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter) {
+ monitorClientInitOnce();
+ createMonitorClient(clusterKey, epSet, pTransporter);
+}
+
+taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count,
+ const char** label_keys) {
+ ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
+
+ if (ppMonitor != NULL && *ppMonitor != NULL) {
+ ClientMonitor* pMonitor = *ppMonitor;
+ taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, name, strlen(name));
+ if (ppCounter != NULL && *ppCounter != NULL) {
+ taosHashRemove(pMonitor->counters, name, strlen(name));
+ uInfo("createClusterCounter remove old counter: %s.", name);
+ }
+
+ taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys);
+ if (newCounter != NULL) {
+ taos_collector_add_metric(pMonitor->colector, newCounter);
+ taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, sizeof(taos_counter_t*));
+ uInfo("createClusterCounter %s(%p):%s : %p.", pMonitor->clusterKey, pMonitor, name, newCounter);
+ return newCounter;
+ } else {
+ return NULL;
+ }
+ } else {
+ return NULL;
+ }
+ return NULL;
+}
+
+int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values) {
+ taosRLockLatch(&monitorLock);
+ ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
+
+ if (ppMonitor != NULL && *ppMonitor != NULL) {
+ ClientMonitor* pMonitor = *ppMonitor;
+ taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName));
+ if (ppCounter != NULL && *ppCounter != NULL) {
+ taos_counter_inc(*ppCounter, label_values);
+ } else {
+ uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName);
+ }
+ } else {
+ uError("taosClusterCounterInc not found pMonitor %s.", clusterKey);
+ }
+ taosRUnLockLatch(&monitorLock);
+ return 0;
+}
+
+void clusterMonitorClose(const char* clusterKey) {
+ taosWLockLatch(&monitorLock);
+ ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
+
+ if (ppMonitor != NULL && *ppMonitor != NULL) {
+ ClientMonitor* pMonitor = *ppMonitor;
+ uInfo("clusterMonitorClose valule:%p clusterKey:%s.", pMonitor, pMonitor->clusterKey);
+ taosHashCleanup(pMonitor->counters);
+ taos_collector_registry_destroy(pMonitor->registry);
+ taosMemoryFree(pMonitor);
+ taosHashRemove(clusterMonitorInfoTable, clusterKey, strlen(clusterKey));
+ }
+ taosWUnLockLatch(&monitorLock);
+}
+
+const char* resultStr(SQL_RESULT_CODE code) {
+ static const char* result_state[] = {"Success", "Failed", "Cancel"};
+ return result_state[code];
+}
diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c
new file mode 100644
index 0000000000..488b4d1d58
--- /dev/null
+++ b/source/libs/monitor/src/monFramework.c
@@ -0,0 +1,763 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+#include "monitor.h"
+#include "monInt.h"
+
+#include "thash.h"
+#include "taos_monitor.h"
+#include "thttp.h"
+#include "ttime.h"
+#include "tglobal.h"
+
+extern SMonitor tsMonitor;
+extern char* tsMonUri;
+extern char* tsMonFwUri;
+
+#define LEVEL_LEN 11
+
+#define CLUSTER_TABLE "taosd_cluster_info"
+
+#define MASTER_UPTIME CLUSTER_TABLE":cluster_uptime"
+#define DBS_TOTAL CLUSTER_TABLE":dbs_total"
+#define TBS_TOTAL CLUSTER_TABLE":tbs_total"
+#define STBS_TOTAL CLUSTER_TABLE":stbs_total"
+#define VGROUPS_TOTAL CLUSTER_TABLE":vgroups_total"
+#define VGROUPS_ALIVE CLUSTER_TABLE":vgroups_alive"
+#define VNODES_TOTAL CLUSTER_TABLE":vnodes_total"
+#define VNODES_ALIVE CLUSTER_TABLE":vnodes_alive"
+#define DNODES_TOTAL CLUSTER_TABLE":dnodes_total"
+#define DNODES_ALIVE CLUSTER_TABLE":dnodes_alive"
+#define MNODES_TOTAL CLUSTER_TABLE":mnodes_total"
+#define MNODES_ALIVE CLUSTER_TABLE":mnodes_alive"
+#define CONNECTIONS_TOTAL CLUSTER_TABLE":connections_total"
+#define TOPICS_TOTAL CLUSTER_TABLE":topics_total"
+#define STREAMS_TOTAL CLUSTER_TABLE":streams_total"
+#define EXPIRE_TIME CLUSTER_TABLE":grants_expire_time"
+#define TIMESERIES_USED CLUSTER_TABLE":grants_timeseries_used"
+#define TIMESERIES_TOTAL CLUSTER_TABLE":grants_timeseries_total"
+
+#define VGROUP_TABLE "taosd_vgroups_info"
+
+#define TABLES_NUM VGROUP_TABLE":tables_num"
+#define STATUS VGROUP_TABLE":status"
+
+#define DNODE_TABLE "taosd_dnodes_info"
+
+#define UPTIME DNODE_TABLE":uptime"
+#define CPU_ENGINE DNODE_TABLE":cpu_engine"
+#define CPU_SYSTEM DNODE_TABLE":cpu_system"
+#define CPU_CORE DNODE_TABLE":cpu_cores"
+#define MEM_ENGINE DNODE_TABLE":mem_engine"
+#define MEM_SYSTEM DNODE_TABLE":mem_free"
+#define MEM_TOTAL DNODE_TABLE":mem_total"
+#define DISK_ENGINE DNODE_TABLE":disk_engine"
+#define DISK_USED DNODE_TABLE":disk_used"
+#define DISK_TOTAL DNODE_TABLE":disk_total"
+#define NET_IN DNODE_TABLE":system_net_in"
+#define NET_OUT DNODE_TABLE":system_net_out"
+#define IO_READ DNODE_TABLE":io_read"
+#define IO_WRITE DNODE_TABLE":io_write"
+#define IO_READ_DISK DNODE_TABLE":io_read_disk"
+#define IO_WRITE_DISK DNODE_TABLE":io_write_disk"
+//#define ERRORS DNODE_TABLE":errors"
+#define VNODES_NUM DNODE_TABLE":vnodes_num"
+#define MASTERS DNODE_TABLE":masters"
+#define HAS_MNODE DNODE_TABLE":has_mnode"
+#define HAS_QNODE DNODE_TABLE":has_qnode"
+#define HAS_SNODE DNODE_TABLE":has_snode"
+#define DNODE_LOG_ERROR DNODE_TABLE":error_log_count"
+#define DNODE_LOG_INFO DNODE_TABLE":info_log_count"
+#define DNODE_LOG_DEBUG DNODE_TABLE":debug_log_count"
+#define DNODE_LOG_TRACE DNODE_TABLE":trace_log_count"
+
+#define DNODE_STATUS "taosd_dnodes_status:status"
+
+#define DATADIR_TABLE "taosd_dnodes_data_dirs"
+
+#define DNODE_DATA_AVAIL DATADIR_TABLE":avail"
+#define DNODE_DATA_USED DATADIR_TABLE":used"
+#define DNODE_DATA_TOTAL DATADIR_TABLE":total"
+
+#define LOGDIR_TABLE "taosd_dnodes_log_dirs"
+
+#define DNODE_LOG_AVAIL LOGDIR_TABLE":avail"
+#define DNODE_LOG_USED LOGDIR_TABLE":used"
+#define DNODE_LOG_TOTAL LOGDIR_TABLE":total"
+
+#define MNODE_ROLE "taosd_mnodes_info:role"
+#define VNODE_ROLE "taosd_vnodes_info:role"
+
+void monInitMonitorFW(){
+ taos_collector_registry_default_init();
+
+ tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+ taos_gauge_t *gauge = NULL;
+
+ int32_t dnodes_label_count = 3;
+ const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"};
+ char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM,
+ MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN,
+ NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, /*ERRORS,*/
+ VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE,
+ DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE};
+ for(int32_t i = 0; i < 25; i++){
+ gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, dnodes_gauges[i], strlen(dnodes_gauges[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+
+ int32_t dnodes_data_label_count = 5;
+ const char *dnodes_data_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name", "data_dir_level"};
+ char *dnodes_data_gauges[] = {DNODE_DATA_AVAIL, DNODE_DATA_USED, DNODE_DATA_TOTAL};
+ for(int32_t i = 0; i < 3; i++){
+ gauge= taos_gauge_new(dnodes_data_gauges[i], "", dnodes_data_label_count, dnodes_data_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, dnodes_data_gauges[i], strlen(dnodes_data_gauges[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+
+ int32_t dnodes_log_label_count = 4;
+ const char *dnodes_log_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name"};
+ char *dnodes_log_gauges[] = {DNODE_LOG_AVAIL, DNODE_LOG_USED, DNODE_LOG_TOTAL};
+ for(int32_t i = 0; i < 3; i++){
+ gauge= taos_gauge_new(dnodes_log_gauges[i], "", dnodes_log_label_count, dnodes_log_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+}
+
+void monCleanupMonitorFW(){
+ taosHashCleanup(tsMonitor.metrics);
+ taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT);
+ TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL;
+}
+
+void monGenClusterInfoTable(SMonInfo *pMonitor){
+ SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster;
+ SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic;
+ SMonGrantInfo *pGrantInfo = &pMonitor->mmInfo.grant;
+
+ char *metric_names[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL,
+ VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE,
+ CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL,
+ DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED,
+ TIMESERIES_TOTAL};
+
+ for(int32_t i = 0; i < 18; i++){
+ if(taos_collector_registry_deregister_metric(metric_names[i]) != 0){
+ uError("failed to delete metric %s", metric_names[i]);
+ }
+
+ taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]));
+ }
+
+ if(pBasicInfo->cluster_id == 0) {
+ uError("failed to generate dnode info table since cluster_id is 0");
+ return;
+ }
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ //cluster info
+ taos_gauge_t *gauge = NULL;
+ int32_t label_count =1;
+ const char *sample_labels1[] = {"cluster_id"};
+
+ for(int32_t i = 0; i < 18; i++){
+ gauge= taos_gauge_new(metric_names[i], "", label_count, sample_labels1);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+
+ char buf[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(buf, TSDB_CLUSTER_ID_LEN, "%"PRId64, pBasicInfo->cluster_id);
+ const char *sample_label_values[] = {buf};
+
+ taos_gauge_t **metric = NULL;
+
+ metric = taosHashGet(tsMonitor.metrics, MASTER_UPTIME, strlen(MASTER_UPTIME));
+ taos_gauge_set(*metric, pInfo->master_uptime, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, DBS_TOTAL, strlen(DBS_TOTAL));
+ taos_gauge_set(*metric, pInfo->dbs_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, TBS_TOTAL, strlen(TBS_TOTAL));
+ taos_gauge_set(*metric, pInfo->tbs_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, STBS_TOTAL, strlen(STBS_TOTAL));
+ taos_gauge_set(*metric, pInfo->stbs_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, VGROUPS_TOTAL, strlen(VGROUPS_TOTAL));
+ taos_gauge_set(*metric, pInfo->vgroups_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, VGROUPS_ALIVE, strlen(VGROUPS_ALIVE));
+ taos_gauge_set(*metric, pInfo->vgroups_alive, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, VNODES_TOTAL, strlen(VNODES_TOTAL));
+ taos_gauge_set(*metric, pInfo->vnodes_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, VNODES_ALIVE, strlen(VNODES_ALIVE));
+ taos_gauge_set(*metric, pInfo->vnodes_alive, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, CONNECTIONS_TOTAL, strlen(CONNECTIONS_TOTAL));
+ taos_gauge_set(*metric, pInfo->connections_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, TOPICS_TOTAL, strlen(TOPICS_TOTAL));
+ taos_gauge_set(*metric, pInfo->topics_toal, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, STREAMS_TOTAL, strlen(STREAMS_TOTAL));
+ taos_gauge_set(*metric, pInfo->streams_total, sample_label_values);
+
+ //dnodes number
+ int32_t dnode_total = taosArrayGetSize(pInfo->dnodes);
+ int32_t dnode_alive = 0;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
+ SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i);
+
+ if(strcmp(pDnodeDesc->status, "ready") == 0){
+ dnode_alive++;
+ }
+ }
+
+ metric = taosHashGet(tsMonitor.metrics, DNODES_TOTAL, strlen(DNODES_TOTAL));
+ taos_gauge_set(*metric, dnode_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE));
+ taos_gauge_set(*metric, dnode_alive, sample_label_values);
+
+ //mnodes number
+ int32_t mnode_total = taosArrayGetSize(pInfo->mnodes);
+ int32_t mnode_alive = 0;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) {
+
+ SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i);
+
+ bool dnodeIsOnline = false;
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
+ SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i);
+
+ if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){
+ if(strcmp(pDnodeDesc->status, "ready") == 0){
+ dnodeIsOnline = true;
+ }
+ }
+ }
+
+ if(dnodeIsOnline){
+ if(pMnodeDesc->syncState != 0){
+ mnode_alive++;
+ }
+ }
+ }
+
+ metric = taosHashGet(tsMonitor.metrics, MNODES_TOTAL, strlen(MNODES_TOTAL));
+ taos_gauge_set(*metric, mnode_total, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, MNODES_ALIVE, strlen(MNODES_ALIVE));
+ taos_gauge_set(*metric, mnode_alive, sample_label_values);
+
+ //grant info
+ metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME));
+ taos_gauge_set(*metric, pGrantInfo->expire_time, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, TIMESERIES_USED, strlen(TIMESERIES_USED));
+ taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_label_values);
+
+ metric = taosHashGet(tsMonitor.metrics, TIMESERIES_TOTAL, strlen(TIMESERIES_TOTAL));
+ taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_label_values);
+}
+
+void monGenVgroupInfoTable(SMonInfo *pMonitor){
+ if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){
+ uError("failed to delete metric "TABLES_NUM);
+ }
+
+ if(taos_collector_registry_deregister_metric(STATUS) != 0){
+ uError("failed to delete metric "STATUS);
+ }
+
+ if(pMonitor->dmInfo.basic.cluster_id == 0) return;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ int32_t vgroup_label_count = 3;
+ const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"};
+ taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels);
+ if(taos_collector_registry_register_metric(tableNumGauge) == 1){
+ taos_counter_destroy(tableNumGauge);
+ }
+ taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels);
+ if(taos_collector_registry_register_metric(statusGauge) == 1){
+ taos_counter_destroy(statusGauge);
+ }
+
+ char cluster_id[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id);
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) {
+ SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i);
+
+ char vgroup_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(vgroup_id, TSDB_NODE_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id);
+
+ const char *sample_labels[] = {cluster_id, vgroup_id, pVgroupDesc->database_name};
+
+ taos_gauge_t **metric = NULL;
+
+ taos_gauge_set(tableNumGauge, pVgroupDesc->tables_num, sample_labels);
+
+ int32_t status = 0;
+ if(strcmp(pVgroupDesc->status, "ready") == 0){
+ status = 1;
+ }
+ taos_gauge_set(statusGauge, status, sample_labels);
+ }
+}
+
+void monGenDnodeInfoTable(SMonInfo *pMonitor) {
+ if(pMonitor->dmInfo.basic.cluster_id == 0) {
+ uError("failed to generate dnode info table since cluster_id is 0");
+ return;
+ }
+
+ char cluster_id[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id);
+
+ char dnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id);
+
+ const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep};
+
+ taos_gauge_t **metric = NULL;
+
+ //dnode info
+ SMonDnodeInfo *pInfo = &pMonitor->dmInfo.dnode;
+ SMonSysInfo *pSys = &pMonitor->dmInfo.sys;
+ SVnodesStat *pStat = &pMonitor->vmInfo.vstat;
+ SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster;
+
+ double interval = (pMonitor->curTime - pMonitor->lastTime) / 1000.0;
+ if (pMonitor->curTime - pMonitor->lastTime == 0) {
+ interval = 1;
+ }
+
+ double cpu_engine = 0;
+ double mem_engine = 0;
+ double net_in = 0;
+ double net_out = 0;
+ double io_read = 0;
+ double io_write = 0;
+ double io_read_disk = 0;
+ double io_write_disk = 0;
+
+ SMonSysInfo *sysArrays[6];
+ sysArrays[0] = &pMonitor->dmInfo.sys;
+ sysArrays[1] = &pMonitor->mmInfo.sys;
+ sysArrays[2] = &pMonitor->vmInfo.sys;
+ sysArrays[3] = &pMonitor->qmInfo.sys;
+ sysArrays[4] = &pMonitor->smInfo.sys;
+ sysArrays[5] = &pMonitor->bmInfo.sys;
+ for (int32_t i = 0; i < 6; ++i) {
+ cpu_engine += sysArrays[i]->cpu_engine;
+ mem_engine += sysArrays[i]->mem_engine;
+ net_in += sysArrays[i]->net_in;
+ net_out += sysArrays[i]->net_out;
+ io_read += sysArrays[i]->io_read;
+ io_write += sysArrays[i]->io_write;
+ io_read_disk += sysArrays[i]->io_read_disk;
+ io_write_disk += sysArrays[i]->io_write_disk;
+ }
+
+ double req_select_rate = pStat->numOfSelectReqs / interval;
+ double req_insert_rate = pStat->numOfInsertReqs / interval;
+ double req_insert_batch_rate = pStat->numOfBatchInsertReqs / interval;
+ double net_in_rate = net_in / interval;
+ double net_out_rate = net_out / interval;
+ double io_read_rate = io_read / interval;
+ double io_write_rate = io_write / interval;
+ double io_read_disk_rate = io_read_disk / interval;
+ double io_write_disk_rate = io_write_disk / interval;
+
+ metric = taosHashGet(tsMonitor.metrics, UPTIME, strlen(UPTIME));
+ taos_gauge_set(*metric, pInfo->uptime, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, CPU_ENGINE, strlen(CPU_ENGINE));
+ taos_gauge_set(*metric, cpu_engine, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, CPU_SYSTEM, strlen(CPU_SYSTEM));
+ taos_gauge_set(*metric, pSys->cpu_system, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, CPU_CORE, strlen(CPU_CORE));
+ taos_gauge_set(*metric, pSys->cpu_cores, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, MEM_ENGINE, strlen(MEM_ENGINE));
+ taos_gauge_set(*metric, mem_engine, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, MEM_SYSTEM, strlen(MEM_SYSTEM));
+ taos_gauge_set(*metric, pSys->mem_system, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, MEM_TOTAL, strlen(MEM_TOTAL));
+ taos_gauge_set(*metric, pSys->mem_total, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DISK_ENGINE, strlen(DISK_ENGINE));
+ taos_gauge_set(*metric, pSys->disk_engine, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DISK_USED, strlen(DISK_USED));
+ taos_gauge_set(*metric, pSys->disk_used, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DISK_TOTAL, strlen(DISK_TOTAL));
+ taos_gauge_set(*metric, pSys->disk_total, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, NET_IN, strlen(NET_IN));
+ taos_gauge_set(*metric, net_in_rate, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, NET_OUT, strlen(NET_OUT));
+ taos_gauge_set(*metric, net_out_rate, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, IO_READ, strlen(IO_READ));
+ taos_gauge_set(*metric, io_read_rate, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, IO_WRITE, strlen(IO_WRITE));
+ taos_gauge_set(*metric, io_write_rate, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, IO_READ_DISK, strlen(IO_READ_DISK));
+ taos_gauge_set(*metric, io_read_disk_rate, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, IO_WRITE_DISK, strlen(IO_WRITE_DISK));
+ taos_gauge_set(*metric, io_write_disk_rate, sample_labels);
+
+ //metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS));
+ //taos_gauge_set(*metric, pStat->errors, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM));
+ taos_gauge_set(*metric, pStat->totalVnodes, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, MASTERS, strlen(MASTERS));
+ taos_gauge_set(*metric, pStat->masterNum, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, HAS_MNODE, strlen(HAS_MNODE));
+ taos_gauge_set(*metric, pInfo->has_mnode, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, HAS_QNODE, strlen(HAS_QNODE));
+ taos_gauge_set(*metric, pInfo->has_qnode, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, HAS_SNODE, strlen(HAS_SNODE));
+ taos_gauge_set(*metric, pInfo->has_snode, sample_labels);
+
+ //log number
+ SMonLogs *logs[6];
+ logs[0] = &pMonitor->log;
+ logs[1] = &pMonitor->mmInfo.log;
+ logs[2] = &pMonitor->vmInfo.log;
+ logs[3] = &pMonitor->smInfo.log;
+ logs[4] = &pMonitor->qmInfo.log;
+ logs[5] = &pMonitor->bmInfo.log;
+
+ int32_t numOfErrorLogs = 0;
+ int32_t numOfInfoLogs = 0;
+ int32_t numOfDebugLogs = 0;
+ int32_t numOfTraceLogs = 0;
+
+ for (int32_t j = 0; j < 6; j++) {
+ SMonLogs *pLog = logs[j];
+ numOfErrorLogs += pLog->numOfErrorLogs;
+ numOfInfoLogs += pLog->numOfInfoLogs;
+ numOfDebugLogs += pLog->numOfDebugLogs;
+ numOfTraceLogs += pLog->numOfTraceLogs;
+ }
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_ERROR, strlen(DNODE_LOG_ERROR));
+ taos_gauge_set(*metric, numOfErrorLogs, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_INFO, strlen(DNODE_LOG_INFO));
+ taos_gauge_set(*metric, numOfInfoLogs, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_DEBUG, strlen(DNODE_LOG_DEBUG));
+ taos_gauge_set(*metric, numOfDebugLogs, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TRACE, strlen(DNODE_LOG_TRACE));
+ taos_gauge_set(*metric, numOfTraceLogs, sample_labels);
+}
+
+void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){
+ if(taos_collector_registry_deregister_metric(DNODE_STATUS) != 0){
+ uError("failed to delete metric "DNODE_STATUS);
+ }
+
+ if(pMonitor->dmInfo.basic.cluster_id == 0) {
+ uError("failed to generate dnode info table since cluster_id is 0");
+ return;
+ }
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ taos_gauge_t *gauge = NULL;
+
+ int32_t dnodes_label_count = 3;
+ const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"};
+
+ gauge= taos_gauge_new(DNODE_STATUS, "", dnodes_label_count, dnodes_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+
+ char cluster_id[TSDB_CLUSTER_ID_LEN];
+ snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id);
+
+ //dnodes status
+
+ SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pClusterInfo->dnodes); ++i) {
+ SMonDnodeDesc *pDnodeDesc = taosArrayGet(pClusterInfo->dnodes, i);
+
+ char dnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pDnodeDesc->dnode_id);
+
+ const char *sample_labels[] = {cluster_id, dnode_id, pDnodeDesc->dnode_ep};
+
+ int32_t status = 0;
+ if(strcmp(pDnodeDesc->status, "ready") == 0){
+ status = 1;
+ }
+ taos_gauge_set(gauge, status, sample_labels);
+ }
+}
+
+void monGenDataDiskTable(SMonInfo *pMonitor){
+ if(pMonitor->dmInfo.basic.cluster_id == 0) return;
+
+ SMonDiskInfo *pInfo = &pMonitor->vmInfo.tfs;
+
+ char cluster_id[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id);
+
+ char dnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id);
+
+ taos_gauge_t **metric = NULL;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) {
+ SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i);
+
+ char level[LEVEL_LEN] = {0};
+ snprintf(level, LEVEL_LEN, "%"PRId32, pDatadirDesc->level);
+
+ const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pDatadirDesc->name, level};
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_AVAIL, strlen(DNODE_DATA_AVAIL));
+ taos_gauge_set(*metric, pDatadirDesc->size.avail, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_USED, strlen(DNODE_DATA_USED));
+ taos_gauge_set(*metric, pDatadirDesc->size.used, sample_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_TOTAL, strlen(DNODE_DATA_TOTAL));
+ taos_gauge_set(*metric, pDatadirDesc->size.total, sample_labels);
+ }
+}
+
+void monGenLogDiskTable(SMonInfo *pMonitor){
+ if(pMonitor->dmInfo.basic.cluster_id == 0) return;
+
+ SMonDiskDesc *pLogDesc = &pMonitor->dmInfo.dnode.logdir;
+ SMonDiskDesc *pTempDesc = &pMonitor->dmInfo.dnode.tempdir;
+
+ char cluster_id[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id);
+
+ char dnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id);
+
+ taos_gauge_t **metric = NULL;
+
+ const char *sample_log_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pLogDesc->name};
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL));
+ taos_gauge_set(*metric, pLogDesc->size.avail, sample_log_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED));
+ taos_gauge_set(*metric, pLogDesc->size.used, sample_log_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL));
+ taos_gauge_set(*metric, pLogDesc->size.total, sample_log_labels);
+
+ const char *sample_temp_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pTempDesc->name};
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL));
+ taos_gauge_set(*metric, pTempDesc->size.avail, sample_temp_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED));
+ taos_gauge_set(*metric, pTempDesc->size.used, sample_temp_labels);
+
+ metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL));
+ taos_gauge_set(*metric, pTempDesc->size.total, sample_temp_labels);
+}
+
+void monGenMnodeRoleTable(SMonInfo *pMonitor){
+ char *mnodes_role_gauges[] = {MNODE_ROLE};
+
+ for(int32_t i = 0; i < 1; i++){
+ if(taos_collector_registry_deregister_metric(mnodes_role_gauges[i]) != 0){
+ uError("failed to delete metric %s", mnodes_role_gauges[i]);
+ }
+
+ taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]));
+ }
+
+ SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+ SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic;
+ if(pBasicInfo->cluster_id == 0) return;
+
+ taos_gauge_t *gauge = NULL;
+ int32_t mnodes_role_label_count = 3;
+ const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"};
+ for(int32_t i = 0; i < 1; i++){
+ gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+
+ char buf[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id);
+
+ taos_gauge_t **metric = NULL;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) {
+
+ SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i);
+
+ char mnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(mnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMnodeDesc->mnode_id);
+
+ const char *sample_labels[] = {buf, mnode_id, pMnodeDesc->mnode_ep};
+
+ bool dnodeIsOnline = false;
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
+ SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i);
+
+ if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){
+ if(strcmp(pDnodeDesc->status, "ready") == 0){
+ dnodeIsOnline = true;
+ }
+ }
+ }
+
+ metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE));
+
+ if(dnodeIsOnline){
+ taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels);
+ }
+ else{
+ taos_gauge_set(*metric, 0, sample_labels);
+ }
+ //metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE));
+ //taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels);
+ }
+}
+
+void monGenVnodeRoleTable(SMonInfo *pMonitor){
+ char *vnodes_role_gauges[] = {VNODE_ROLE};
+ taos_gauge_t *gauge = NULL;
+
+ for(int32_t i = 0; i < 1; i++){
+ if(taos_collector_registry_deregister_metric(vnodes_role_gauges[i]) != 0){
+ uError("failed to delete metric %s", vnodes_role_gauges[i]);
+ }
+
+ taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]));
+ }
+
+ SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic;
+ if(pBasicInfo->cluster_id == 0) return;
+
+ int32_t vnodes_role_label_count = 4;
+ const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"};
+ for(int32_t i = 0; i < 1; i++){
+ gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels);
+ if(taos_collector_registry_register_metric(gauge) == 1){
+ taos_counter_destroy(gauge);
+ }
+ taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *));
+ }
+
+ char buf[TSDB_CLUSTER_ID_LEN] = {0};
+ snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id);
+
+ taos_gauge_t **metric = NULL;
+
+ for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) {
+ SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i);
+
+ char vgroup_id[TSDB_VGROUP_ID_LEN] = {0};
+ snprintf(vgroup_id, TSDB_VGROUP_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id);
+
+ for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
+ SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j];
+ if (pVnodeDesc->dnode_id <= 0) continue;
+
+ char dnode_id[TSDB_NODE_ID_LEN] = {0};
+ snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pVnodeDesc->dnode_id);
+
+ const char *sample_labels[] = {buf, vgroup_id, pVgroupDesc->database_name, dnode_id};
+
+ metric = taosHashGet(tsMonitor.metrics, VNODE_ROLE, strlen(VNODE_ROLE));
+ taos_gauge_set(*metric, pVnodeDesc->syncState, sample_labels);
+ }
+ }
+}
+
+void monSendPromReport() {
+ char ts[50] = {0};
+ sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
+
+ char* promStr = NULL;
+ char* pCont = (char *)taos_collector_registry_bridge_new(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64, &promStr);
+ if(tsMonitorLogProtocol){
+ if(pCont != NULL){
+ uInfoL("report cont:\n%s", pCont);
+ uDebugL("report cont prom:\n%s", promStr);
+ }
+ else{
+ uInfo("report cont is null");
+ }
+ }
+ if (pCont != NULL) {
+ EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
+ if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
+ uError("failed to send monitor msg");
+ }else{
+ taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT);
+ }
+ taosMemoryFreeClear(pCont);
+ }
+ if(promStr != NULL){
+ taosMemoryFreeClear(promStr);
+ }
+}
\ No newline at end of file
diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c
index 56cf0a2b51..4ecc6f1261 100644
--- a/source/libs/monitor/src/monMain.c
+++ b/source/libs/monitor/src/monMain.c
@@ -18,9 +18,13 @@
#include "taoserror.h"
#include "thttp.h"
#include "ttime.h"
+#include "taos_monitor.h"
+#include "tglobal.h"
-static SMonitor tsMonitor = {0};
-static char* tsMonUri = "/report";
+SMonitor tsMonitor = {0};
+char* tsMonUri = "/report";
+char* tsMonFwUri = "/general-metric";
+char* tsMonFwBasicUri = "/taosd-cluster-basic";
void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
taosThreadMutexLock(&tsMonitor.lock);
@@ -108,6 +112,9 @@ int32_t monInit(const SMonCfg *pCfg) {
tsLogFp = monRecordLog;
tsMonitor.lastTime = taosGetTimestampMs();
taosThreadMutexInit(&tsMonitor.lock, NULL);
+
+ monInitMonitorFW();
+
return 0;
}
@@ -121,6 +128,8 @@ void monCleanup() {
tFreeSMonQmInfo(&tsMonitor.qmInfo);
tFreeSMonBmInfo(&tsMonitor.bmInfo);
taosThreadMutexDestroy(&tsMonitor.lock);
+
+ monCleanupMonitorFW();
}
static void monCleanupMonitorInfo(SMonInfo *pMonitor) {
@@ -186,6 +195,22 @@ static void monGenBasicJson(SMonInfo *pMonitor) {
tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol);
}
+static void monGenBasicJsonBasic(SMonInfo *pMonitor) {
+ SMonBasicInfo *pInfo = &pMonitor->dmInfo.basic;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ SJson *pJson = pMonitor->pJson;
+ char buf[40] = {0};
+
+ sprintf(buf, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
+ tjsonAddStringToObject(pJson, "ts", buf);
+ tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id);
+ tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep);
+ snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id);
+ tjsonAddStringToObject(pJson, "cluster_id", buf);
+ tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol);
+}
+
static void monGenClusterJson(SMonInfo *pMonitor) {
SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
@@ -244,6 +269,17 @@ static void monGenClusterJson(SMonInfo *pMonitor) {
}
}
+static void monGenClusterJsonBasic(SMonInfo *pMonitor) {
+ SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster;
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
+
+ //tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep);
+ tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst);
+ tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id);
+ tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version);
+ //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval);
+}
+
static void monGenVgroupJson(SMonInfo *pMonitor) {
SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
@@ -517,21 +553,11 @@ static void monGenLogJson(SMonInfo *pMonitor) {
if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace);
}
-void monSendReport() {
- SMonInfo *pMonitor = monCreateMonitorInfo();
- if (pMonitor == NULL) return;
-
- monGenBasicJson(pMonitor);
- monGenClusterJson(pMonitor);
- monGenVgroupJson(pMonitor);
- monGenStbJson(pMonitor);
- monGenGrantJson(pMonitor);
- monGenDnodeJson(pMonitor);
- monGenDiskJson(pMonitor);
- monGenLogJson(pMonitor);
-
+void monSendReport(SMonInfo *pMonitor){
char *pCont = tjsonToString(pMonitor->pJson);
- // uDebugL("report cont:%s\n", pCont);
+ if(tsMonitorLogProtocol){
+ uInfoL("report cont:\n%s", pCont);
+ }
if (pCont != NULL) {
EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
@@ -539,6 +565,83 @@ void monSendReport() {
}
taosMemoryFree(pCont);
}
+}
+
+void monGenAndSendReport() {
+ SMonInfo *pMonitor = monCreateMonitorInfo();
+ if (pMonitor == NULL) return;
+
+ if(!tsMonitorForceV2){
+ monGenBasicJson(pMonitor);
+ monGenClusterJson(pMonitor);
+ monGenVgroupJson(pMonitor);
+ monGenStbJson(pMonitor);
+ monGenGrantJson(pMonitor);
+ monGenDnodeJson(pMonitor);
+ monGenDiskJson(pMonitor);
+ monGenLogJson(pMonitor);
+
+ monSendReport(pMonitor);
+ }
+ else{
+ monGenClusterInfoTable(pMonitor);
+ monGenVgroupInfoTable(pMonitor);
+ monGenDnodeInfoTable(pMonitor);
+ monGenDnodeStatusInfoTable(pMonitor);
+ monGenDataDiskTable(pMonitor);
+ monGenLogDiskTable(pMonitor);
+ monGenMnodeRoleTable(pMonitor);
+ monGenVnodeRoleTable(pMonitor);
+
+ monSendPromReport();
+ }
monCleanupMonitorInfo(pMonitor);
}
+
+void monSendReportBasic(SMonInfo *pMonitor){
+ char *pCont = tjsonToString(pMonitor->pJson);
+ if(tsMonitorLogProtocol){
+ if(pCont != NULL){
+ uInfoL("report cont basic:\n%s", pCont);
+ }
+ else{
+ uInfo("report cont basic is null");
+ }
+ }
+ if (pCont != NULL) {
+ EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
+ if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
+ uError("failed to send monitor msg");
+ }
+ taosMemoryFree(pCont);
+ }
+}
+
+void monGenAndSendReportBasic() {
+ SMonInfo *pMonitor = monCreateMonitorInfo();
+
+ monGenBasicJsonBasic(pMonitor);
+ monGenClusterJsonBasic(pMonitor);
+
+ if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) {
+ monSendReportBasic(pMonitor);
+ }
+
+ monCleanupMonitorInfo(pMonitor);
+}
+
+void monSendContent(char *pCont) {
+ if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
+ if(tsMonitorLogProtocol){
+ if (pCont != NULL){
+ uInfoL("report client cont:\n%s\n", pCont);
+ }
+ }
+ if (pCont != NULL) {
+ EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
+ if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
+ uError("failed to send monitor msg");
+ }
+ }
+}
\ No newline at end of file
diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp
index 3f7b1b51da..2660cff216 100644
--- a/source/libs/monitor/test/monTest.cpp
+++ b/source/libs/monitor/test/monTest.cpp
@@ -283,10 +283,10 @@ TEST_F(MonitorTest, 01_Full) {
tFreeSMonSmInfo(&smInfo);
tFreeSMonQmInfo(&qmInfo);
tFreeSMonBmInfo(&bmInfo);
- monSendReport();
+ monGenAndSendReport();
}
TEST_F(MonitorTest, 02_Log) {
AddLogInfo2();
- monSendReport();
+ monGenAndSendReport();
}
diff --git a/source/libs/monitorfw/CMakeLists.txt b/source/libs/monitorfw/CMakeLists.txt
new file mode 100644
index 0000000000..610cd63985
--- /dev/null
+++ b/source/libs/monitorfw/CMakeLists.txt
@@ -0,0 +1,9 @@
+aux_source_directory(src MONITOR2_SRC)
+add_library(monitorfw STATIC ${MONITOR2_SRC})
+target_include_directories(
+ monitorfw
+ PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw"
+ PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(monitorfw os util common transport)
diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_assert.h
new file mode 100644
index 0000000000..d3226eed26
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_assert.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+#ifndef TAOS_ASSERT_H
+#define TAOS_ASSERT_H
+
+#ifdef TAOS_ASSERT_ENABLE
+#define TAOS_ASSERT(i) assert(i);
+#else
+#define TAOS_ASSERT(i)
+#endif // TAOS_TEST
+
+#endif // TAOS_ASSERT_H
diff --git a/source/libs/monitorfw/inc/taos_collector_registry_i.h b/source/libs/monitorfw/inc/taos_collector_registry_i.h
new file mode 100644
index 0000000000..ed5bb1fe19
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_collector_registry_i.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "taos_collector_registry_t.h"
+
+#ifndef TAOS_COLLECTOR_REGISTRY_I_INCLUDED
+#define TAOS_COLLECTOR_REGISTRY_I_INCLUDED
+
+int taos_collector_registry_enable_custom_process_metrics(taos_collector_registry_t *self,
+ const char *process_limits_path,
+ const char *process_stats_path);
+
+#endif // TAOS_COLLECTOR_REGISTRY_I_INCLUDED
diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h
new file mode 100644
index 0000000000..8e8a881fca
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_REGISTRY_T_H
+#define TAOS_REGISTRY_T_H
+
+#include
+#include
+
+// Public
+#include "taos_collector_registry.h"
+
+// Private
+#include "taos_map_t.h"
+#include "taos_metric_formatter_t.h"
+#include "taos_string_builder_t.h"
+
+struct taos_collector_registry {
+ const char *name;
+ bool disable_process_metrics; /**< Disables the collection of process metrics */
+ taos_map_t *collectors; /**< Map of collectors keyed by name */
+ taos_string_builder_t *string_builder; /**< Enables string building */
+ taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */
+ pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */
+ taos_string_builder_t *string_builder_batch;
+};
+
+#endif // TAOS_REGISTRY_T_H
diff --git a/source/libs/monitorfw/inc/taos_collector_t.h b/source/libs/monitorfw/inc/taos_collector_t.h
new file mode 100644
index 0000000000..264e8e9ad9
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_collector_t.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_COLLECTOR_T_H
+#define TAOS_COLLECTOR_T_H
+
+#include "taos_collector.h"
+#include "taos_map_t.h"
+#include "taos_string_builder_t.h"
+
+struct taos_collector {
+ const char *name;
+ taos_map_t *metrics;
+ taos_collect_fn *collect_fn;
+ taos_string_builder_t *string_builder;
+ const char *proc_limits_file_path;
+ const char *proc_stat_file_path;
+};
+
+#endif // TAOS_COLLECTOR_T_H
diff --git a/source/libs/monitorfw/inc/taos_errors.h b/source/libs/monitorfw/inc/taos_errors.h
new file mode 100644
index 0000000000..ee2a894df3
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_errors.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define TAOS_STDIO_CLOSE_DIR_ERROR "failed to close dir"
+#define TAOS_STDIO_OPEN_DIR_ERROR "failed to open dir"
+#define TAOS_METRIC_INCORRECT_TYPE "incorrect metric type"
+#define TAOS_METRIC_INVALID_LABEL_NAME "invalid label name"
+#define TAOS_PTHREAD_RWLOCK_DESTROY_ERROR "failed to destroy the pthread_rwlock_t*"
+#define TAOS_PTHREAD_RWLOCK_INIT_ERROR "failed to initialize the pthread_rwlock_t*"
+#define TAOS_PTHREAD_RWLOCK_LOCK_ERROR "failed to lock the pthread_rwlock_t*"
+#define TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR "failed to unlock the pthread_rwlock_t*"
+#define TAOS_REGEX_REGCOMP_ERROR "failed to compile the regular expression"
+#define TAOS_REGEX_REGEXEC_ERROR "failed to execute the regular expression"
diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h
new file mode 100644
index 0000000000..ed11a76427
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_linked_list_i.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_LIST_I_INCLUDED
+#define TAOS_LIST_I_INCLUDED
+
+// Private
+#include "taos_linked_list_t.h"
+
+/**
+ * @brief API PRIVATE Returns a pointer to a taos_linked_list
+ */
+taos_linked_list_t *taos_linked_list_new(void);
+
+/**
+ * @brief API PRIVATE removes all nodes from the given taos_linked_list *
+ */
+int taos_linked_list_purge(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Destroys a taos_linked_list
+ */
+int taos_linked_list_destroy(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Append an item to the back of the list
+ */
+int taos_linked_list_append(taos_linked_list_t *self, void *item);
+
+/**
+ * @brief API PRIVATE Push an item onto the front of the list
+ */
+int taos_linked_list_push(taos_linked_list_t *self, void *item);
+
+/**
+ * @brief API PRIVATE Pop the first item off of the list
+ */
+void *taos_linked_list_pop(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Returns the item at the head of the list or NULL if not present
+ */
+void *taos_linked_list_first(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Returns the item at the tail of the list or NULL if not present
+ */
+void *taos_linked_list_last(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Removes an item from the linked list
+ */
+int taos_linked_list_remove(taos_linked_list_t *self, void *item);
+
+/**
+ * @brief API PRIVATE Compares two items within a linked list
+ */
+taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *node_b);
+
+/**
+ * @brief API PRIVATE Get the size
+ */
+size_t taos_linked_list_size(taos_linked_list_t *self);
+
+/**
+ * @brief API PRIVATE Set the free_fn member on taos_linked_list
+ */
+int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn);
+
+/**
+ * @brief API PRIVATE Set the compare_fn member on the taos_linked_list
+ */
+int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn);
+
+/**
+ * API PRIVATE
+ * @brief does nothing
+ */
+void taos_linked_list_no_op_free(void *item);
+
+#endif // TAOS_LIST_I_INCLUDED
diff --git a/source/libs/monitorfw/inc/taos_linked_list_t.h b/source/libs/monitorfw/inc/taos_linked_list_t.h
new file mode 100644
index 0000000000..ccd82ffb61
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_linked_list_t.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_LIST_T_H
+#define TAOS_LIST_T_H
+
+#include "taos_linked_list.h"
+
+typedef enum { TAOS_LESS = -1, TAOS_EQUAL = 0, TAOS_GREATER = 1 } taos_linked_list_compare_t;
+
+/**
+ * @brief API PRIVATE Frees an item in a taos_linked_list_node
+ */
+typedef void (*taos_linked_list_free_item_fn)(void *);
+
+/**
+ * @brief API PRIVATE Compares two items within a taos_linked_list
+ */
+typedef taos_linked_list_compare_t (*taos_linked_list_compare_item_fn)(void *item_a, void *item_b);
+
+/**
+ * @brief API PRIVATE A struct containing a generic item, represented as a void pointer, and next, a pointer to the
+ * next taos_linked_list_node*
+ */
+typedef struct taos_linked_list_node {
+ struct taos_linked_list_node *next;
+ void *item;
+} taos_linked_list_node_t;
+
+/**
+ * @brief API PRIVATE A linked list comprised of taos_linked_list_node* instances
+ */
+struct taos_linked_list {
+ taos_linked_list_node_t *head;
+ taos_linked_list_node_t *tail;
+ size_t size;
+ taos_linked_list_free_item_fn free_fn;
+ taos_linked_list_compare_item_fn compare_fn;
+};
+
+#endif // TAOS_LIST_T_H
diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h
new file mode 100644
index 0000000000..fecbda59e5
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_log.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+#ifndef TAOS_LOG_H
+#define TAOS_LOG_H
+
+//#define TAOS_LOG_ENABLE
+
+#ifdef TAOS_LOG_ENABLE
+#define TAOS_LOG(msg) printf("monitor_log %s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg);
+#else
+#define TAOS_LOG(msg)
+#endif // TAOS_LOG_ENABLE
+
+#endif // TAOS_LOG_H
diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h
new file mode 100644
index 0000000000..55f248b96a
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_map_i.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_MAP_I_INCLUDED
+#define TAOS_MAP_I_INCLUDED
+
+#include "taos_map_t.h"
+
+taos_map_t *taos_map_new(void);
+
+int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn);
+
+void *taos_map_get(taos_map_t *self, const char *key);
+
+void *taos_map_get_withoutlock(taos_map_t *self, const char *key);
+
+int taos_map_set(taos_map_t *self, const char *key, void *value);
+
+int taos_map_delete(taos_map_t *self, const char *key);
+
+int taos_map_destroy(taos_map_t *self);
+
+size_t taos_map_size(taos_map_t *self);
+
+taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn);
+
+#endif // TAOS_MAP_I_INCLUDED
diff --git a/source/libs/monitorfw/inc/taos_map_t.h b/source/libs/monitorfw/inc/taos_map_t.h
new file mode 100644
index 0000000000..6fcbda1366
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_map_t.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_MAP_T_H
+#define TAOS_MAP_T_H
+
+#include
+
+// Public
+#include "taos_map.h"
+
+// Private
+#include "taos_linked_list_t.h"
+
+typedef void (*taos_map_node_free_value_fn)(void *);
+
+struct taos_map_node {
+ const char *key;
+ void *value;
+ taos_map_node_free_value_fn free_value_fn;
+};
+
+struct taos_map {
+ size_t size; /**< contains the size of the map */
+ size_t max_size; /**< stores the current max_size */
+ taos_linked_list_t *keys; /**< linked list containing containing all keys present */
+ taos_linked_list_t **addrs; /**< Sequence of linked lists. Each list contains nodes with the same index */
+ pthread_rwlock_t *rwlock;
+ taos_map_node_free_value_fn free_value_fn;
+};
+
+#endif // TAOS_MAP_T_H
diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h
new file mode 100644
index 0000000000..f3e4ebae75
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_METRIC_FORMATTER_CUSTOMV2_I_H
+#define TAOS_METRIC_FORMATTER_CUSTOMV2_I_H
+
+#define ALLOW_FORBID_FUNC
+
+#include
+#include "tjson.h"
+
+int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample,
+ char *ts, char *format, char *metricName, int32_t metric_type,
+ SJson *arrayMetricGroups);
+int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format,
+ SJson* tableArray);
+int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts,
+ char *format, SJson* tableArray);
+#endif // TAOS_METRIC_FORMATTER_CUSTOMV2_I_H
\ No newline at end of file
diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h
new file mode 100644
index 0000000000..fba30d9eeb
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_METRIC_FORMATTER_I_H
+#define TAOS_METRIC_FORMATTER_I_H
+
+// Private
+#include "taos_metric_formatter_t.h"
+#include "taos_metric_t.h"
+
+/**
+ * @brief API PRIVATE taos_metric_formatter constructor
+ */
+taos_metric_formatter_t *taos_metric_formatter_new();
+
+/**
+ * @brief API PRIVATE taos_metric_formatter destructor
+ */
+int taos_metric_formatter_destroy(taos_metric_formatter_t *self);
+
+/**
+ * @brief API PRIVATE Loads the help text
+ */
+int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help);
+
+/**
+ * @brief API PRIVATE Loads the type text
+ */
+int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type);
+
+/**
+ * @brief API PRIVATE Loads the formatter with a metric sample L-value
+ * @param name The metric name
+ * @param suffix The metric suffix. This is applicable to Summary and Histogram metric types.
+ * @param label_count The number of labels for the given metric.
+ * @param label_keys An array of constant strings.
+ * @param label_values An array of constant strings.
+ *
+ * The number of const char **and taos_label_value must be the same.
+ */
+int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter, const char *name, const char *suffix,
+ size_t label_count, const char **label_keys, const char **label_values);
+
+/**
+ * @brief API PRIVATE Loads the formatter with a metric sample
+ */
+int taos_metric_formatter_load_sample(taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample,
+ char *ts, char *format);
+
+/**
+ * @brief API PRIVATE Loads a metric in the string exposition format
+ */
+int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format);
+
+/**
+ * @brief API PRIVATE Loads the given metrics
+ */
+int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format);
+
+/**
+ * @brief API PRIVATE Clear the underlying string_builder
+ */
+int taos_metric_formatter_clear(taos_metric_formatter_t *self);
+
+/**
+ * @brief API PRIVATE Returns the string built by taos_metric_formatter
+ */
+char *taos_metric_formatter_dump(taos_metric_formatter_t *metric_formatter);
+
+#endif // TAOS_METRIC_FORMATTER_I_H
diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_t.h b/source/libs/monitorfw/inc/taos_metric_formatter_t.h
new file mode 100644
index 0000000000..0d7425aa59
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_formatter_t.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_METRIC_FORMATTER_T_H
+#define TAOS_METRIC_FORMATTER_T_H
+
+#include "taos_string_builder_t.h"
+
+typedef struct taos_metric_formatter {
+ taos_string_builder_t *string_builder;
+ taos_string_builder_t *err_builder;
+} taos_metric_formatter_t;
+
+#endif // TAOS_METRIC_FORMATTER_T_H
diff --git a/source/libs/monitorfw/inc/taos_metric_i.h b/source/libs/monitorfw/inc/taos_metric_i.h
new file mode 100644
index 0000000000..e8ae799547
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_i.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// Private
+#include "taos_metric_t.h"
+
+#ifndef TAOS_METRIC_I_INCLUDED
+#define TAOS_METRIC_I_INCLUDED
+
+/**
+ * @brief API PRIVATE Returns a *taos_metric
+ */
+taos_metric_t *taos_metric_new(taos_metric_type_t type, const char *name, const char *help, size_t label_key_count,
+ const char **label_keys);
+
+/**
+ * @brief API PRIVATE Destroys a *taos_metric
+ */
+int taos_metric_destroy(taos_metric_t *self);
+
+/**
+ * @brief API PRIVATE takes a generic item, casts to a *taos_metric_t and destroys it
+ */
+int taos_metric_destroy_generic(void *item);
+
+/**
+ * @brief API Private takes a generic item, casts to a *taos_metric_t and destroys it. Discards any errors.
+ */
+void taos_metric_free_generic(void *item);
+
+#endif // TAOS_METRIC_I_INCLUDED
diff --git a/source/libs/monitorfw/inc/taos_metric_sample_i.h b/source/libs/monitorfw/inc/taos_metric_sample_i.h
new file mode 100644
index 0000000000..b5e90f1933
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_sample_i.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "taos_metric_sample_t.h"
+#include "taos_metric_t.h"
+
+#ifndef TAOS_METRIC_SAMPLE_I_H
+#define TAOS_METRIC_SAMPLE_I_H
+
+/**
+ * @brief API PRIVATE Return a taos_metric_sample_t*
+ *
+ * @param type The type of metric sample
+ * @param l_value The entire left value of the metric e.g metric_name{foo="bar"}
+ * @param r_value A double representing the value of the sample
+ */
+taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value);
+
+/**
+ * @brief API PRIVATE Destroy the taos_metric_sample**
+ */
+int taos_metric_sample_destroy(taos_metric_sample_t *self);
+
+/**
+ * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor
+ */
+int taos_metric_sample_destroy_generic(void *);
+
+/**
+ * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor.
+ *
+ * This function ignores any errors.
+ */
+void taos_metric_sample_free_generic(void *gen);
+
+#endif // TAOS_METRIC_SAMPLE_I_H
diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h
new file mode 100644
index 0000000000..387512af9d
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_METRIC_SAMPLE_T_H
+#define TAOS_METRIC_SAMPLE_T_H
+
+#include "taos_metric_sample.h"
+#include "taos_metric_t.h"
+
+#if !defined(WINDOWS)
+#define C11_ATOMIC
+#else
+#define DOUBLE_ATOMIC
+#endif
+
+#ifdef C11_ATOMIC
+#include
+#endif
+
+struct taos_metric_sample {
+ taos_metric_type_t type; /**< type is the metric type for the sample */
+ char *l_value; /**< l_value is the full metric name and label set represeted as a string */
+#ifdef C11_ATOMIC
+ _Atomic double r_value; /**< r_value is the value of the metric sample */
+#else
+#ifdef DOUBLE_ATOMIC
+ double r_value; /**< r_value is the value of the metric sample */
+#else
+ int64_t r_value;
+#endif
+#endif
+};
+
+#endif // TAOS_METRIC_SAMPLE_T_H
diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h
new file mode 100644
index 0000000000..da237aa814
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_metric_t.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_METRIC_T_H
+#define TAOS_METRIC_T_H
+
+#include
+
+// Public
+#include "taos_metric.h"
+
+// Private
+#include "taos_map_i.h"
+#include "taos_map_t.h"
+#include "taos_metric_formatter_t.h"
+
+/**
+ * @brief API PRIVATE Contains metric type constants
+ */
+typedef enum taos_metric_type { TAOS_COUNTER, TAOS_GAUGE, TAOS_HISTOGRAM, TAOS_SUMMARY } taos_metric_type_t;
+
+/**
+ * @brief API PRIVATE Maps metric type constants to human readable string values
+ */
+extern char *taos_metric_type_map[4];
+
+/**
+ * @brief API PRIVATE An opaque struct to users containing metric metadata; one or more metric samples; and a metric
+ * formatter for locating metric samples and exporting metric data
+ */
+struct taos_metric {
+ taos_metric_type_t type; /**< metric_type The type of metric */
+ char *name; /**< name The name of the metric */
+ const char *help; /**< help The help output for the metric */
+ taos_map_t *samples; /**< samples Map comprised of samples for the given metric */
+ size_t label_key_count; /**< label_keys_count The count of labe_keys*/
+ taos_metric_formatter_t *formatter; /**< formatter The metric formatter */
+ pthread_rwlock_t *rwlock; /**< rwlock Required for locking on certain non-atomic operations */
+ const char **label_keys; /**< labels Array comprised of const char **/
+};
+
+#endif // TAOS_METRIC_T_H
diff --git a/source/libs/monitorfw/inc/taos_monitor_util_i.h b/source/libs/monitorfw/inc/taos_monitor_util_i.h
new file mode 100644
index 0000000000..fe072204a3
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_monitor_util_i.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_MONITOR_UTIL_I_H
+#define TAOS_MONITOR_UTIL_I_H
+
+#include
+
+void taos_monitor_split_str(char** arr, char* str, const char* del);
+int taos_monitor_count_occurrences(char *str, char *toSearch);
+void taos_monitor_strip(char *s);
+bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count);
+
+#endif // TAOS_MONITOR_UTIL_I_H
\ No newline at end of file
diff --git a/source/libs/monitorfw/inc/taos_string_builder_i.h b/source/libs/monitorfw/inc/taos_string_builder_i.h
new file mode 100644
index 0000000000..142ca020ba
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_string_builder_i.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_STRING_BUILDER_I_H
+#define TAOS_STRING_BUILDER_I_H
+
+#include
+
+#include "taos_string_builder_t.h"
+
+/**
+ * API PRIVATE
+ * @brief Constructor for taos_string_builder
+ */
+taos_string_builder_t *taos_string_builder_new(void);
+
+/**
+ * API PRIVATE
+ * @brief Destroys a taos_string_builder*
+ */
+int taos_string_builder_destroy(taos_string_builder_t *self);
+
+/**
+ * API PRIVATE
+ * @brief Adds a string
+ */
+int taos_string_builder_add_str(taos_string_builder_t *self, const char *str);
+
+/**
+ * API PRIVATE
+ * @brief Adds a char
+ */
+int taos_string_builder_add_char(taos_string_builder_t *self, char c);
+
+/**
+ * API PRIVATE
+ * @brief Clear the string
+ */
+int taos_string_builder_clear(taos_string_builder_t *self);
+
+/**
+ * API PRIVATE
+ * @brief Remove data from the end
+ */
+int taos_string_buillder_truncate(taos_string_builder_t *self, size_t len);
+
+/**
+ * API PRIVATE
+ * @brief Returns the length of the string
+ */
+size_t taos_string_builder_len(taos_string_builder_t *self);
+
+/**
+ * API PRIVATE
+ * @brief Returns a copy of the string. The returned string must be deallocated when no longer needed.
+ */
+char *taos_string_builder_dump(taos_string_builder_t *self);
+
+/**
+ * API PRIVATE
+ * @brief Getter for str member
+ */
+char *taos_string_builder_str(taos_string_builder_t *self);
+
+#endif // TAOS_STRING_BUILDER_I_H
diff --git a/source/libs/monitorfw/inc/taos_string_builder_t.h b/source/libs/monitorfw/inc/taos_string_builder_t.h
new file mode 100644
index 0000000000..edd3d574fa
--- /dev/null
+++ b/source/libs/monitorfw/inc/taos_string_builder_t.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TAOS_STRING_BUILDER_T_H
+#define TAOS_STRING_BUILDER_T_H
+
+struct taos_string_builder;
+/**
+ * @brief API PRIVATE A structure with functions responsible for building a string
+ */
+typedef struct taos_string_builder taos_string_builder_t;
+
+#endif // TAOS_STRING_BUILDER_T_H
diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c
new file mode 100644
index 0000000000..17d324462c
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_collector.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+// Public
+#include "taos_alloc.h"
+#include "taos_collector.h"
+#include "taos_collector_registry.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_collector_t.h"
+#include "taos_log.h"
+#include "taos_map_i.h"
+#include "taos_metric_i.h"
+#include "taos_string_builder_i.h"
+
+taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self->metrics; }
+
+taos_collector_t *taos_collector_new(const char *name) {
+ int r = 0;
+ taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t));
+ self->name = taos_strdup(name);
+ self->metrics = taos_map_new();
+ if (self->metrics == NULL) {
+ taos_collector_destroy(self);
+ return NULL;
+ }
+ r = taos_map_set_free_value_fn(self->metrics, &taos_metric_free_generic);
+ if (r) {
+ taos_collector_destroy(self);
+ return NULL;
+ }
+ self->collect_fn = &taos_collector_default_collect;
+ self->string_builder = taos_string_builder_new();
+ if (self->string_builder == NULL) {
+ taos_collector_destroy(self);
+ return NULL;
+ }
+ self->proc_limits_file_path = NULL;
+ self->proc_stat_file_path = NULL;
+ return self;
+}
+
+int taos_collector_destroy(taos_collector_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+
+ int r = 0;
+ int ret = 0;
+
+ r = taos_map_destroy(self->metrics);
+ if (r) ret = r;
+ self->metrics = NULL;
+
+ r = taos_string_builder_destroy(self->string_builder);
+ if (r) ret = r;
+ self->string_builder = NULL;
+
+ taos_free((char *)self->name);
+ self->name = NULL;
+ taos_free(self);
+ self = NULL;
+
+ return ret;
+}
+
+int taos_collector_destroy_generic(void *gen) {
+ int r = 0;
+ taos_collector_t *self = (taos_collector_t *)gen;
+ r = taos_collector_destroy(self);
+ self = NULL;
+ return r;
+}
+
+void taos_collector_free_generic(void *gen) {
+ taos_collector_t *self = (taos_collector_t *)gen;
+ taos_collector_destroy(self);
+}
+
+int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ self->collect_fn = fn;
+ return 0;
+}
+
+int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (taos_map_get(self->metrics, metric->name) != NULL) {
+ TAOS_LOG("metric already found in collector");
+ return 1;
+ }
+ return taos_map_set(self->metrics, metric->name, metric);
+}
+
+int taos_collector_remove_metric(taos_collector_t *self, const char* key){
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ return taos_map_delete(self->metrics, key);
+}
+
+taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name){
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return NULL;
+ return taos_map_get(self->metrics, metric_name);
+}
\ No newline at end of file
diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c
new file mode 100644
index 0000000000..55b08775b4
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_collector_registry.c
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include
+#include
+
+// Public
+#include "taos_alloc.h"
+#include "taos_collector.h"
+#include "taos_collector_registry.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_collector_registry_t.h"
+#include "taos_collector_t.h"
+#include "taos_errors.h"
+#include "taos_log.h"
+#include "taos_map_i.h"
+#include "taos_metric_formatter_i.h"
+#include "taos_metric_i.h"
+#include "taos_metric_t.h"
+#include "taos_string_builder_i.h"
+#include "taos_metric_formatter_custom_i.h"
+
+#define ALLOW_FORBID_FUNC
+#include "tjson.h"
+
+taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT;
+
+taos_collector_registry_t *taos_collector_registry_new(const char *name) {
+ int r = 0;
+
+ taos_collector_registry_t *self = (taos_collector_registry_t *)taos_malloc(sizeof(taos_collector_registry_t));
+
+ self->disable_process_metrics = false;
+
+ self->name = taos_strdup(name);
+ self->collectors = taos_map_new();
+ taos_map_set_free_value_fn(self->collectors, &taos_collector_free_generic);
+ taos_map_set(self->collectors, "default", taos_collector_new("default"));
+
+ self->metric_formatter = taos_metric_formatter_new();
+ self->string_builder = taos_string_builder_new();
+ self->string_builder_batch = taos_string_builder_new();
+ self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t));
+ r = pthread_rwlock_init(self->lock, NULL);
+ if (r) {
+ TAOS_LOG("failed to initialize rwlock");
+ return NULL;
+ }
+ return self;
+}
+
+int taos_collector_registry_default_init(void) {
+ if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0;
+
+ TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default");
+ //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) {
+ // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT);
+ //}
+ return 1;
+}
+
+int taos_collector_registry_destroy(taos_collector_registry_t *self) {
+ if (self == NULL) return 0;
+
+ int r = 0;
+ int ret = 0;
+
+ r = taos_map_destroy(self->collectors);
+ self->collectors = NULL;
+ if (r) ret = r;
+
+ r = taos_metric_formatter_destroy(self->metric_formatter);
+ self->metric_formatter = NULL;
+ if (r) ret = r;
+
+ r = taos_string_builder_destroy(self->string_builder);
+ self->string_builder = NULL;
+ if (r) ret = r;
+
+ r = taos_string_builder_destroy(self->string_builder_batch);
+ self->string_builder_batch = NULL;
+ if (r) ret = r;
+
+ r = pthread_rwlock_destroy(self->lock);
+ taos_free(self->lock);
+ self->lock = NULL;
+ if (r) ret = r;
+
+ taos_free((char *)self->name);
+ self->name = NULL;
+
+ taos_free(self);
+ self = NULL;
+
+ return ret;
+}
+
+int taos_collector_registry_register_metric(taos_metric_t *metric) {
+ TAOS_ASSERT(metric != NULL);
+
+ taos_collector_t *default_collector =
+ (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default");
+
+ if (default_collector == NULL) {
+ return 1;
+ }
+
+ return taos_collector_add_metric(default_collector, metric);
+}
+
+int taos_collector_registry_deregister_metric(const char *key) {
+ TAOS_ASSERT(metric != NULL);
+
+ taos_collector_t *default_collector =
+ (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default");
+
+ if (default_collector == NULL) {
+ return 1;
+ }
+
+ return taos_collector_remove_metric(default_collector, key);
+}
+
+taos_metric_t *taos_collector_registry_get_metric(char* metric_name){
+ TAOS_ASSERT(metric != NULL);
+
+ taos_collector_t *default_collector =
+ (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default");
+
+ if (default_collector == NULL) {
+ return NULL;
+ }
+
+ return taos_collector_get_metric(default_collector, metric_name);
+}
+
+taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) {
+ int err = taos_collector_registry_register_metric(metric);
+ if (err != 0) {
+ //exit(err);
+ return NULL;
+ }
+ return metric;
+}
+
+int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = pthread_rwlock_wrlock(self->lock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR);
+ return 1;
+ }
+ if (taos_map_get(self->collectors, collector->name) != NULL) {
+ TAOS_LOG("the given taos_collector_t* is already registered");
+ int rr = pthread_rwlock_unlock(self->lock);
+ if (rr) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return rr;
+ } else {
+ return 1;
+ }
+ }
+ r = taos_map_set(self->collectors, collector->name, collector);
+ if (r) {
+ int rr = pthread_rwlock_unlock(self->lock);
+ if (rr) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return rr;
+ } else {
+ return r;
+ }
+ }
+ r = pthread_rwlock_unlock(self->lock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return 1;
+ }
+ return 0;
+}
+
+int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name) {
+ regex_t r;
+ int ret = 0;
+ ret = regcomp(&r, "^[a-zA-Z_:][a-zA-Z0-9_:]*$", REG_EXTENDED);
+ if (ret) {
+ TAOS_LOG(TAOS_REGEX_REGCOMP_ERROR);
+ regfree(&r);
+ return ret;
+ }
+
+ ret = regexec(&r, metric_name, 0, NULL, 0);
+ if (ret) {
+ TAOS_LOG(TAOS_REGEX_REGEXEC_ERROR);
+ regfree(&r);
+ return ret;
+ }
+ regfree(&r);
+ return 0;
+}
+
+const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format) {
+ taos_metric_formatter_clear(self->metric_formatter);
+ taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts, format);
+ char *out = taos_metric_formatter_dump(self->metric_formatter);
+
+ int r = 0;
+ r = taos_string_builder_add_str(self->string_builder_batch, out);
+ if (r) return NULL;
+ taos_free(out);
+
+ return taos_string_builder_str(self->string_builder_batch);
+}
+
+int taos_collector_registry_clear_batch(taos_collector_registry_t *self){
+ return taos_string_builder_clear(self->string_builder_batch);
+}
+
+const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str) {
+ taos_metric_formatter_clear(self->metric_formatter);
+
+ SJson* pJson = tjsonCreateArray();
+ SJson* item = tjsonCreateObject();
+ tjsonAddItemToArray(pJson, item);
+ tjsonAddStringToObject(item, "ts", ts);
+ tjsonAddDoubleToObject(item, "protocol", 2);
+ SJson* array = tjsonCreateArray();
+ tjsonAddItemToObject(item, "tables", array);
+
+ if(taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array) != 0){
+ TAOS_LOG("failed to load metrics");
+ tjsonDelete(pJson);
+ return NULL;
+ }
+
+ if(tjsonGetArraySize(array) == 0){
+ tjsonDelete(pJson);
+ return NULL;
+ }
+
+ //caller free this
+ //generate prom protocol for debug
+ if(prom_str != NULL){
+ *prom_str = taos_metric_formatter_dump(self->metric_formatter);
+ }
+
+ //add this result to batch cache, format in batch cache is {},{}
+ int r = 0;
+ char* old_str = taos_string_builder_str(self->string_builder_batch);
+ if(old_str[0] != '\0'){
+ r = taos_string_builder_add_str(self->string_builder_batch, ",");
+ if (r) goto _OVER;
+ }
+ char * item_str = tjsonToString(item);
+ r = taos_string_builder_add_str(self->string_builder_batch, item_str);
+ taos_free(item_str);
+ if (r) goto _OVER;;
+
+ //generate final array format result, ie, add [] to str in batch cache
+ taos_string_builder_t* tmp_builder = taos_string_builder_new();
+
+ r = taos_string_builder_add_str(tmp_builder, "[");
+ if (r) goto _OVER;;
+
+ r = taos_string_builder_add_str(tmp_builder, taos_string_builder_str(self->string_builder_batch));
+ if (r) goto _OVER;;
+
+ r = taos_string_builder_add_str(tmp_builder, "]");
+ if (r) goto _OVER;;
+
+ //caller free this
+ char *data = taos_string_builder_dump(tmp_builder);
+ if (data == NULL) goto _OVER;;
+ r = taos_string_builder_clear(tmp_builder);
+ if (r) goto _OVER;;
+
+ r = taos_string_builder_destroy(tmp_builder);
+ tmp_builder = NULL;
+ if (r) goto _OVER;;
+
+ tjsonDelete(pJson);
+ return data;
+
+_OVER:
+ tjsonDelete(pJson);
+
+ return NULL;
+}
diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c
new file mode 100644
index 0000000000..d522411b2b
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_counter.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// Public
+#include "taos_counter.h"
+
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_errors.h"
+#include "taos_log.h"
+#include "taos_metric_i.h"
+#include "taos_metric_sample_i.h"
+#include "taos_metric_sample_t.h"
+#include "taos_metric_t.h"
+
+taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) {
+ return (taos_counter_t *)taos_metric_new(TAOS_COUNTER, name, help, label_key_count, label_keys);
+}
+
+int taos_counter_destroy(taos_counter_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+ int r = 0;
+ r = taos_metric_destroy(self);
+ self = NULL;
+ return r;
+}
+
+int taos_counter_inc(taos_counter_t *self, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_COUNTER) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_add(sample, 1.0);
+}
+
+int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_COUNTER) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_add(sample, r_value);
+}
diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c
new file mode 100644
index 0000000000..74d2665194
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_gauge.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// Public
+#include "taos_gauge.h"
+
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_errors.h"
+#include "taos_log.h"
+#include "taos_metric_i.h"
+#include "taos_metric_sample_i.h"
+#include "taos_metric_sample_t.h"
+#include "taos_metric_t.h"
+
+taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) {
+ return (taos_gauge_t *)taos_metric_new(TAOS_GAUGE, name, help, label_key_count, label_keys);
+}
+
+int taos_gauge_destroy(taos_gauge_t *self) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ r = taos_metric_destroy(self);
+ self = NULL;
+ return r;
+}
+
+int taos_gauge_inc(taos_gauge_t *self, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_add(sample, 1.0);
+}
+
+int taos_gauge_dec(taos_gauge_t *self, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_sub(sample, 1.0);
+}
+
+int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_add(sample, r_value);
+}
+
+int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_sub(sample, r_value);
+}
+
+int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+ taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values);
+ if (sample == NULL) return 1;
+ return taos_metric_sample_set(sample, r_value);
+}
diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c
new file mode 100644
index 0000000000..675400a6fa
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_linked_list.c
@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_linked_list_i.h"
+#include "taos_linked_list_t.h"
+#include "taos_log.h"
+
+taos_linked_list_t *taos_linked_list_new(void) {
+ taos_linked_list_t *self = (taos_linked_list_t *)taos_malloc(sizeof(taos_linked_list_t));
+ self->head = NULL;
+ self->tail = NULL;
+ self->free_fn = NULL;
+ self->compare_fn = NULL;
+ self->size = 0;
+ return self;
+}
+
+int taos_linked_list_purge(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ taos_linked_list_node_t *node = self->head;
+ while (node != NULL) {
+ taos_linked_list_node_t *next = node->next;
+ if (node->item != NULL) {
+ if (self->free_fn) {
+ (*self->free_fn)(node->item);
+ } else {
+ taos_free(node->item);
+ }
+ }
+ taos_free(node);
+ node = NULL;
+ node = next;
+ }
+ self->head = NULL;
+ self->tail = NULL;
+ self->size = 0;
+ return 0;
+}
+
+int taos_linked_list_destroy(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ int ret = 0;
+
+ r = taos_linked_list_purge(self);
+ if (r) ret = r;
+ taos_free(self);
+ self = NULL;
+ return ret;
+}
+
+void *taos_linked_list_first(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self->head) {
+ return self->head->item;
+ } else {
+ return NULL;
+ }
+}
+
+void *taos_linked_list_last(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self->tail) {
+ return self->tail->item;
+ } else {
+ return NULL;
+ }
+}
+
+int taos_linked_list_append(taos_linked_list_t *self, void *item) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t));
+
+ node->item = item;
+ if (self->tail) {
+ self->tail->next = node;
+ } else {
+ self->head = node;
+ }
+ self->tail = node;
+ node->next = NULL;
+ self->size++;
+ return 0;
+}
+
+int taos_linked_list_push(taos_linked_list_t *self, void *item) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t));
+
+ node->item = item;
+ node->next = self->head;
+ self->head = node;
+ if (self->tail == NULL) {
+ self->tail = node;
+ }
+ self->size++;
+ return 0;
+}
+
+void *taos_linked_list_pop(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return NULL;
+ taos_linked_list_node_t *node = self->head;
+ void *item = NULL;
+ if (node != NULL) {
+ item = node->item;
+ self->head = node->next;
+ if (self->tail == node) {
+ self->tail = NULL;
+ }
+ if (node->item != NULL) {
+ if (self->free_fn) {
+ (*self->free_fn)(node->item);
+ } else {
+ taos_free(node->item);
+ }
+ }
+ node->item = NULL;
+ node = NULL;
+ self->size--;
+ }
+ return item;
+}
+
+int taos_linked_list_remove(taos_linked_list_t *self, void *item) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ taos_linked_list_node_t *node;
+ taos_linked_list_node_t *prev_node = NULL;
+#ifdef TAOS_LOG_ENABLE
+ int32_t count = 0;
+ char tmp[200] = {0};
+
+ count = 0;
+ for (node = self->head; node != NULL; node = node->next) {
+ count++;
+ }
+ sprintf(tmp, "list count:%d", count);
+ TAOS_LOG(tmp);
+#endif
+
+ // Locate the node
+#ifdef TAOS_LOG_ENABLE
+ count = 0;
+#endif
+ for (node = self->head; node != NULL; node = node->next) {
+#ifdef TAOS_LOG_ENABLE
+ count++;
+#endif
+ if (self->compare_fn) {
+ if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) {
+ break;
+ }
+ } else {
+ if (node->item == item) {
+ break;
+ }
+ }
+ prev_node = node;
+ }
+
+#ifdef TAOS_LOG_ENABLE
+ sprintf(tmp, "remove item:%d", count);
+ TAOS_LOG(tmp);
+#endif
+
+ if (node == NULL) return 0;
+
+ if (prev_node) {
+ prev_node->next = node->next;
+ } else {
+ self->head = node->next;
+ }
+ if (node->next == NULL) {
+ self->tail = prev_node;
+ }
+
+ if (node->item != NULL) {
+ if (self->free_fn) {
+ (*self->free_fn)(node->item);
+ } else {
+ taos_free(node->item);
+ }
+ }
+
+ node->item = NULL;
+ taos_free(node);
+ node = NULL;
+ self->size--;
+
+#ifdef TAOS_LOG_ENABLE
+ count = 0;
+ for (node = self->head; node != NULL; node = node->next) {
+ count++;
+ }
+
+ sprintf(tmp, "list count:%d", count);
+ TAOS_LOG(tmp);
+#endif
+
+ return 0;
+}
+
+taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *item_b) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (self->compare_fn) {
+ return (*self->compare_fn)(item_a, item_b);
+ } else {
+ return strcmp(item_a, item_b);
+ }
+}
+
+size_t taos_linked_list_size(taos_linked_list_t *self) {
+ TAOS_ASSERT(self != NULL);
+ return self->size;
+}
+
+int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ self->free_fn = free_fn;
+ return 0;
+}
+
+int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ self->compare_fn = compare_fn;
+ return 0;
+}
+
+void taos_linked_list_no_op_free(void *item) {}
diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c
new file mode 100644
index 0000000000..8f0b83884e
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_map.c
@@ -0,0 +1,422 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+#include
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_errors.h"
+#include "taos_linked_list_i.h"
+#include "taos_linked_list_t.h"
+#include "taos_log.h"
+#include "taos_map_i.h"
+#include "taos_map_t.h"
+
+#define TAOS_MAP_INITIAL_SIZE 32
+
+static void destroy_map_node_value_no_op(void *value) {}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// taos_map_node
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn) {
+ taos_map_node_t *self = taos_malloc(sizeof(taos_map_node_t));
+ self->key = taos_strdup(key);
+ self->value = value;
+ self->free_value_fn = free_value_fn;
+ return self;
+}
+
+int taos_map_node_destroy(taos_map_node_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+ taos_free((void *)self->key);
+ self->key = NULL;
+ if (self->value != NULL) (*self->free_value_fn)(self->value);
+ self->value = NULL;
+ taos_free(self);
+ self = NULL;
+ return 0;
+}
+
+void taos_map_node_free(void *item) {
+ taos_map_node_t *map_node = (taos_map_node_t *)item;
+ taos_map_node_destroy(map_node);
+}
+
+taos_linked_list_compare_t taos_map_node_compare(void *item_a, void *item_b) {
+ taos_map_node_t *map_node_a = (taos_map_node_t *)item_a;
+ taos_map_node_t *map_node_b = (taos_map_node_t *)item_b;
+
+ return strcmp(map_node_a->key, map_node_b->key);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// taos_map
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+taos_map_t *taos_map_new() {
+ int r = 0;
+
+ taos_map_t *self = (taos_map_t *)taos_malloc(sizeof(taos_map_t));
+ self->size = 0;
+ self->max_size = TAOS_MAP_INITIAL_SIZE;
+
+ self->keys = taos_linked_list_new();
+ if (self->keys == NULL) return NULL;
+
+ // These each key will be allocated once by taos_map_node_new and used here as well to save memory. With that said
+ // we will only have to deallocate each key once. That will happen on taos_map_node_destroy.
+ r = taos_linked_list_set_free_fn(self->keys, taos_linked_list_no_op_free);
+ if (r) {
+ taos_map_destroy(self);
+ return NULL;
+ }
+
+ self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size);
+ self->free_value_fn = destroy_map_node_value_no_op;
+
+ for (int i = 0; i < self->max_size; i++) {
+ self->addrs[i] = taos_linked_list_new();
+ r = taos_linked_list_set_free_fn(self->addrs[i], taos_map_node_free);
+ if (r) {
+ taos_map_destroy(self);
+ return NULL;
+ }
+ r = taos_linked_list_set_compare_fn(self->addrs[i], taos_map_node_compare);
+ if (r) {
+ taos_map_destroy(self);
+ return NULL;
+ }
+ }
+
+ self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t));
+ r = pthread_rwlock_init(self->rwlock, NULL);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR);
+ taos_map_destroy(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+int taos_map_destroy(taos_map_t *self) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ int ret = 0;
+
+ r = taos_linked_list_destroy(self->keys);
+ if (r) ret = r;
+ self->keys = NULL;
+
+ for (size_t i = 0; i < self->max_size; i++) {
+ r = taos_linked_list_destroy(self->addrs[i]);
+ if (r) ret = r;
+ self->addrs[i] = NULL;
+ }
+ taos_free(self->addrs);
+ self->addrs = NULL;
+
+ r = pthread_rwlock_destroy(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR)
+ ret = r;
+ }
+
+ taos_free(self->rwlock);
+ self->rwlock = NULL;
+ taos_free(self);
+ self = NULL;
+
+ return ret;
+}
+
+static size_t taos_map_get_index_internal(const char *key, size_t *size, size_t *max_size) {
+ size_t index;
+ size_t a = 31415, b = 27183;
+ for (index = 0; *key != '\0'; key++, a = a * b % (*max_size - 1)) {
+ index = (a * index + *key) % *max_size;
+ }
+ return index;
+}
+
+/**
+ * @brief API PRIVATE hash function that returns an array index from the given key and taos_map.
+ *
+ * The algorithm is based off of Horner's method. In a simpler version, you set the return value to 0. Next, for each
+ * character in the string, you add the integer value of the current character to the product of the prime number and
+ * the current return value, set the result to the return value, then finally return the return value.
+ *
+ * In this version of the algorithm, we attempt to achieve a probabily of key to index conversion collisions to
+ * 1/M (with M being the max_size of the map). This optimizes dispersion and consequently, evens out the performance
+ * for gets and sets for each item. Instead of using a fixed prime number, we generate a coefficient for each iteration
+ * through the loop.
+ *
+ * Reference:
+ * * Algorithms in C: Third Edition by Robert Sedgewick, p579
+ */
+size_t taos_map_get_index(taos_map_t *self, const char *key) {
+ return taos_map_get_index_internal(key, &self->size, &self->max_size);
+}
+
+static void *taos_map_get_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys,
+ taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) {
+ size_t index = taos_map_get_index_internal(key, size, max_size);
+ taos_linked_list_t *list = addrs[index];
+ taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn);
+
+ for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) {
+ taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item;
+ taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node);
+ if (result == TAOS_EQUAL) {
+ taos_map_node_destroy(temp_map_node);
+ temp_map_node = NULL;
+ return current_map_node->value;
+ }
+ }
+ taos_map_node_destroy(temp_map_node);
+ temp_map_node = NULL;
+ return NULL;
+}
+
+void *taos_map_get(taos_map_t *self, const char *key) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ r = pthread_rwlock_wrlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR);
+ NULL;
+ }
+ void *payload =
+ taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn);
+ r = pthread_rwlock_unlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return NULL;
+ }
+ return payload;
+}
+
+void *taos_map_get_withoutlock(taos_map_t *self, const char *key) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ void *payload =
+ taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn);
+ return payload;
+}
+
+static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys,
+ taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn,
+ bool destroy_current_value) {
+ taos_map_node_t *map_node = taos_map_node_new(key, value, free_value_fn);
+ if (map_node == NULL) return 1;
+
+ size_t index = taos_map_get_index_internal(key, size, max_size);
+ taos_linked_list_t *list = addrs[index];
+ for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) {
+ taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item;
+ taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, map_node);
+ if (result == TAOS_EQUAL) {
+ if (destroy_current_value) {
+ free_value_fn(current_map_node->value);
+ current_map_node->value = NULL;
+ }
+ taos_free((char *)current_map_node->key);
+ current_map_node->key = NULL;
+ taos_free(current_map_node);
+ current_map_node = NULL;
+ current_node->item = map_node;
+ return 0;
+ }
+ }
+ taos_linked_list_append(list, map_node);
+ taos_linked_list_append(keys, (char *)map_node->key);
+ (*size)++;
+ return 0;
+}
+
+int taos_map_ensure_space(taos_map_t *self) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+
+ if (self->size <= self->max_size / 2) {
+ return 0;
+ }
+
+ // Increase the max size
+ size_t new_max = self->max_size * 2;
+ size_t new_size = 0;
+
+ // Create a new list of keys
+ taos_linked_list_t *new_keys = taos_linked_list_new();
+ if (new_keys == NULL) return 1;
+
+ r = taos_linked_list_set_free_fn(new_keys, taos_linked_list_no_op_free);
+ if (r) return r;
+
+ // Create a new array of addrs
+ taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max);
+
+ // Initialize the new array
+ for (int i = 0; i < new_max; i++) {
+ new_addrs[i] = taos_linked_list_new();
+ r = taos_linked_list_set_free_fn(new_addrs[i], taos_map_node_free);
+ if (r) return r;
+ r = taos_linked_list_set_compare_fn(new_addrs[i], taos_map_node_compare);
+ if (r) return r;
+ }
+
+ // Iterate through each linked-list at each memory region in the map's backbone
+ for (int i = 0; i < self->max_size; i++) {
+ // Create a new map node for each node in the linked list and insert it into the new map. Afterwards, deallocate
+ // the old map node
+ taos_linked_list_t *list = self->addrs[i];
+ taos_linked_list_node_t *current_node = list->head;
+ while (current_node != NULL) {
+ taos_map_node_t *map_node = (taos_map_node_t *)current_node->item;
+ r = taos_map_set_internal(map_node->key, map_node->value, &new_size, &new_max, new_keys, new_addrs,
+ self->free_value_fn, false);
+ if (r) return r;
+
+ taos_linked_list_node_t *next = current_node->next;
+ taos_free(current_node);
+ current_node = NULL;
+ taos_free((void *)map_node->key);
+ map_node->key = NULL;
+ taos_free(map_node);
+ map_node = NULL;
+ current_node = next;
+ }
+ // We're done deallocating each map node in the linked list, so deallocate the linked-list object
+ taos_free(self->addrs[i]);
+ self->addrs[i] = NULL;
+ }
+ // Destroy the collection of keys in the map
+ taos_linked_list_destroy(self->keys);
+ self->keys = NULL;
+
+ // Deallocate the backbone of the map
+ taos_free(self->addrs);
+ self->addrs = NULL;
+
+ // Update the members of the current map
+ self->size = new_size;
+ self->max_size = new_max;
+ self->keys = new_keys;
+ self->addrs = new_addrs;
+
+ return 0;
+}
+
+int taos_map_set(taos_map_t *self, const char *key, void *value) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ r = pthread_rwlock_wrlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR);
+ return r;
+ }
+
+ r = taos_map_ensure_space(self);
+ if (r) {
+ int rr = 0;
+ rr = pthread_rwlock_unlock(self->rwlock);
+ if (rr) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return rr;
+ } else {
+ return r;
+ }
+ }
+ r = taos_map_set_internal(key, value, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn,
+ true);
+ if (r) {
+ int rr = 0;
+ rr = pthread_rwlock_unlock(self->rwlock);
+ if (rr) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ return rr;
+ } else {
+ return r;
+ }
+ }
+ r = pthread_rwlock_unlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ }
+ return r;
+}
+
+static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys,
+ taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) {
+ int r = 0;
+ size_t index = taos_map_get_index_internal(key, size, max_size);
+ taos_linked_list_t *list = addrs[index];
+ taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn);
+
+ for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) {
+ taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item;
+ taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node);
+ if (result == TAOS_EQUAL) {
+ r = taos_linked_list_remove(keys, (char*)current_map_node->key);
+ if (r) return r;
+
+ r = taos_linked_list_remove(list, current_node->item);
+ if (r) return r;
+
+ (*size)--;
+ break;
+ }
+ }
+ r = taos_map_node_destroy(temp_map_node);
+ temp_map_node = NULL;
+ return r;
+}
+
+int taos_map_delete(taos_map_t *self, const char *key) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ int ret = 0;
+ r = pthread_rwlock_wrlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR);
+ ret = r;
+ }
+ r = taos_map_delete_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn);
+ if (r) ret = r;
+ r = pthread_rwlock_unlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
+ ret = r;
+ }
+ return ret;
+}
+
+int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn) {
+ TAOS_ASSERT(self != NULL);
+ self->free_value_fn = free_value_fn;
+ return 0;
+}
+
+size_t taos_map_size(taos_map_t *self) {
+ TAOS_ASSERT(self != NULL);
+ return self->size;
+}
diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c
new file mode 100644
index 0000000000..4e9af35f34
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_metric.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_errors.h"
+#include "taos_log.h"
+#include "taos_map_i.h"
+#include "taos_metric_formatter_i.h"
+#include "taos_metric_i.h"
+#include "taos_metric_sample_i.h"
+
+char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"};
+
+taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, const char *help,
+ size_t label_key_count, const char **label_keys) {
+ int r = 0;
+ taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t));
+ self->type = metric_type;
+ int len = strlen(name) + 1;
+ self->name = taos_malloc(len);
+ memset(self->name, 0, len);
+ strcpy(self->name, name);
+ //self->name = name;
+ self->help = help;
+
+ const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count);
+
+ for (int i = 0; i < label_key_count; i++) {
+ if (strcmp(label_keys[i], "le") == 0) {
+ TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME);
+ taos_metric_destroy(self);
+ return NULL;
+ }
+ if (strcmp(label_keys[i], "quantile") == 0) {
+ TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME);
+ taos_metric_destroy(self);
+ return NULL;
+ }
+ k[i] = taos_strdup(label_keys[i]);
+ }
+ self->label_keys = k;
+ self->label_key_count = label_key_count;
+ self->samples = taos_map_new();
+
+ if (metric_type == TAOS_HISTOGRAM) {
+
+ } else {
+ r = taos_map_set_free_value_fn(self->samples, &taos_metric_sample_free_generic);
+ if (r) {
+ taos_metric_destroy(self);
+ return NULL;
+ }
+ }
+
+ self->formatter = taos_metric_formatter_new();
+ if (self->formatter == NULL) {
+ taos_metric_destroy(self);
+ return NULL;
+ }
+ self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t));
+ r = pthread_rwlock_init(self->rwlock, NULL);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR);
+ return NULL;
+ }
+ return self;
+}
+
+int taos_metric_destroy(taos_metric_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+
+ int r = 0;
+ int ret = 0;
+
+ r = taos_map_destroy(self->samples);
+ self->samples = NULL;
+ if (r) ret = r;
+
+ r = taos_metric_formatter_destroy(self->formatter);
+ self->formatter = NULL;
+ if (r) ret = r;
+
+ r = pthread_rwlock_destroy(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR);
+ ret = r;
+ }
+
+ taos_free(self->rwlock);
+ self->rwlock = NULL;
+
+ for (int i = 0; i < self->label_key_count; i++) {
+ taos_free((void *)self->label_keys[i]);
+ self->label_keys[i] = NULL;
+ }
+ taos_free(self->label_keys);
+ self->label_keys = NULL;
+
+ taos_free(self->name);
+ self->name = NULL;
+
+ taos_free(self);
+ self = NULL;
+
+ return ret;
+}
+
+int taos_metric_destroy_generic(void *item) {
+ int r = 0;
+ taos_metric_t *self = (taos_metric_t *)item;
+ r = taos_metric_destroy(self);
+ self = NULL;
+ return r;
+}
+
+void taos_metric_free_generic(void *item) {
+ taos_metric_t *self = (taos_metric_t *)item;
+ taos_metric_destroy(self);
+}
+
+taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ r = pthread_rwlock_wrlock(self->rwlock);
+ if (r) {
+ TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR);
+ return NULL;
+ }
+
+#define TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK() \
+ r = pthread_rwlock_unlock(self->rwlock); \
+ if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); \
+ return NULL;
+
+ // Get l_value
+ r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys,
+ label_values);
+ if (r) {
+ TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
+ }
+
+ // This must be freed before returning
+ const char *l_value = taos_metric_formatter_dump(self->formatter);
+ if (l_value == NULL) {
+ TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
+ }
+
+ // Get sample
+ taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(self->samples, l_value);
+ if (sample == NULL) {
+ sample = taos_metric_sample_new(self->type, l_value, 0.0);
+ r = taos_map_set(self->samples, l_value, sample);
+ if (r) {
+ TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
+ }
+ }
+ pthread_rwlock_unlock(self->rwlock);
+ taos_free((void *)l_value);
+ return sample;
+}
+
diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c
new file mode 100644
index 0000000000..a9f35c3e8d
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_metric_formatter.c
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_collector_t.h"
+#include "taos_linked_list_t.h"
+#include "taos_map_i.h"
+#include "taos_metric_formatter_i.h"
+#include "taos_metric_sample_t.h"
+#include "taos_metric_t.h"
+#include "taos_string_builder_i.h"
+
+
+taos_metric_formatter_t *taos_metric_formatter_new() {
+ taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t));
+ self->string_builder = taos_string_builder_new();
+ if (self->string_builder == NULL) {
+ taos_metric_formatter_destroy(self);
+ return NULL;
+ }
+ self->err_builder = taos_string_builder_new();
+ if (self->err_builder == NULL) {
+ taos_metric_formatter_destroy(self);
+ return NULL;
+ }
+ return self;
+}
+
+int taos_metric_formatter_destroy(taos_metric_formatter_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+
+ int r = 0;
+ int ret = 0;
+
+ r = taos_string_builder_destroy(self->string_builder);
+ self->string_builder = NULL;
+ if (r) ret = r;
+
+ r = taos_string_builder_destroy(self->err_builder);
+ self->err_builder = NULL;
+ if (r) ret = r;
+
+ taos_free(self);
+ self = NULL;
+ return ret;
+}
+
+int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = taos_string_builder_add_str(self->string_builder, "# HELP ");
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, name);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, ' ');
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, help);
+ if (r) return r;
+
+ return taos_string_builder_add_char(self->string_builder, '\n');
+}
+
+int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = taos_string_builder_add_str(self->string_builder, "# TYPE ");
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, name);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, ' ');
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, taos_metric_type_map[metric_type]);
+ if (r) return r;
+
+ return taos_string_builder_add_char(self->string_builder, '\n');
+}
+
+int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix,
+ size_t label_count, const char **label_keys, const char **label_values) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = taos_string_builder_add_str(self->string_builder, name);
+ if (r) return r;
+
+ if (suffix != NULL) {
+ r = taos_string_builder_add_char(self->string_builder, '_');
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, suffix);
+ if (r) return r;
+ }
+
+ if (label_count == 0) return 0;
+
+ for (int i = 0; i < label_count; i++) {
+ if (i == 0) {
+ r = taos_string_builder_add_char(self->string_builder, '{');
+ if (r) return r;
+ }
+ r = taos_string_builder_add_str(self->string_builder, (const char *)label_keys[i]);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, '=');
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, '"');
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, (const char *)label_values[i]);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, '"');
+ if (r) return r;
+
+ if (i == label_count - 1) {
+ r = taos_string_builder_add_char(self->string_builder, '}');
+ if (r) return r;
+ } else {
+ r = taos_string_builder_add_char(self->string_builder, ',');
+ if (r) return r;
+ }
+ }
+ return 0;
+}
+
+int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample,
+ char *ts, char *format) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = taos_string_builder_add_str(self->string_builder, sample->l_value);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, ' ');
+ if (r) return r;
+
+ char buffer[50];
+ sprintf(buffer, format, sample->r_value);
+ r = taos_string_builder_add_str(self->string_builder, buffer);
+ if (r) return r;
+
+ r = taos_string_builder_add_char(self->string_builder, ' ');
+ if (r) return r;
+
+ r = taos_string_builder_add_str(self->string_builder, ts);
+ if (r) return r;
+
+ //taos_metric_sample_set(sample, 0);
+
+ return taos_string_builder_add_char(self->string_builder, '\n');
+}
+
+int taos_metric_formatter_clear(taos_metric_formatter_t *self) {
+ TAOS_ASSERT(self != NULL);
+ return taos_string_builder_clear(self->string_builder);
+}
+
+char *taos_metric_formatter_dump(taos_metric_formatter_t *self) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ if (self == NULL) return NULL;
+ char *data = taos_string_builder_dump(self->string_builder);
+ if (data == NULL) return NULL;
+ r = taos_string_builder_clear(self->string_builder);
+ if (r) {
+ taos_free(data);
+ return NULL;
+ }
+ return data;
+}
+
+int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ r = taos_metric_formatter_load_help(self, metric->name, metric->help);
+ if (r) return r;
+
+ r = taos_metric_formatter_load_type(self, metric->name, metric->type);
+ if (r) return r;
+
+ for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *key = (const char *)current_node->item;
+ if (metric->type == TAOS_HISTOGRAM) {
+
+ } else {
+ taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key);
+ if (sample == NULL) return 1;
+ r = taos_metric_formatter_load_sample(self, sample, ts, format);
+ if (r) return r;
+ }
+ }
+ return taos_string_builder_add_char(self->string_builder, '\n');
+}
+
+int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+ for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *collector_name = (const char *)current_node->item;
+ taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name);
+ if (collector == NULL) return 1;
+
+ taos_map_t *metrics = collector->collect_fn(collector);
+ if (metrics == NULL) return 1;
+
+ for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *metric_name = (const char *)current_node->item;
+ taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name);
+ if (metric == NULL) return 1;
+ r = taos_metric_formatter_load_metric(self, metric, ts, format);
+ if (r) return r;
+ }
+ }
+ return r;
+}
diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c
new file mode 100644
index 0000000000..3b1318dfc0
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define ALLOW_FORBID_FUNC
+
+#include
+#include "taos_metric_formatter_i.h"
+#include "taos_metric_sample_t.h"
+#include "tjson.h"
+#include "taos_monitor_util_i.h"
+#include "taos_assert.h"
+#include "tdef.h"
+#include "taos_collector_t.h"
+#include "taos_log.h"
+
+int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample,
+ char *ts, char *format, char *metricName, int32_t metric_type,
+ SJson *arrayMetricGroups) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ char* start = strstr(sample->l_value, "{");
+ char* end = strstr(sample->l_value, "}");
+
+ int32_t len = end -start;
+
+ char* keyvalues = taosMemoryMalloc(len);
+ memset(keyvalues, 0, len);
+ memcpy(keyvalues, start + 1, len - 1);
+
+ int32_t count = taos_monitor_count_occurrences(keyvalues, ",");
+
+ char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1));
+ memset(keyvalue, 0, sizeof(char*) * (count + 1));
+ taos_monitor_split_str(keyvalue, keyvalues, ",");
+
+ char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2);
+ memset(arr, 0, sizeof(char*) * (count + 1) * 2);
+
+ bool isfound = true;
+ for(int32_t i = 0; i < count + 1; i++){
+ char* str = *(keyvalue + i);
+
+ char** pair = arr + i * 2;
+ taos_monitor_split_str(pair, str, "=");
+
+ taos_monitor_strip(pair[1]);
+ }
+
+ int32_t table_size = tjsonGetArraySize(arrayMetricGroups);
+
+ SJson* item = NULL;
+ for(int32_t i = 0; i < table_size; i++){
+ SJson *cur = tjsonGetArrayItem(arrayMetricGroups, i);
+
+ SJson* tag = tjsonGetObjectItem(cur, "tags");
+
+ if(taos_monitor_is_match(tag, arr, count + 1)) {
+ item = cur;
+ break;
+ }
+ }
+
+ SJson* metrics = NULL;
+ if(item == NULL) {
+ item = tjsonCreateObject();
+
+ SJson* arrayTag = tjsonCreateArray();
+ for(int32_t i = 0; i < count + 1; i++){
+ char** pair = arr + i * 2;
+
+ char* key = *pair;
+ char* value = *(pair + 1);
+
+ SJson* tag = tjsonCreateObject();
+ tjsonAddStringToObject(tag, "name", key);
+ tjsonAddStringToObject(tag, "value", value);
+
+ tjsonAddItemToArray(arrayTag, tag);
+ }
+ tjsonAddItemToObject(item, "tags", arrayTag);
+
+ metrics = tjsonCreateArray();
+ tjsonAddItemToObject(item, "metrics", metrics);
+
+ tjsonAddItemToArray(arrayMetricGroups, item);
+ }
+ else{
+ metrics = tjsonGetObjectItem(item, "metrics");
+ }
+
+ taosMemoryFreeClear(arr);
+ taosMemoryFreeClear(keyvalue);
+ taosMemoryFreeClear(keyvalues);
+
+ SJson* metric = tjsonCreateObject();
+ tjsonAddStringToObject(metric, "name", metricName);
+
+ double old_value = 0;
+#define USE_EXCHANGE
+#ifdef USE_EXCHANGE
+ taos_metric_sample_exchange(sample, 0, &old_value);
+#else
+ old_value = sample->r_value;
+ taos_metric_sample_set(sample, 0);
+#endif
+
+ tjsonAddDoubleToObject(metric, "value", old_value);
+ tjsonAddDoubleToObject(metric, "type", metric_type);
+ tjsonAddItemToArray(metrics, metric);
+
+ return 0;
+}
+
+int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format,
+ SJson* tableArray) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+
+ int r = 0;
+
+ int32_t size = strlen(metric->name);
+ char* name = taosMemoryMalloc(size + 1);
+ memset(name, 0, size + 1);
+ memcpy(name, metric->name, size);
+ char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name
+ taos_monitor_split_str((char**)&arr, name, ":");
+
+ bool isFound = false;
+ SJson* table = NULL;
+ SJson* arrayMetricGroups = NULL;
+
+ int32_t table_count = tjsonGetArraySize(tableArray);
+ for(int32_t i = 0; i < table_count; i++){
+ SJson* table = tjsonGetArrayItem(tableArray, i);
+
+ char tableName[MONITOR_TABLENAME_LEN] = {0};
+ tjsonGetStringValue(table, "name", tableName);
+ if(strcmp(tableName, arr[0]) == 0){
+ isFound = true;
+ arrayMetricGroups = tjsonGetObjectItem(table, "metric_groups");
+ break;
+ }
+ }
+
+ if(!isFound){
+ table = tjsonCreateObject();
+
+ tjsonAddStringToObject(table, "name", arr[0]);
+
+ arrayMetricGroups = tjsonCreateArray();
+ tjsonAddItemToObject(table, "metric_groups", arrayMetricGroups);
+ }
+
+ int32_t sample_count = 0;
+ for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *key = (const char *)current_node->item;
+ if (metric->type == TAOS_HISTOGRAM) {
+
+ } else {
+ taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key);
+ if (sample == NULL) return 1;
+ r = taos_metric_formatter_load_sample_new(self, sample, ts, format, arr[1], metric->type, arrayMetricGroups);
+ if (r) return r;
+ }
+ sample_count++;
+ }
+
+ if(!isFound && sample_count > 0){
+ tjsonAddItemToArray(tableArray, table);
+ }
+ else{
+ if(table != NULL) tjsonDelete(table);
+ }
+
+ taosMemoryFreeClear(name);
+ return r;
+}
+
+int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts,
+ char *format, SJson* tableArray) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+
+ for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *collector_name = (const char *)current_node->item;
+ taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name);
+ if (collector == NULL) return 1;
+
+ taos_map_t *metrics = collector->collect_fn(collector);
+ if (metrics == NULL) return 1;
+
+ //if(strcmp(collector->name, "custom") != 0 ){
+
+ r = pthread_rwlock_wrlock(metrics->rwlock);
+ if (r) {
+ TAOS_LOG("failed to lock");
+ return r;
+ }
+
+#ifdef TAOS_LOG_ENABLE
+ int32_t count = 0;
+#endif
+ for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+#ifdef TAOS_LOG_ENABLE
+ count++;
+#endif
+ const char *metric_name = (const char *)current_node->item;
+ taos_metric_t *metric = (taos_metric_t *)taos_map_get_withoutlock(metrics, metric_name);
+ if (metric == NULL) {
+#ifdef TAOS_LOG_ENABLE
+ char tmp[200] = {0};
+ sprintf(tmp, "fail to get metric(%d):%s", count, metric_name);
+ TAOS_LOG(tmp);
+#endif
+ continue;;
+ }
+ r = taos_metric_formatter_load_metric_new(self, metric, ts, format, tableArray);
+ if (r) {
+ TAOS_LOG("failed to load metric");
+ continue;
+ }
+ }
+
+#ifdef TAOS_LOG_ENABLE
+ char tmp[20] = {0};
+ sprintf(tmp, "list count:%d", count);
+ TAOS_LOG(tmp);
+#endif
+ r = pthread_rwlock_unlock(metrics->rwlock);
+ if (r) {
+ TAOS_LOG("failed to unlock");
+ return r;
+ }
+ //}
+ //else{
+ /*
+ for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL;
+ current_node = current_node->next) {
+ const char *metric_name = (const char *)current_node->item;
+ taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name);
+ if (metric == NULL) return 1;
+ r = taos_metric_formatter_load_metric(self, metric, ts, format);
+ if (r) return r;
+ }
+ */
+ //}
+ }
+ return r;
+}
\ No newline at end of file
diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c
new file mode 100644
index 0000000000..ca6ea30028
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_metric_sample.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_errors.h"
+#include "taos_log.h"
+#include "taos_metric_sample_i.h"
+#include "taos_metric_sample_t.h"
+
+#ifdef C11_ATOMIC
+#include
+#else
+#define ALLOW_FORBID_FUNC
+#include "tdef.h"
+#include "osAtomic.h"
+#endif
+
+taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) {
+ taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t));
+ self->type = type;
+ self->l_value = taos_strdup(l_value);
+ self->r_value = 0;
+ return self;
+}
+
+int taos_metric_sample_destroy(taos_metric_sample_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+ taos_free((void *)self->l_value);
+ self->l_value = NULL;
+ taos_free((void *)self);
+ self = NULL;
+ return 0;
+}
+
+int taos_metric_sample_destroy_generic(void *gen) {
+ int r = 0;
+
+ taos_metric_sample_t *self = (taos_metric_sample_t *)gen;
+ r = taos_metric_sample_destroy(self);
+ self = NULL;
+ return r;
+}
+
+void taos_metric_sample_free_generic(void *gen) {
+ taos_metric_sample_t *self = (taos_metric_sample_t *)gen;
+ taos_metric_sample_destroy(self);
+}
+
+int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) {
+ TAOS_ASSERT(self != NULL);
+ if (r_value < 0) {
+ return 1;
+ }
+
+#ifdef C11_ATOMIC
+ /*_Atomic*/ double old = atomic_load(&self->r_value);
+
+ for (;;) {
+ _Atomic double new = ATOMIC_VAR_INIT(old + r_value);
+ if (atomic_compare_exchange_weak(&self->r_value, &old, new)) {
+ return 0;
+ }
+ }
+#else
+#ifdef DOUBLE_ATOMIC
+ atomic_fetch_add_double(&self->r_value, r_value);
+#else
+ atomic_fetch_add_64(&self->r_value, r_value);
+#endif
+#endif
+
+ return 0;
+}
+
+int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) {
+ TAOS_ASSERT(self != NULL);
+ if (self->type != TAOS_GAUGE) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+
+#ifdef C11_ATOMIC
+ /*_Atomic*/ double old = atomic_load(&self->r_value);
+ for (;;) {
+ _Atomic double new = ATOMIC_VAR_INIT(old - r_value);
+ if (atomic_compare_exchange_weak(&self->r_value, &old, new)) {
+ return 0;
+ }
+ }
+#else
+#ifdef DOUBLE_ATOMIC
+ atomic_fetch_sub_double(&self->r_value, r_value);
+#else
+ atomic_fetch_sub_64(&self->r_value, r_value);
+#endif
+#endif
+
+ return 0;
+}
+
+int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) {
+ if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+
+#ifdef C11_ATOMIC
+ atomic_store(&self->r_value, r_value);
+#else
+#ifdef DOUBLE_ATOMIC
+ atomic_store_double(&self->r_value, r_value);
+#else
+ atomic_store_64(&self->r_value, r_value);
+#endif
+#endif
+
+ return 0;
+}
+
+int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value) {
+ if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) {
+ TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE);
+ return 1;
+ }
+
+#ifdef C11_ATOMIC
+ _Atomic double new = ATOMIC_VAR_INIT(r_value);
+ for (;;) {
+ /*_Atomic*/ double old = atomic_load(&self->r_value);
+ *old_value = old;
+ if (atomic_compare_exchange_weak(&self->r_value, &old, new)) {
+ return 0;
+ }
+ }
+#else
+#ifdef DOUBLE_ATOMIC
+ *old_value = atomic_exchange_double(&self->r_value, r_value);
+#else
+ *old_value = atomic_exchange_64(&self->r_value, r_value);
+#endif
+#endif
+
+ return 0;
+}
\ No newline at end of file
diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c
new file mode 100644
index 0000000000..182402b3ff
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_monitor_util.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+
+#define ALLOW_FORBID_FUNC
+#include "tjson.h"
+#include
+#include
+#include
+#include "osMemory.h"
+#include "tdef.h"
+
+#include "taos_metric_t.h"
+
+void taos_monitor_split_str(char** arr, char* str, const char* del) {
+ char *lasts;
+ char* s = strsep(&str, del);
+ while (s != NULL) {
+ *arr++ = s;
+ s = strsep(&str, del);
+ }
+}
+
+void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) {
+ int32_t size = strlen(metric->name);
+ char* name = taosMemoryMalloc(size + 1);
+ memset(name, 0, size + 1);
+ memcpy(name, metric->name, size);
+
+ char* s = strtok(name, del);
+ while (s != NULL) {
+ *arr++ = s;
+ s = strtok(NULL, del);
+ }
+
+ *buf = name;
+}
+
+const char* taos_monitor_get_metric_name(taos_metric_t* metric){
+ return metric->name;
+}
+
+int taos_monitor_count_occurrences(char *str, char *toSearch) {
+ int count = 0;
+ char *ptr = str;
+ while ((ptr = strstr(ptr, toSearch)) != NULL) {
+ count++;
+ ptr++;
+ }
+ return count;
+}
+
+void taos_monitor_strip(char *s)
+{
+ size_t i;
+ size_t len = strlen(s);
+ size_t offset = 0;
+ for(i = 0; i < len; ++i){
+ char c = s[i];
+ if(c=='\"') ++offset;
+ else s[i-offset] = c;
+ }
+ s[len-offset] = '\0';
+}
+
+bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count) {
+ int32_t size = tjsonGetArraySize(tags);
+ if(size != count) return false;
+
+ for(int32_t i = 0; i < size; i++){
+ SJson* item = tjsonGetArrayItem(tags, i);
+
+ char item_name[MONITOR_TAG_NAME_LEN] = {0};
+ tjsonGetStringValue(item, "name", item_name);
+
+ char item_value[MONITOR_TAG_VALUE_LEN] = {0};
+ tjsonGetStringValue(item, "value", item_value);
+
+ bool isfound = false;
+ for(int32_t j = 0; j < count; j++){
+
+ char** pair = pairs + j * 2;
+
+ char* key = *pair;
+ char* value = *(pair + 1);
+
+
+ if(strcmp(value, item_value) == 0 && strcmp(key, item_name) == 0){
+ isfound = true;
+ break;
+ }
+ }
+
+ if(!isfound) return false;
+ }
+
+ return true;
+}
diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c
new file mode 100644
index 0000000000..0f3940cdab
--- /dev/null
+++ b/source/libs/monitorfw/src/taos_string_builder.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include
+
+// Public
+#include "taos_alloc.h"
+
+// Private
+#include "taos_assert.h"
+#include "taos_string_builder_i.h"
+#include "taos_string_builder_t.h"
+
+// The initial size of a string created via taos_string_builder
+#define TAOS_STRING_BUILDER_INIT_SIZE 32
+
+// taos_string_builder_init prototype declaration
+int taos_string_builder_init(taos_string_builder_t *self);
+
+struct taos_string_builder {
+ char *str; /**< the target string */
+ size_t allocated; /**< the size allocated to the string in bytes */
+ size_t len; /**< the length of str */
+ size_t init_size; /**< the initialize size of space to allocate */
+};
+
+taos_string_builder_t *taos_string_builder_new(void) {
+ int r = 0;
+
+ taos_string_builder_t *self = (taos_string_builder_t *)taos_malloc(sizeof(taos_string_builder_t));
+ self->init_size = TAOS_STRING_BUILDER_INIT_SIZE;
+ r = taos_string_builder_init(self);
+ if (r) {
+ taos_string_builder_destroy(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+int taos_string_builder_init(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ self->str = (char *)taos_malloc(self->init_size);
+ *self->str = '\0';
+ self->allocated = self->init_size;
+ self->len = 0;
+ return 0;
+}
+
+int taos_string_builder_destroy(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 0;
+ taos_free(self->str);
+ self->str = NULL;
+ taos_free(self);
+ self = NULL;
+ return 0;
+}
+
+/**
+ * @brief API PRIVATE Grows the size of the string given the value we want to add
+ *
+ * The method continuously shifts left until the new size is large enough to accommodate add_len. This private method
+ * is called in methods that need to add one or more characters to the underlying string.
+ */
+static int taos_string_builder_ensure_space(taos_string_builder_t *self, size_t add_len) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (add_len == 0 || self->allocated >= self->len + add_len + 1) return 0;
+ while (self->allocated < self->len + add_len + 1) self->allocated <<= 1;
+ self->str = (char *)taos_realloc(self->str, self->allocated);
+ return 0;
+}
+
+int taos_string_builder_add_str(taos_string_builder_t *self, const char *str) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+
+ if (self == NULL) return 1;
+ if (str == NULL || *str == '\0') return 0;
+
+ size_t len = strlen(str);
+ r = taos_string_builder_ensure_space(self, len);
+ if (r) return r;
+
+ memcpy(self->str + self->len, str, len);
+ self->len += len;
+ self->str[self->len] = '\0';
+ return 0;
+}
+
+int taos_string_builder_add_char(taos_string_builder_t *self, char c) {
+ TAOS_ASSERT(self != NULL);
+ int r = 0;
+
+ if (self == NULL) return 1;
+ r = taos_string_builder_ensure_space(self, 1);
+ if (r) return r;
+
+ self->str[self->len] = c;
+ self->len++;
+ self->str[self->len] = '\0';
+ return 0;
+}
+
+int taos_string_builder_truncate(taos_string_builder_t *self, size_t len) {
+ TAOS_ASSERT(self != NULL);
+ if (self == NULL) return 1;
+ if (len >= self->len) return 0;
+
+ self->len = len;
+ self->str[self->len] = '\0';
+ return 0;
+}
+
+int taos_string_builder_clear(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ taos_free(self->str);
+ self->str = NULL;
+ return taos_string_builder_init(self);
+}
+
+size_t taos_string_builder_len(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ return self->len;
+}
+
+char *taos_string_builder_dump(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ // +1 to accommodate \0
+ char *out = (char *)taos_malloc((self->len + 1) * sizeof(char));
+ memcpy(out, self->str, self->len + 1);
+ return out;
+}
+
+char *taos_string_builder_str(taos_string_builder_t *self) {
+ TAOS_ASSERT(self != NULL);
+ return self->str;
+}
diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c
index c2cd4786db..4f523bf4f2 100644
--- a/source/libs/parser/src/parAuthenticator.c
+++ b/source/libs/parser/src/parAuthenticator.c
@@ -347,8 +347,11 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
+ case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
case QUERY_NODE_SHOW_VNODES_STMT:
case QUERY_NODE_SHOW_SCORES_STMT:
+ case QUERY_NODE_SHOW_USERS_STMT:
+ case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 8900ef8e7f..808ab71b92 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -285,7 +285,6 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) {
pTmp->suid = pSrc->suid;
pTmp->uid = pSrc->uid;
pTmp->sver = pSrc->sver;
- pTmp->source = pSrc->source;
pTmp->pCreateTbReq = NULL;
if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
if (pSrc->pCreateTbReq) {
@@ -653,7 +652,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end;
}
- pTableCxt->pData->source = SOURCE_TAOSX;
+ pTableCxt->pData->flags |= TD_REQ_FROM_TAOX;
if(tmp == NULL){
ret = initTableColSubmitData(pTableCxt);
if (ret != TSDB_CODE_SUCCESS) {
@@ -721,7 +720,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end;
}
fields += sizeof(int8_t) + sizeof(int32_t);
- if (needChangeLength && version == 1) {
+ if (needChangeLength && version == BLOCK_VERSION_1) {
pStart += htonl(colLength[j]);
} else {
pStart += colLength[j];
@@ -752,7 +751,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end;
}
fields += sizeof(int8_t) + sizeof(int32_t);
- if (needChangeLength && version == 1) {
+ if (needChangeLength && version == BLOCK_VERSION_1) {
pStart += htonl(colLength[i]);
} else {
pStart += colLength[i];
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index f51f297491..8a518bef41 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -390,6 +390,7 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) {
return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName));
}
+#ifdef BUILD_NO_CALL
static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
@@ -413,6 +414,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac
}
return code;
}
+#endif
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) {
int32_t code = TSDB_CODE_SUCCESS;
@@ -791,9 +793,11 @@ static bool isAggFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId));
}
+#ifdef BUILD_NO_CALL
static bool isSelectFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId));
}
+#endif
static bool isWindowPseudoColumnFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId));
@@ -807,9 +811,11 @@ static bool isInterpPseudoColumnFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId));
}
+#ifdef BUILD_NO_CALL
static bool isTimelineFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId));
}
+#endif
static bool isImplicitTsFunc(const SNode* pNode) {
return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsImplicitTsFunc(((SFunctionNode*)pNode)->funcId));
@@ -1700,11 +1706,11 @@ static int32_t dataTypeComp(const SDataType* l, const SDataType* r) {
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
if (isMultiResFunc(pOp->pLeft)) {
- generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
+ generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pLeft))->userAlias);
return DEAL_RES_ERROR;
}
if (isMultiResFunc(pOp->pRight)) {
- generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
+ generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pRight))->userAlias);
return DEAL_RES_ERROR;
}
@@ -8126,9 +8132,11 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl
return code;
}
+#ifdef BUILD_NO_CALL
static bool isEventWindowQuery(SSelectStmt* pSelect) {
return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow);
}
+#endif
static bool hasJsonTypeProjection(SSelectStmt* pSelect) {
SNode* pProj = NULL;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 35543ba1ad..04d6b53f2e 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Out of memory";
case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS:
return "ORDER BY \"%s\" is ambiguous";
+ case TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT:
+ return "Operator not supported multi result: %s";
case TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR:
return "Not supported window join having expr";
default:
diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp
index 390faab537..d7b31d2ac8 100644
--- a/source/libs/parser/test/parSelectTest.cpp
+++ b/source/libs/parser/test/parSelectTest.cpp
@@ -381,7 +381,7 @@ TEST_F(ParserSelectTest, semanticCheck) {
// TSDB_CODE_PAR_WRONG_VALUE_TYPE
run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE);
- run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE);
+ run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT);
run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_FUNC_FUNTION_PARA_NUM);
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 37930a5c5d..91cf4dd1d1 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -16,6 +16,7 @@
#include "planInt.h"
#include "filter.h"
#include "functionMgt.h"
+#include "tglobal.h"
typedef struct SLogicPlanContext {
SPlanContext* pPlanCxt;
@@ -1098,6 +1099,10 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo
return TSDB_CODE_OUT_OF_MEMORY;
}
+ if (!pCxt->pPlanCxt->streamQuery && tsDisableCount) {
+ return TSDB_CODE_FAILED;
+ }
+
pWindow->winType = WINDOW_TYPE_COUNT;
pWindow->node.groupAction = getGroupAction(pCxt, pSelect);
pWindow->node.requireDataOrder =
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 1c90e61ea3..1aa21e73a3 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -57,7 +57,7 @@ typedef enum {
#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ
#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20
-#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000
+#define SCH_DEFAULT_TASK_TIMEOUT_USEC 30000000
#define SCH_MAX_TASK_TIMEOUT_USEC 300000000
#define SCH_DEFAULT_MAX_RETRY_NUM 6
#define SCH_MIN_AYSNC_EXEC_NUM 3
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 841066a4c9..795c21c234 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -33,7 +33,7 @@ int32_t schedulerInit() {
schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM;
schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables;
schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY;
- schMgmt.cfg.enableReSchedule = true;
+ schMgmt.cfg.enableReSchedule = false;
qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d",
schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule);
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 9383383dc0..78b914c3db 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -575,7 +575,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
!isAutoTableName(pDataBlock->info.parTbName) &&
!alreadyAddGroupId(pDataBlock->info.parTbName) &&
groupId != 0){
- buildCtbNameAddGruopId(pDataBlock->info.parTbName, groupId);
+ buildCtbNameAddGroupId(pDataBlock->info.parTbName, groupId);
}
} else {
buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName);
@@ -907,7 +907,6 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SStreamChkptReadyInfo info = {.upStreamTaskId = pInfo->taskId, .upstreamNodeEpset = pInfo->epSet};
initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead));
- info.msg.info.noResp = 1; // refactor later.
stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64
":0x%x (vgId:%d) idx:%d, vgId:%d",
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index e2aae130e5..3d0241df75 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -488,6 +488,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS
void* pFileStore = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
if (pCur) {
+ pCur->pStreamFileState = pFileState;
SSessionKey key = {0};
void* pVal = NULL;
int len = 0;
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index f3494377d6..9639921c77 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -494,6 +494,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
pRange->range.maxVer = ver;
pRange->range.minVer = ver;
} else {
+ // the initial value of processedVer/nextProcessVer/checkpointVer for stream task with related fill-history task
+ // is set at the mnode.
if (pTask->info.fillHistory == 1) {
pChkInfo->checkpointVer = pRange->range.maxVer;
pChkInfo->processedVer = pRange->range.maxVer;
@@ -502,6 +504,15 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
pChkInfo->checkpointVer = pRange->range.minVer - 1;
pChkInfo->processedVer = pRange->range.minVer - 1;
pChkInfo->nextProcessVer = pRange->range.minVer;
+
+ { // for compatible purpose, remove it later
+ if (pRange->range.minVer == 0) {
+ pChkInfo->checkpointVer = 0;
+ pChkInfo->processedVer = 0;
+ pChkInfo->nextProcessVer = 1;
+ stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr);
+ }
+ }
}
}
}
@@ -923,9 +934,8 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
}
tEncoderClear(&encoder);
- SRpcMsg msg = {.info.noResp = 1};
+ SRpcMsg msg = {0};
initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen);
-
stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId);
tmsgSendReq(&pTask->info.mnodeEpset, &msg);
diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h
index 7483588593..5b18d56d70 100644
--- a/source/libs/transport/inc/transComm.h
+++ b/source/libs/transport/inc/transComm.h
@@ -427,6 +427,8 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg,
void transDQCancel(SDelayQueue* queue, SDelayTask* task);
bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
+
+bool transEpSetIsEqual2(SEpSet* a, SEpSet* b);
/*
* init global func
*/
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 2370efa460..e937d2e65e 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -2224,7 +2224,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) {
EPSET_FORWARD_INUSE(&pCtx->epSet);
}
} else {
- if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) {
+ if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) {
tDebug("epset not equal, retry new epset1");
transPrintEpSet(&pCtx->epSet);
transPrintEpSet(&epSet);
@@ -2251,7 +2251,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) {
EPSET_FORWARD_INUSE(&pCtx->epSet);
}
} else {
- if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) {
+ if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) {
tDebug("epset not equal, retry new epset2");
transPrintEpSet(&pCtx->epSet);
transPrintEpSet(&epSet);
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index 4619722743..27099c76fc 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) {
char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead));
STransMsgHead* pNewHead = (STransMsgHead*)buf;
int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content,
- len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
+ len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen);
memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead));
pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead));
@@ -603,6 +603,17 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
}
return true;
}
+bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) {
+ if (a->numOfEps != b->numOfEps) {
+ return false;
+ }
+ for (int i = 0; i < a->numOfEps; i++) {
+ if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) {
+ return false;
+ }
+ }
+ return true;
+}
static void transInitEnv() {
refMgt = transOpenRefMgt(50000, transDestroyExHandle);
diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c
index a54c301de9..c891ee4579 100644
--- a/source/os/src/osAtomic.c
+++ b/source/os/src/osAtomic.c
@@ -16,6 +16,12 @@
#define ALLOW_FORBID_FUNC
#include "os.h"
+typedef union {
+ volatile int64_t i;
+ volatile double d;
+ //double d;
+} double_number;
+
#ifdef WINDOWS
// add
@@ -339,6 +345,21 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) {
#endif
}
+double atomic_store_double(double volatile *ptr, double val){
+ for (;;) {
+ double_number old_num = {0};
+ old_num.d = *ptr; // current old value
+
+ double_number new_num = {0};
+ new_num.d = val;
+
+ double_number ret_num = {0};
+ ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i);
+
+ if (ret_num.i == old_num.i) return ret_num.d;
+ }
+}
+
void atomic_store_ptr(void* ptr, void* val) {
#ifdef WINDOWS
((*(void* volatile*)(ptr)) = (void*)(val));
@@ -393,6 +414,23 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) {
#endif
}
+double atomic_exchange_double(double volatile *ptr, int64_t val){
+ for (;;) {
+ double_number old_num = {0};
+ old_num.d = *ptr; // current old value
+
+ double_number new_num = {0};
+ int64_t iNew = val;
+
+ double_number ret_num = {0};
+ ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i);
+
+ if (ret_num.i == old_num.i) {
+ return ret_num.d;
+ }
+ }
+}
+
void* atomic_exchange_ptr(void* ptr, void* val) {
#ifdef WINDOWS
#ifdef _WIN64
@@ -551,6 +589,21 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) {
#endif
}
+double atomic_fetch_add_double(double volatile *ptr, double val){
+ for (;;) {
+ double_number old_num = {0};
+ old_num.d = *ptr; // current old value
+
+ double_number new_num = {0};
+ new_num.d = old_num.d + val;
+
+ double_number ret_num = {0};
+ ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i);
+
+ if (ret_num.i == old_num.i) return ret_num.d;
+ }
+}
+
void* atomic_fetch_add_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val));
@@ -657,6 +710,21 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) {
#endif
}
+double atomic_fetch_sub_double(double volatile *ptr, double val){
+ for (;;) {
+ double_number old_num = {0};
+ old_num.d = *ptr; // current old value
+
+ double_number new_num = {0};
+ new_num.d = old_num.d - val;
+
+ double_number ret_num = {0};
+ ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i);
+
+ if (ret_num.i == old_num.i) return ret_num.d;
+ }
+}
+
void* atomic_fetch_sub_ptr(void* ptr, void* val) {
#ifdef WINDOWS
return interlocked_fetch_sub_ptr(ptr, val);
diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c
index fea7a4f63d..187461826a 100644
--- a/source/os/src/osSysinfo.c
+++ b/source/os/src/osSysinfo.c
@@ -563,9 +563,9 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) {
}
void taosGetCpuUsage(double *cpu_system, double *cpu_engine) {
- static int64_t lastSysUsed = 0;
- static int64_t lastSysTotal = 0;
- static int64_t lastProcTotal = 0;
+ static int64_t lastSysUsed = -1;
+ static int64_t lastSysTotal = -1;
+ static int64_t lastProcTotal = -1;
static int64_t curSysUsed = 0;
static int64_t curSysTotal = 0;
static int64_t curProcTotal = 0;
@@ -580,12 +580,14 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) {
curSysTotal = curSysUsed + sysCpu.idle;
curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime;
- if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) {
- if (cpu_system != NULL) {
- *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100;
- }
- if (cpu_engine != NULL) {
- *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100;
+ if(lastSysUsed >= 0 && lastSysTotal >=0 && lastProcTotal >=0){
+ if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) {
+ if (cpu_system != NULL) {
+ *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100;
+ }
+ if (cpu_engine != NULL) {
+ *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100;
+ }
}
}
@@ -821,19 +823,27 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int
}
void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) {
- static int64_t last_rchars = 0;
- static int64_t last_wchars = 0;
- static int64_t last_read_bytes = 0;
- static int64_t last_write_bytes = 0;
+ static int64_t last_rchars = -1;
+ static int64_t last_wchars = -1;
+ static int64_t last_read_bytes = -1;
+ static int64_t last_write_bytes = -1;
static int64_t cur_rchars = 0;
static int64_t cur_wchars = 0;
static int64_t cur_read_bytes = 0;
static int64_t cur_write_bytes = 0;
if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) {
- *rchars = cur_rchars - last_rchars;
- *wchars = cur_wchars - last_wchars;
- *read_bytes = cur_read_bytes - last_read_bytes;
- *write_bytes = cur_write_bytes - last_write_bytes;
+ if(last_rchars >=0 && last_wchars >=0 && last_read_bytes >=0 && last_write_bytes >= 0){
+ *rchars = cur_rchars - last_rchars;
+ *wchars = cur_wchars - last_wchars;
+ *read_bytes = cur_read_bytes - last_read_bytes;
+ *write_bytes = cur_write_bytes - last_write_bytes;
+ }
+ else{
+ *rchars = 0;
+ *wchars = 0;
+ *read_bytes = 0;
+ *write_bytes = 0;
+ }
last_rchars = cur_rchars;
last_wchars = cur_wchars;
last_read_bytes = cur_read_bytes;
@@ -900,13 +910,20 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) {
}
void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) {
- static int64_t last_receive_bytes = 0;
- static int64_t last_transmit_bytes = 0;
+ static int64_t last_receive_bytes = -1;
+ static int64_t last_transmit_bytes = -1;
int64_t cur_receive_bytes = 0;
int64_t cur_transmit_bytes = 0;
if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) {
- *receive_bytes = cur_receive_bytes - last_receive_bytes;
- *transmit_bytes = cur_transmit_bytes - last_transmit_bytes;
+ if(last_receive_bytes >= 0 && last_transmit_bytes >= 0){
+ *receive_bytes = cur_receive_bytes - last_receive_bytes;
+ *transmit_bytes = cur_transmit_bytes - last_transmit_bytes;
+ }
+ else{
+ *receive_bytes = 0;
+ *transmit_bytes = 0;
+ }
+
last_receive_bytes = cur_receive_bytes;
last_transmit_bytes = cur_transmit_bytes;
} else {
diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c
index fa3ecadd14..5eb72879c3 100644
--- a/source/util/src/tbase58.c
+++ b/source/util/src/tbase58.c
@@ -18,24 +18,29 @@
#include
#include
-#define BASE_BUF_SIZE 256
+#define TBASE_BUF_SIZE 256
static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
char *base58_encode(const uint8_t *value, int32_t vlen) {
const uint8_t *pb = value;
const uint8_t *pe = pb + vlen;
- uint8_t buf[BASE_BUF_SIZE] = {0};
+ uint8_t buf[TBASE_BUF_SIZE] = {0};
uint8_t *pbuf = &buf[0];
bool bfree = false;
int32_t nz = 0, size = 0, len = 0;
+ if (vlen > TBASE_MAX_ILEN) {
+ terrno = TSDB_CODE_INVALID_PARA;
+ return NULL;
+ }
+
while (pb != pe && *pb == 0) {
++pb;
++nz;
}
size = (pe - pb) * 69 / 50 + 1;
- if (size > BASE_BUF_SIZE) {
+ if (size > TBASE_BUF_SIZE) {
if (!(pbuf = taosMemoryCalloc(1, size))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) {
int32_t num = *pb;
int32_t i = 0;
for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) {
- num += ((int32_t)buf[j]) << 8;
+ num += ((int32_t)pbuf[j]) << 8;
pbuf[j] = num % 58;
num /= 58;
}
@@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) {
const uint8_t *pi = pbuf + (size - len);
while (pi != pbuf + size && *pi == 0) ++pi;
- uint8_t *result = taosMemoryCalloc(1, size + 1);
+ uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1);
if (!result) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if (bfree) taosMemoryFree(pbuf);
@@ -82,20 +87,35 @@ static const signed char index_58[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
+ const char *pb = value;
const char *pe = value + inlen;
- uint8_t buf[BASE_BUF_SIZE] = {0};
+ uint8_t buf[TBASE_BUF_SIZE] = {0};
uint8_t *pbuf = &buf[0];
bool bfree = false;
int32_t nz = 0, size = 0, len = 0;
- while (*value && isspace(*value)) ++value;
- while (*value == '1') {
- ++nz;
- ++value;
+ if (inlen > TBASE_MAX_OLEN) {
+ terrno = TSDB_CODE_INVALID_PARA;
+ return NULL;
}
- size = (int32_t)(pe - value) * 733 / 1000 + 1;
- if (size > BASE_BUF_SIZE) {
+ while (pb != pe) {
+ if (*pb == 0) {
+ terrno = TSDB_CODE_INVALID_PARA;
+ return NULL;
+ }
+ ++pb;
+ }
+
+ pb = value;
+ while (pb != pe && *pb && isspace(*pb)) ++pb;
+ while (pb != pe && *pb == '1') {
+ ++nz;
+ ++pb;
+ }
+
+ size = (int32_t)(pe - pb) * 733 / 1000 + 1;
+ if (size > TBASE_BUF_SIZE) {
if (!(pbuf = taosMemoryCalloc(1, size))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@@ -103,9 +123,10 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
bfree = true;
}
- while (*value && !isspace(*value)) {
- int32_t num = index_58[(uint8_t)*value];
+ while (pb != pe && *pb && !isspace(*pb)) {
+ int32_t num = index_58[(uint8_t)*pb];
if (num == -1) {
+ terrno = TSDB_CODE_INVALID_PARA;
if (bfree) taosMemoryFree(pbuf);
return NULL;
}
@@ -116,18 +137,18 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
num >>= 8;
}
len = i;
- ++value;
+ ++pb;
}
- while (isspace(*value)) ++value;
- if (*value != 0) {
+ while (pb != pe && isspace(*pb)) ++pb;
+ if (*pb != 0) {
if (bfree) taosMemoryFree(pbuf);
return NULL;
}
const uint8_t *it = pbuf + (size - len);
while (it != pbuf + size && *it == 0) ++it;
- uint8_t *result = taosMemoryCalloc(1, size + 1);
+ uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1);
if (!result) {
if (bfree) taosMemoryFree(pbuf);
terrno = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index edb902b504..e24f1d7c8e 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -621,10 +621,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid stream quer
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_QUERY_MISMATCH, "Columns number mismatch with query result")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table")
+TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, "Operator not supported multi result")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR, "Invalid window join having expr")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED, "GROUP BY/PARTITION BY/WINDOW-clause can't be used in WINDOW join")
-
//planner
TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_EXPECTED_TS_EQUAL, "Expect ts equal")
diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c
index 0b7ad330a7..82993e8449 100644
--- a/source/util/src/tjson.c
+++ b/source/util/src/tjson.c
@@ -375,4 +375,8 @@ bool tjsonValidateJson(const char* jIn) {
return true;
}
-const char* tjsonGetError() { return cJSON_GetErrorPtr(); }
\ No newline at end of file
+const char* tjsonGetError() { return cJSON_GetErrorPtr(); }
+
+void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName) {
+ cJSON_DeleteItemFromObject((cJSON*)pJson, pName);
+}
\ No newline at end of file
diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt
index f4f3880388..3514c578e9 100644
--- a/source/util/test/CMakeLists.txt
+++ b/source/util/test/CMakeLists.txt
@@ -100,3 +100,11 @@ add_test(
NAME talgoTest
COMMAND talgoTest
)
+
+# tbaseCodecTest
+add_executable(tbaseCodecTest "tbaseCodecTest.cpp")
+target_link_libraries(tbaseCodecTest os util common gtest_main)
+add_test(
+ NAME tbaseCodecTest
+ COMMAND tbaseCodecTest
+)
diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp
new file mode 100644
index 0000000000..4c56979885
--- /dev/null
+++ b/source/util/test/tbaseCodecTest.cpp
@@ -0,0 +1,83 @@
+#include
+#include
+
+#include
+#include "os.h"
+#include "osTime.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "tbase58.h"
+#include "tglobal.h"
+
+using namespace std;
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+int main(int argc, char **argv) {
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) {
+ int64_t start = taosGetTimestampUs();
+ char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen);
+ ASSERT_NE(nullptr, pEnc);
+
+ int32_t encLen = strlen(pEnc);
+ int64_t endOfEnc = taosGetTimestampUs();
+ std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl;
+ int32_t decLen = 0;
+ char *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen);
+ std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us"
+ << std::endl;
+ ASSERT_NE(nullptr, pDec);
+ ASSERT_EQ(rawLen, decLen);
+ ASSERT_LE(rawLen, encLen);
+ ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen));
+ taosMemoryFreeClear(pDec);
+ taosMemoryFreeClear(pEnc);
+}
+
+TEST(TD_BASE_CODEC_TEST, tbase58_test) {
+ const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN;
+ const int32_t TEST_LEN_STEP = 10;
+ int32_t rawLen = 0;
+ uint8_t *pRaw = NULL;
+
+ pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX);
+ ASSERT_NE(nullptr, pRaw);
+
+ // 1. normal case
+ // string blend with char and '\0'
+ rawLen = TEST_LEN_MAX;
+ for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) {
+ checkBase58Codec(pRaw, rawLen, i);
+ pRaw[i] = i & 127;
+ }
+
+ // string without '\0'
+ for (int32_t i = 0; i < TEST_LEN_MAX; ++i) {
+ pRaw[i] = i & 127;
+ }
+ checkBase58Codec(pRaw, TEST_LEN_MAX, 0);
+ for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) {
+ rawLen = i;
+ checkBase58Codec(pRaw, rawLen, i);
+ }
+ taosMemoryFreeClear(pRaw);
+ ASSERT_EQ(nullptr, pRaw);
+
+ // 2. overflow case
+ char tmp[1];
+ char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1);
+ ASSERT_EQ(nullptr, pEnc);
+ char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL);
+ ASSERT_EQ(nullptr, pDec);
+
+ taosMemoryFreeClear(pRaw);
+ ASSERT_EQ(nullptr, pRaw);
+}
\ No newline at end of file
diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py
index 6bcf547136..85f030eb03 100644
--- a/tests/army/community/cluster/incSnapshot.py
+++ b/tests/army/community/cluster/incSnapshot.py
@@ -67,7 +67,6 @@ class TDTestCase(TBase):
dirs = glob.glob(dnodesRootDir)
for dir in dirs:
if os.path.isdir(dir):
- tdLog.debug("delete dir: %s " % (dnodesRootDir))
self.remove_directory(os.path.join(dir, "wal"))
sc.dnodeStart(1)
@@ -88,7 +87,7 @@ class TDTestCase(TBase):
if bFinish:
break
- self.timestamp_step = 1
+ self.timestamp_step = 1000
self.insert_rows = 6000
self.checkInsertCorrect()
self.checkAggCorrect()
diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py
index b21cbb1ad8..eef650cc77 100644
--- a/tests/army/community/cluster/snapshot.py
+++ b/tests/army/community/cluster/snapshot.py
@@ -25,6 +25,7 @@ from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
+from frame.srvCtl import *
class TDTestCase(TBase):
@@ -65,6 +66,21 @@ class TDTestCase(TBase):
sql = f"select avg(dc) from {self.db}.{self.stb}"
tdSql.checkFirstValue(sql, 200)
+ def alterReplica3(self):
+ sql = f"alter database {self.db} replica 3"
+ tdSql.execute(sql, show=True)
+ time.sleep(2)
+ sc.dnodeStop(2)
+ sc.dnodeStop(3)
+ time.sleep(5)
+ sc.dnodeStart(2)
+ sc.dnodeStart(3)
+
+ if self.waitTransactionZero() is False:
+ tdLog.exit(f"{sql} transaction not finished")
+ return False
+ return True
+
def doAction(self):
tdLog.info(f"do action.")
self.flushDb()
@@ -81,7 +97,7 @@ class TDTestCase(TBase):
self.alterReplica(1)
self.checkAggCorrect()
self.compactDb()
- self.alterReplica(3)
+ self.alterReplica3()
vgids = self.getVGroup(self.db)
selid = random.choice(vgids)
diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py
new file mode 100644
index 0000000000..8b6844820e
--- /dev/null
+++ b/tests/army/community/query/show.py
@@ -0,0 +1,152 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+import random
+
+import taos
+import frame
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+
+
+class TDTestCase(TBase):
+ updatecfgDict = {
+ }
+
+ def insertData(self):
+ tdLog.info(f"create table and insert data.")
+ self.stb = "stb"
+ self.db = "db"
+ self.childtable_count = 10
+ self.insert_rows = 10000
+
+ self.autoGen = AutoGen(startTs = 1600000000000*1000*1000, batch=500, fillOne=True)
+ self.autoGen.create_db(self.db, 2, 3, "precision 'ns'")
+ self.autoGen.create_stable(stbname = self.stb, tag_cnt = 5, column_cnt = 20, binary_len = 10, nchar_len = 5)
+ self.autoGen.create_child(self.stb, "child", self.childtable_count)
+ self.autoGen.insert_data(self.insert_rows, True)
+
+ tdLog.info("create view.")
+ tdSql.execute(f"use {self.db}")
+ sqls = [
+ "create view viewc0c1 as select c0,c1 from stb ",
+ "create view viewc0c1c2 as select c0,c1,c2 from stb ",
+ "create view viewc0c3 as select c0,c3 from stb where c3=1",
+ "create view viewc0c4c5 as select c4,c5 from stb ",
+ "create view viewc0c6 as select c0,c1,c6 from stb ",
+ "create view viewc0c7 as select c0,c1 from stb ",
+ "create view viewc0c7c8 as select c0,c7,c8 from stb where c8>0",
+ "create view viewc0c3c1 as select c0,c3,c1 from stb ",
+ "create view viewc2c4 as select c2,c4 from stb ",
+ "create view viewc2c5 as select c2,c5 from stb ",
+ ]
+ tdSql.executes(sqls)
+
+ def checkView(self):
+ tdLog.info(f"check view like.")
+
+ # like
+ sql = f"show views like 'view%'"
+ tdSql.query(sql)
+ tdSql.checkRows(10)
+
+ sql = f"show views like 'vie_c0c1c2'"
+ tdSql.query(sql)
+ tdSql.checkRows(1)
+ tdSql.checkData(0,0,"viewc0c1c2")
+
+ sql = f"show views like '%c2c_'"
+ tdSql.query(sql)
+ tdSql.checkRows(2)
+ tdSql.checkData(0,0, "viewc2c4")
+ tdSql.checkData(1,0, "viewc2c5")
+
+ sql = f"show views like '%' "
+ tdSql.query(sql)
+ tdSql.checkRows(10)
+
+ # zero
+ sql = "show views like '_' "
+ tdSql.query(sql)
+ tdSql.checkRows(0)
+ sql = "show views like 'a%' "
+ tdSql.query(sql)
+ tdSql.checkRows(0)
+
+
+ def doQuery(self):
+ tdLog.info(f"do query.")
+
+ # __group_key
+ sql = f"select count(*) from {self.stb} "
+ tdSql.query(sql)
+ # column index 1 value same with 2
+ allRows = self.insert_rows * self.childtable_count
+ tdSql.checkFirstValue(sql, allRows)
+
+ def checkShow(self):
+ # not support
+ sql = "show accounts;"
+ tdSql.error(sql)
+
+ # check result
+ sql = "SHOW CLUSTER;"
+ tdSql.query(sql)
+ tdSql.checkRows(1)
+ sql = "SHOW COMPACTS;"
+ tdSql.query(sql)
+ tdSql.checkRows(0)
+ sql = "SHOW COMPACT 1;"
+ tdSql.query(sql)
+ tdSql.checkRows(0)
+ sql = "SHOW CLUSTER MACHINES;"
+ tdSql.query(sql)
+ tdSql.checkRows(1)
+
+ # run to check crash
+ sqls = [
+ "show scores;",
+ "SHOW CLUSTER VARIABLES",
+ "SHOW BNODES;",
+ ]
+ tdSql.executes(sqls)
+
+
+ # run
+ def run(self):
+ tdLog.debug(f"start to excute {__file__}")
+
+ # insert data
+ self.insertData()
+
+ # check view
+ self.checkView()
+
+ # do action
+ self.doQuery()
+
+
+ tdLog.success(f"{__file__} successfully executed")
+
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json
index d7544a897c..747ac7c8ec 100644
--- a/tests/army/enterprise/s3/s3_basic.json
+++ b/tests/army/enterprise/s3/s3_basic.json
@@ -6,7 +6,7 @@
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
- "num_of_records_per_req": 2000,
+ "num_of_records_per_req": 4000,
"prepared_rand": 1000,
"thread_count": 2,
"create_table_thread_count": 1,
@@ -18,26 +18,27 @@
"drop": "yes",
"vgroups": 2,
"replica": 1,
- "duration":"1d",
- "keep": "3d,6d,30d"
+ "duration":"15d",
+ "flush_each_batch":"yes",
+ "keep": "60d,100d,200d"
},
"super_tables": [
{
"name": "stb",
"child_table_exists": "no",
- "childtable_count": 4,
- "insert_rows": 1000000,
+ "childtable_count": 2,
+ "insert_rows": 2000000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1000,
- "start_timestamp":"now-13d",
+ "start_timestamp":"now-90d",
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc" },
{ "type": "double", "name": "dc"},
- { "type": "tinyint", "name": "ti", "values":["1"]},
+ { "type": "tinyint", "name": "ti"},
{ "type": "smallint", "name": "si" },
- { "type": "int", "name": "ic" },
+ { "type": "int", "name": "ic" ,"max": 1,"min": 1},
{ "type": "bigint", "name": "bi" },
{ "type": "utinyint", "name": "uti"},
{ "type": "usmallint", "name": "usi"},
diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py
index e7bc188ca5..e9173dda00 100644
--- a/tests/army/enterprise/s3/s3_basic.py
+++ b/tests/army/enterprise/s3/s3_basic.py
@@ -28,12 +28,12 @@ from frame import *
from frame.eos import *
#
-# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+# 192.168.1.52 MINIO S3
#
'''
s3EndPoint http://192.168.1.52:9000
-s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+s3AccessKey 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX'
s3BucketName ci-bucket
s3UploadDelaySec 60
'''
@@ -42,7 +42,7 @@ s3UploadDelaySec 60
class TDTestCase(TBase):
updatecfgDict = {
's3EndPoint': 'http://192.168.1.52:9000',
- 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa',
+ 's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX',
's3BucketName': 'ci-bucket',
's3BlockSize': '10240',
's3BlockCacheSize': '320',
@@ -58,8 +58,8 @@ class TDTestCase(TBase):
tdSql.execute(f"use {self.db}")
# come from s3_basic.json
- self.childtable_count = 4
- self.insert_rows = 1000000
+ self.childtable_count = 2
+ self.insert_rows = 2000000
self.timestamp_step = 1000
def createStream(self, sname):
@@ -78,14 +78,27 @@ class TDTestCase(TBase):
self.trimDb(True)
rootPath = sc.clusterRootPath()
- cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
+ cmd = f"ls {rootPath}/dnode1/data2*/vnode/vnode*/tsdb/*.data"
tdLog.info(cmd)
loop = 0
- while len(eos.runRetList(cmd)) > 0 and loop < 40:
- time.sleep(5)
+ rets = []
+ while loop < 180:
+ time.sleep(3)
+ rets = eos.runRetList(cmd)
+ cnt = len(rets)
+ if cnt == 0:
+ tdLog.info("All data file upload to server over.")
+ break
self.trimDb(True)
+ tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...")
+ if loop == 0:
+ sc.dnodeStop(1)
+ time.sleep(2)
+ sc.dnodeStart(1)
loop += 1
- tdLog.info(f"loop={loop} wait 5s...")
+
+ if len(rets) > 0:
+ tdLog.exit(f"s3 can not upload all data to server. data files cnt={len(rets)} list={rets}")
def checkStreamCorrect(self):
sql = f"select count(*) from {self.db}.stm1"
diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py
index d1f02e7865..cf21977c75 100644
--- a/tests/army/frame/autogen.py
+++ b/tests/army/frame/autogen.py
@@ -14,15 +14,18 @@ import time
# Auto Gen class
#
class AutoGen:
- def __init__(self, fillOne=False):
- self.ts = 1600000000000
- self.batch_size = 100
+ def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False):
+ self.startTs = startTs
+ self.ts = startTs
+ self.step = step
+ self.batch_size = batch
+ self.fillOne = fillOne
seed = time.time() % 10000
random.seed(seed)
- self.fillOne = fillOne
# set start ts
def set_start_ts(self, ts):
+ self.startTs = ts
self.ts = ts
# set batch size
@@ -111,9 +114,9 @@ class AutoGen:
return ''.join(random.choice(letters) for i in range(count))
# create db
- def create_db(self, dbname, vgroups = 2, replica = 1):
+ def create_db(self, dbname, vgroups = 2, replica = 1, others=""):
self.dbname = dbname
- tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
+ tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica} {others}')
# create table or stable
def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
@@ -167,12 +170,12 @@ class AutoGen:
def insert_data(self, cnt, bContinue=False):
if not bContinue:
- self.ts = 1600000000000
+ self.ts = self.startTs
- currTs = 1600000000000
+ currTs = self.startTs
for i in range(self.child_cnt):
name = f"{self.child_name}{i}"
- currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
+ currTs = self.insert_data_child(name, cnt, self.batch_size, self.step)
self.ts = currTs
tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py
index 091856056b..3a9b0cdf4b 100644
--- a/tests/army/frame/srvCtl.py
+++ b/tests/army/frame/srvCtl.py
@@ -33,14 +33,14 @@ class srvCtl:
# control server
#
- # start
+ # start idx base is 1
def dnodeStart(self, idx):
if clusterDnodes.getModel() == 'cluster':
return clusterDnodes.starttaosd(idx)
return tdDnodes.starttaosd(idx)
- # stop
+ # stop idx base is 1
def dnodeStop(self, idx):
if clusterDnodes.getModel() == 'cluster':
return clusterDnodes.stoptaosd(idx)
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index cc15df33a1..bdccf33c32 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -15,10 +15,11 @@
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py
,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
-,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
+,,n,army,python3 ./test.py -f community/query/show.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
#
@@ -49,6 +50,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tms_memleak.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3
diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim
index c3a75d635b..b7c629e538 100644
--- a/tests/script/tsim/query/query_count0.sim
+++ b/tests/script/tsim/query/query_count0.sim
@@ -9,6 +9,8 @@ print =============== create database
sql create database test vgroups 1;
sql use test;
+sql alter local 'disableCount' '0' ;
+
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql insert into t1 values(1648791213000,0,1,1,1.0);
@@ -175,5 +177,51 @@ if $data33 != 3 then
goto loop3
endi
+print step3
+print =============== create database
+sql create database test3 vgroups 1;
+sql use test3;
+
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql insert into t1 values(1648791213000,0,1,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.1);
+sql insert into t1 values(1648791223002,0,3,3,2.1);
+sql insert into t1 values(1648791223003,1,4,3,3.1);
+sql insert into t1 values(1648791223004,1,5,3,4.1);
+sql insert into t1 values(1648791223005,2,6,3,4.1);
+
+$loop_count = 0
+loop4:
+
+sleep 300
+print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3;
+sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+# row 0
+if $rows != 1 then
+ print ======rows=$rows
+ goto loop4
+endi
+
+print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6;
+sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+
+# row 0
+if $rows != 0 then
+ print ======rows=$rows
+ return -1
+endi
+
print query_count0 end
system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim
index 0694ab062a..0c40303e57 100644
--- a/tests/script/tsim/query/query_count1.sim
+++ b/tests/script/tsim/query/query_count1.sim
@@ -9,6 +9,8 @@ print =============== create database
sql create database test vgroups 4;
sql use test;
+sql alter local 'disableCount' '0' ;
+
sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
diff --git a/tests/script/tsim/query/query_count_sliding0.sim b/tests/script/tsim/query/query_count_sliding0.sim
index 464aec6b97..13a6c94451 100644
--- a/tests/script/tsim/query/query_count_sliding0.sim
+++ b/tests/script/tsim/query/query_count_sliding0.sim
@@ -9,6 +9,8 @@ print =============== create database
sql create database test vgroups 1;
sql use test;
+sql alter local 'disableCount' '0' ;
+
sql create table t1(ts timestamp, a int, b int , c int, d double);
sql insert into t1 values(1648791213000,0,1,1,1.0);
diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim
index 567d963c86..f193e375d9 100644
--- a/tests/script/tsim/user/privilege_sysinfo.sim
+++ b/tests/script/tsim/user/privilege_sysinfo.sim
@@ -99,8 +99,14 @@ sql select * from d2.stb2
sql select * from d2.ctb2
sql select * from d2.ntb2
-print =============== check show
+$loop_cnt = 5
+$loop_idx = 0
+$loop_flag = 0
+
+loop_check_sysinfo_0:
+print =============== check show of sysinfo 0
sql_error show users
+sql_error show user privileges
sql_error show cluster
sql_error select * from information_schema.ins_dnodes
sql_error select * from information_schema.ins_mnodes
@@ -129,6 +135,9 @@ sql show topics
sql show subscriptions
sql show functions
sql_error show grants
+sql_error show grants full;
+sql_error show grants logs;
+sql_error show cluster machines;
sql show queries
sql show connections
sql show apps
@@ -142,7 +151,7 @@ sql_error show dnode 1 variables;
sql show variables;
-print =============== check information_schema
+print =============== check information_schema of sysinfo 0
sql show databases
if $rows != 3 then
return -1
@@ -155,7 +164,7 @@ sql_error select * from information_schema.ins_modules
sql_error select * from information_schema.ins_qnodes
sql_error select * from information_schema.ins_cluster
sql_error select * from information_schema.ins_users
-sql_error select * from information_schema.ins_user_privilege
+sql_error select * from information_schema.ins_user_privileges
sql select * from information_schema.ins_databases
sql select * from information_schema.ins_functions
sql select * from information_schema.ins_indexes
@@ -166,11 +175,14 @@ sql select * from information_schema.ins_topics
sql select * from information_schema.ins_subscriptions
sql select * from information_schema.ins_streams
sql_error select * from information_schema.ins_grants
+sql_error select * from information_schema.ins_grants_full
+sql_error select * from information_schema.ins_grants_logs
+sql_error select * from information_schema.ins_machines
sql_error select * from information_schema.ins_vgroups
sql select * from information_schema.ins_configs
sql_error select * from information_schema.ins_dnode_variables
-print =============== check performance_schema
+print =============== check performance_schema of sysinfo 0
sql use performance_schema;
sql select * from performance_schema.perf_connections
sql select * from performance_schema.perf_queries
@@ -178,4 +190,114 @@ sql select * from performance_schema.perf_consumers
sql select * from performance_schema.perf_trans
sql select * from performance_schema.perf_apps
+goto loop_check_switch
+
+loop_check_sysinfo_1:
+print =============== check show of sysinfo 1
+sql show users
+sql show user privileges
+sql show cluster
+sql select * from information_schema.ins_dnodes
+sql select * from information_schema.ins_mnodes
+sql show snodes
+sql select * from information_schema.ins_qnodes
+sql show dnodes
+sql show snodes
+sql show qnodes
+sql show mnodes
+sql show db.vgroups
+sql_error show db.stables
+sql_error show db.tables
+sql show indexes from stb from db
+sql show databases
+sql show d2.vgroups
+sql show d2.stables
+sql show d2.tables
+sql show indexes from stb2 from d2
+#sql_error show create database db
+sql_error show create table db.stb;
+sql_error show create table db.ctb;
+sql_error show create table db.ntb;
+sql show streams
+sql show consumers
+sql show topics
+sql show subscriptions
+sql show functions
+sql show grants
+sql show grants full;
+sql show grants logs;
+sql show cluster machines;
+sql show queries
+sql show connections
+sql show apps
+sql show transactions
+sql show create database d2
+sql show create table d2.stb2;
+sql show create table d2.ctb2;
+sql show create table d2.ntb2;
+sql show local variables;
+sql show dnode 1 variables;
+sql show variables;
+
+
+print =============== check information_schema of sysinfo 1
+sql show databases
+if $rows != 3 then
+ return -1
+endi
+
+sql use information_schema;
+sql select * from information_schema.ins_dnodes
+sql select * from information_schema.ins_mnodes
+sql_error select * from information_schema.ins_modules
+sql select * from information_schema.ins_qnodes
+sql select * from information_schema.ins_cluster
+sql select * from information_schema.ins_users
+sql select * from information_schema.ins_user_privileges
+sql select * from information_schema.ins_databases
+sql select * from information_schema.ins_functions
+sql select * from information_schema.ins_indexes
+sql select * from information_schema.ins_stables
+sql select * from information_schema.ins_tables
+sql select * from information_schema.ins_tags
+sql select * from information_schema.ins_topics
+sql select * from information_schema.ins_subscriptions
+sql select * from information_schema.ins_streams
+sql select * from information_schema.ins_grants
+sql select * from information_schema.ins_grants_full
+sql select * from information_schema.ins_grants_logs
+sql select * from information_schema.ins_machines
+sql select * from information_schema.ins_vgroups
+sql select * from information_schema.ins_configs
+sql select * from information_schema.ins_dnode_variables
+
+print =============== check performance_schema of sysinfo 1
+sql use performance_schema;
+sql select * from performance_schema.perf_connections
+sql select * from performance_schema.perf_queries
+sql select * from performance_schema.perf_consumers
+sql select * from performance_schema.perf_trans
+sql select * from performance_schema.perf_apps
+
+loop_check_switch:
+if $loop_idx > $loop_cnt then
+ goto loop_check_end
+endi
+$loop_idx = $loop_idx + 1
+
+if $loop_flag == 0 then
+ system taos -P7100 -s 'alter user sysinfo0 sysinfo 1'
+ sleep 3000
+ $loop_flag = 1
+ goto loop_check_sysinfo_1
+else
+ system taos -P7100 -s 'alter user sysinfo0 sysinfo 0'
+ sleep 3000
+ $loop_flag = 0
+ goto loop_check_sysinfo_0
+endi
+
+loop_check_end:
+print =============== check end
+
#system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py
index 169c70e675..690de65dbb 100644
--- a/tests/system-test/0-others/taosdMonitor.py
+++ b/tests/system-test/0-others/taosdMonitor.py
@@ -267,6 +267,7 @@ class TDTestCase:
updatecfgDict["monitorInterval"] = "5"
updatecfgDict["monitorMaxLogs"] = "10"
updatecfgDict["monitorComp"] = "1"
+ updatecfgDict["monitorForceV2"] = "0"
updatecfgDict["audit"] = '0'
diff --git a/tests/system-test/2-query/tms_memleak.py b/tests/system-test/2-query/tms_memleak.py
new file mode 100644
index 0000000000..0d5cdd8272
--- /dev/null
+++ b/tests/system-test/2-query/tms_memleak.py
@@ -0,0 +1,51 @@
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from math import inf
+
+class TDTestCase:
+ def caseDescription(self):
+ '''
+ case1: [TD-]
+ '''
+ return
+
+ def init(self, conn, logSql, replicaVer=1):
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor(), True)
+ self.conn = conn
+
+ def restartTaosd(self, index=1, dbname="db"):
+ tdDnodes.stop(index)
+ tdDnodes.startWithoutSleep(index)
+ tdSql.execute(f"use tms_memleak")
+
+ def run(self):
+ print("running {}".format(__file__))
+ tdSql.execute("drop database if exists tms_memleak")
+ tdSql.execute("create database if not exists tms_memleak")
+ tdSql.execute('use tms_memleak')
+
+ tdSql.execute('create table st(ts timestamp, f int) tags (t int);')
+
+ tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)('2021-04-19 00:00:02', 2)('2021-04-19 00:00:03', 3)('2021-04-19 00:00:04', 4)")
+
+ tdSql.execute("insert into ct2 using st tags(2) values('2021-04-20 00:00:01', 5)('2021-04-20 00:00:02', 6)('2021-04-20 00:00:03', 7)('2021-04-20 00:00:04', 8)")
+
+ tdSql.execute("insert into ct3 using st tags(3) values('2021-04-21 00:00:01', 5)('2021-04-21 00:00:02', 6)('2021-04-21 00:00:03', 7)('2021-04-21 00:00:04', 8)")
+
+ tdSql.execute("insert into ct4 using st tags(4) values('2021-04-22 00:00:01', 5)('2021-04-22 00:00:02', 6)('2021-04-22 00:00:03', 7)('2021-04-22 00:00:04', 8)")
+
+ tdSql.query("select * from st order by ts limit 1 ");
+ tdSql.checkRows(1)
+ tdSql.checkData(0, 1, 1);
+
+ tdSql.execute('drop database tms_memleak')
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index ac8a6f377c..c547385d70 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -323,7 +323,7 @@ class TDTestCase:
tdSql.query("select * from st")
tdSql.checkRows(8)
- tdSql.execute(f'create topic topic_excluded with meta as database d1')
+ tdSql.execute(f'create topic topic_all with meta as database d1')
consumer_dict = {
"group.id": "g1",
"td.connect.user": "root",
@@ -333,7 +333,7 @@ class TDTestCase:
consumer = Consumer(consumer_dict)
try:
- consumer.subscribe(["topic_excluded"])
+ consumer.subscribe(["topic_all"])
except TmqError:
tdLog.exit(f"subscribe error")
diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c
index e9b9b9e944..b6917e5a76 100644
--- a/tools/shell/src/shellAuto.c
+++ b/tools/shell/src/shellAuto.c
@@ -105,7 +105,8 @@ SWords shellCommands[] = {
{"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL},
{"create user pass sysinfo 0;", 0, 0, NULL},
{"create user pass sysinfo 1;", 0, 0, NULL},
-#ifdef TD_ENTERPRISE
+#ifdef TD_ENTERPRISE
+ {"create view as select", 0, 0, NULL},
{"compact database ", 0, 0, NULL},
#endif
{"describe ", 0, 0, NULL},
@@ -162,13 +163,20 @@ SWords shellCommands[] = {
{"show create database \\G;", 0, 0, NULL},
{"show create stable \\G;", 0, 0, NULL},
{"show create table \\G;", 0, 0, NULL},
+#ifdef TD_ENTERPRISE
+ {"show create view \\G;", 0, 0, NULL},
+#endif
{"show connections;", 0, 0, NULL},
+ {"show compact", 0, 0, NULL},
+ {"show compacts;", 0, 0, NULL},
{"show cluster;", 0, 0, NULL},
{"show cluster alive;", 0, 0, NULL},
+ {"show cluster machines;", 0, 0, NULL},
{"show databases;", 0, 0, NULL},
{"show dnodes;", 0, 0, NULL},
{"show dnode variables;", 0, 0, NULL},
{"show functions;", 0, 0, NULL},
+ {"show licences;", 0, 0, NULL},
{"show mnodes;", 0, 0, NULL},
{"show queries;", 0, 0, NULL},
// 80
@@ -185,6 +193,7 @@ SWords shellCommands[] = {
{"show table distributed ", 0, 0, NULL},
{"show tags from ", 0, 0, NULL},
{"show tags from ", 0, 0, NULL},
+ {"show table tags from ", 0, 0, NULL},
{"show topics;", 0, 0, NULL},
{"show transactions;", 0, 0, NULL},
{"show users;", 0, 0, NULL},
@@ -194,6 +203,8 @@ SWords shellCommands[] = {
{"show vgroups;", 0, 0, NULL},
{"show consumers;", 0, 0, NULL},
{"show grants;", 0, 0, NULL},
+ {"show grants full;", 0, 0, NULL},
+ {"show grants logs;", 0, 0, NULL},
#ifdef TD_ENTERPRISE
{"split vgroup ", 0, 0, NULL},
#endif
@@ -302,6 +313,20 @@ char* key_systable[] = {
char* udf_language[] = {"\'Python\'", "\'C\'"};
+// global keys can tips on anywhere
+char* global_keys[] = {
+ "tbname",
+ "now",
+ "_wstart",
+ "_wend",
+ "_wduration",
+ "_qstart",
+ "_qend",
+ "_qduration",
+ "_qtag",
+ "_isfilled"
+ };
+
//
// ------- global variant define ---------
//
@@ -341,8 +366,9 @@ bool waitAutoFill = false;
#define WT_VAR_KEYSELECT 20
#define WT_VAR_SYSTABLE 21
#define WT_VAR_LANGUAGE 22
+#define WT_VAR_GLOBALKEYS 23
-#define WT_VAR_CNT 23
+#define WT_VAR_CNT 24
#define WT_TEXT 0xFF
@@ -494,10 +520,12 @@ void showHelp() {
show connections;\n\
show cluster;\n\
show cluster alive;\n\
+ show cluster machines;\n\
show databases;\n\
show dnodes;\n\
show dnode variables;\n\
show functions;\n\
+ show licences;\n\
show mnodes;\n\
show queries;\n\
show query ;\n\
@@ -513,6 +541,7 @@ void showHelp() {
show table distributed ;\n\
show tags from \n\
show tags from \n\
+ show table tags from \n\
show topics;\n\
show transactions;\n\
show users;\n\
@@ -522,6 +551,8 @@ void showHelp() {
show vgroups;\n\
show consumers;\n\
show grants;\n\
+ show grants full;\n\
+ show grants logs;\n\
----- T ----- \n\
trim database ;\n\
----- U ----- \n\
@@ -534,8 +565,12 @@ void showHelp() {
balance vgroup ;\n\
balance vgroup leader on \n\
compact database ; \n\
+ crate view as select ...\n\
redistribute vgroup dnode ;\n\
- split vgroup ;");
+ split vgroup ;\n\
+ show compacts;\n\
+ show compact \n\
+ show create view ;");
#endif
printf("\n\n");
@@ -699,6 +734,7 @@ bool shellAutoInit() {
GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*));
GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*));
GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*));
+ GenerateVarType(WT_VAR_GLOBALKEYS, global_keys, sizeof(global_keys) / sizeof(char*));
return true;
}
@@ -1800,6 +1836,13 @@ bool matchEnd(TAOS* con, SShellCmd* cmd) {
goto _return;
}
+ // global keys
+ if (fillWithType(con, cmd, last, WT_VAR_GLOBALKEYS)) {
+ ret = true;
+ goto _return;
+ }
+
+
_return:
taosMemoryFree(ps);
return ret;
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 056b7dc6cf..2257089f06 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -20,6 +20,7 @@
#include
#include "taos.h"
#include "types.h"
+#include "tmsg.h"
static int running = 1;
TdFilePtr g_fp = NULL;
@@ -966,7 +967,14 @@ void testConsumeExcluded(int topic_type){
tmq_raw_data raw = {0};
tmq_get_raw(msg, &raw);
if(topic_type == 1){
- assert(raw.raw_type != 2 && raw.raw_type != 4);
+ assert(raw.raw_type != 2 && raw.raw_type != 4 &&
+ raw.raw_type != TDMT_VND_CREATE_STB &&
+ raw.raw_type != TDMT_VND_ALTER_STB &&
+ raw.raw_type != TDMT_VND_CREATE_TABLE &&
+ raw.raw_type != TDMT_VND_ALTER_TABLE &&
+ raw.raw_type != TDMT_VND_DELETE);
+ assert(raw.raw_type == TDMT_VND_DROP_STB ||
+ raw.raw_type == TDMT_VND_DROP_TABLE);
}else if(topic_type == 2){
assert(0);
}