diff --git a/cmake/cmake.version b/cmake/cmake.version index 0e35fa316f..4abc854e71 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.3.0.alpha") + SET(TD_VER_NUMBER "3.2.4.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index dbad0112cf..37077ef63b 100644 Binary files a/deps/arm/dm_static/libdmodule.a and b/deps/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/arm/dm_static/libdmodule.a b/deps/darwin/arm/dm_static/libdmodule.a index 2aab587b18..246b2247af 100644 Binary files a/deps/darwin/arm/dm_static/libdmodule.a and b/deps/darwin/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/x64/dm_static/libdmodule.a b/deps/darwin/x64/dm_static/libdmodule.a index 1fb6794f65..8745f57636 100644 Binary files a/deps/darwin/x64/dm_static/libdmodule.a and b/deps/darwin/x64/dm_static/libdmodule.a differ diff --git a/deps/mips/dm_static/libdmodule.a b/deps/mips/dm_static/libdmodule.a index d4b0582498..855a6a41d9 100644 Binary files a/deps/mips/dm_static/libdmodule.a and b/deps/mips/dm_static/libdmodule.a differ diff --git a/deps/x86/dm_static/libdmodule.a b/deps/x86/dm_static/libdmodule.a index 9d37818a79..6a3c0d45c2 100644 Binary files a/deps/x86/dm_static/libdmodule.a and b/deps/x86/dm_static/libdmodule.a differ diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index aacfd0fe53..3110afcf10 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) In addition to native connections, the client library also supports subscriptions via websockets. -The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). +The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll` ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -902,16 +902,14 @@ while True: -The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 6f2343d347..e1bf18c854 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -41,16 +41,28 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. +`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. + +`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. + For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## Partitions of Stream diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f5a4789976..902e62de73 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index e4dc177388..881838f8f0 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -15,11 +15,11 @@ import Node from "./_sub_node.mdx"; import CSharp from "./_sub_cs.mdx"; import CDemo from "./_sub_c.mdx"; - 为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine 提供了类似 kafka 的数据订阅功能。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。 -# 介绍 -## 主题 +## 数据订阅介绍 + +### 主题 与 kafka 一样，你需要定义 topic, TDengine 的 topic 有三种，可以是数据库，超级表，或者一个 `SELECT` 语句，具体的语法参见 [CREATE TOPIC](../../taos-sql/tmq)。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。 如下图，每个 topic 涉及到的数据表可能分布在多个 vnode（相当于 kafka 里的 partition） 上，每个 vnode 上的数据保存在 WAL(Write-Ahead-Log) 文件中，WAL 文件里的数据是顺序写入的（由于 WAL 文件中存储的不只有数据，还有元数据，写入消息等，所以数据的版本号不是连续的）。 @@ -30,11 +30,12 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 对于 `SELECT` 语句形式的 topic，在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。 -## 生产者 +### 生产者 写入 topic 相关联的数据表中数据的都是生产者，生产者实际生产的数据写入到了子表或普通表中，即表所在 vnode 的 WAL 里。 -## 消费者 -### 消费者组 +### 消费者 + +#### 消费者组 消费者订阅 topic 后，可以消费 topic 里的所有数据（这些数据所在的表可能分布在多个 vnode 上，即 db 所在的所有 vnode）。订阅 topic 时，需要指定一个消费者组 (consumer group)，如果这个消费者组里只有一个消费者，那么这个消费者会顺序的消费这些 vnode 上的数据。 为了提高消费速度，便于多线程、分布式地消费数据，可以在一个消费组里添加多个消费者，这些消费者将均分数据所在的 vnode 进行消费（比如数据分布在 4 个 vnode 上，有 2 个消费者的话，那么每个消费者消费 2 个 vnode；有 3 个消费者的话，2 个消费者各消费 1 个 vnode，1 个消费者消费 2 个 vnode；有 5 个消费者的话，4 个各分配 1 个 vnode 消费，另外 1 个不消费），如下图： @@ -44,7 +45,8 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 在一个消费组里添加一个消费者后，在 Mnode 上通过 rebalance 的机制实现消费者的重新分配，该操作对用户是透明的。 一个消费者可以订阅多个 topic。TDengine 的数据订阅在宕机、重启等复杂环境下确保 at least once 消费。 -### 消费进度 + +#### 消费进度 在 topic 的一个消费组的一个 vnode 上有消费进度。消费者消费的同时，可以提交消费进度，消费进度即 vnode 上 WAL 的版本号（对于 kafka 里的 offset），消费进度可以手动提交，也可以通过参数（auto.commit.interval.ms）设置为周期性自动提交。 首次消费数据时通过订阅参数（auto.offset.reset）来确定消费位置为最新数据（latest）还是最旧数据（earliest）。 @@ -59,16 +61,16 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 作为一个数据库产品， WAL 文件中存储的不全是数据，也包括其他写入消息，元数据等，所以消费进度不是连续的。 -##说明 +### 说明 从3.2.0.0版本开始，数据订阅支持vnode迁移和分裂。 由于数据订阅依赖wal文件，而在vnode迁移和分裂的过程中，wal并不会同步过去，所以迁移或分裂后，之前没消费完的wal数据后消费不到。所以请保证迁移和分裂之前把数据全部消费完后，再进行vnode迁移或分裂，否则，消费会丢失数据。 -# 语法说明 +## 数据订阅语法说明 具体的语法参见 [数据订阅](../../taos-sql/tmq) -# 消费参数 +## 数据订阅相关参数 消费参数主要用于消费者创建时指定，基础配置项如下表所示： @@ -86,9 +88,9 @@ TDengine 会为 WAL 文件自动创建索引以支持快速随机访问，并提 | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true） |默认关闭 | | `enable.replay` | boolean | 是否开启数据回放功能 |默认关闭 | -# 主要数据结构和 API 接口 +## 数据订阅主要 API 接口 -不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）： +不同语言下， TMQ 订阅相关的 API 及数据结构如下（详细的接口说明可以参考连接器章节数据订阅部分，注意consumer结构不是线程安全的，在一个线程使用consumer时，不要在另一个线程close这个consumer）： @@ -310,8 +312,8 @@ void Close() -# 数据订阅示例 -## 写入数据 +## 数据订阅示例 +### 写入数据 首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如： @@ -324,7 +326,7 @@ CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00'); INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); ``` -## 创建 topic +### 创建 topic 使用 SQL 创建一个 topic： @@ -332,7 +334,7 @@ INSERT INTO tmqdb.ctb1 VALUES(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1; ``` -## 创建消费者 *consumer* +### 创建消费者 consumer 对于不同编程语言，其设置方式如下： @@ -499,7 +501,7 @@ var consumer = new ConsumerBuilder>(cfg).Build(); 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。 -## 订阅 *topics* +### 订阅 topics 一个 consumer 支持同时订阅多个 topic。 @@ -578,7 +580,7 @@ consumer.Subscribe(topics); -## 消费 +### 消费 以下代码展示了不同语言下如何对 TMQ 消息进行消费。 @@ -714,7 +716,7 @@ while (true) -## 结束消费 +### 结束消费 消费结束后，应当取消订阅。 @@ -795,7 +797,7 @@ consumer.Close(); -## 完整示例代码 +### 完整示例代码 以下是各语言的完整示例代码。 @@ -838,8 +840,8 @@ consumer.Close(); -#订阅高级功能 -##数据回放 +## 数据订阅高级功能 +### 数据回放 - 订阅支持 replay 功能，按照数据写入的时间回放。 比如，如下时间写入三条数据 ```sql diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx index 60810ec275..c0e9e6c937 100644 --- a/docs/zh/07-develop/_sub_java.mdx +++ b/docs/zh/07-develop/_sub_java.mdx @@ -6,3 +6,4 @@ ``` ```java {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} +``` diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index e7160ab094..71dc82316e 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创 ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` @@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -911,16 +911,14 @@ while True: -Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。 ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 979fc436b9..8868b728f8 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -49,10 +49,14 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) @@ -61,6 +65,12 @@ window_clause: { ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## 流式计算的 partition diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b0a81e01a1..1c51f934fe 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.3.0 + + + ## 3.2.2.0 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ef3e61e1a2..93f17fa887 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -106,6 +106,9 @@ extern char tsMonitorFqdn[]; extern uint16_t tsMonitorPort; extern int32_t tsMonitorMaxLogs; extern bool tsMonitorComp; +extern bool tsMonitorLogProtocol; +extern int32_t tsMonitorIntervalForBasic; +extern bool tsMonitorForceV2; // audit extern bool tsEnableAudit; @@ -230,10 +233,10 @@ int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); -void taosSetAllDebugFlag(int32_t flag); -void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); -void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(int8_t type); +void taosSetGlobalDebugFlag(int32_t flag); +void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); +void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 6d11608d21..ca9e030117 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -67,28 +67,28 @@ int32_t grantCheckLE(EGrantType grant); // #ifndef GRANTS_CFG #ifdef TD_ENTERPRISE -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #else -#define GRANTS_SCHEMA \ - static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ +#define GRANTS_SCHEMA \ + static const SSysDbTableSchema grantsSchema[] = { \ + {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #endif // #define GRANT_CFG_ADD diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fe53217eac..958789178a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -581,9 +581,9 @@ typedef struct { }; } SSubmitRsp; -int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); -int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitBlkRsp(void* param); +// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); +// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); +// void tFreeSSubmitBlkRsp(void* param); void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) @@ -885,8 +885,8 @@ typedef struct { int64_t maxStorage; } SCreateAcctReq, SAlterAcctReq; -int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); -int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { char user[TSDB_USER_LEN]; @@ -1324,14 +1324,13 @@ typedef struct { } SDnodeListReq; int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); -int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); typedef struct { int32_t useless; // useless } SServerVerReq; int32_t tSerializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); -int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); +// int32_t tDeserializeSServerVerReq(void* buf, int32_t bufLen, SServerVerReq* pReq); typedef struct { char ver[TSDB_VERSION_LEN]; @@ -1582,6 +1581,15 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t contLen; + char* pCont; +} SStatisReq; + +int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); +void tFreeSStatisReq(SStatisReq *pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; @@ -1629,14 +1637,14 @@ typedef struct { } SMTimerReq; int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); -int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); +// int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); typedef struct { int64_t tick; } SMStreamTickReq; int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); -int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); +// int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq); typedef struct { int32_t id; @@ -1907,7 +1915,7 @@ typedef struct { } SShowVariablesReq; int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); -int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); +// int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; @@ -1937,7 +1945,7 @@ typedef struct { } SShowReq; int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); -int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); void tFreeSShowReq(SShowReq* pReq); typedef struct { @@ -1945,9 +1953,9 @@ typedef struct { STableMetaRsp tableMeta; } SShowRsp, SVShowTablesRsp; -int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); -void tFreeSShowRsp(SShowRsp* pRsp); +// int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +// void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -2205,7 +2213,7 @@ typedef struct { } SForceBecomeFollowerReq; int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); -int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); +// int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); typedef struct { int32_t vgId; @@ -2222,8 +2230,8 @@ typedef struct { char ckey[TSDB_PASSWORD_LEN]; } SAuthReq, SAuthRsp; -int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); -int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +// int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); typedef struct { int32_t statusCode; @@ -2756,8 +2764,8 @@ typedef struct { int tEncodeSVCreateTbBatchRsp(SEncoder* pCoder, const SVCreateTbBatchRsp* pRsp); int tDecodeSVCreateTbBatchRsp(SDecoder* pCoder, SVCreateTbBatchRsp* pRsp); -int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); -int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +// int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); // TDMT_VND_DROP_TABLE ================= typedef struct { @@ -3212,8 +3220,8 @@ typedef struct { int32_t hasCheckPoint; } SMVStreamGatherInfoReq; -int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); -int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); +// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); typedef struct { int64_t leftForVer; @@ -3438,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR typedef SMDropSmaReq SDropTagIndexReq; -int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); typedef struct { @@ -3490,7 +3498,7 @@ void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq); -int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); +// int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); typedef struct { int32_t number; @@ -3559,8 +3567,8 @@ typedef struct { int8_t igNotExists; } SMDropFullTextReq; -int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); -int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); typedef struct { char indexFName[TSDB_INDEX_FNAME_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index dc5cb24564..62392a9caa 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -215,6 +215,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STATIS, "statis", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) diff --git a/include/common/trow.h b/include/common/trow.h index 8332c10ed2..92713e8e63 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -289,7 +289,7 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { - pBuilder->rowType = TD_ROW_TP; // default STpRow + pBuilder->rowType = pBuilder->rowType; pBuilder->sver = sver; } int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); @@ -331,7 +331,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType); bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); diff --git a/include/libs/monitor/clientMonitor.h b/include/libs/monitor/clientMonitor.h new file mode 100644 index 0000000000..9c0302a15f --- /dev/null +++ b/include/libs/monitor/clientMonitor.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENT_MONITOR_H +#define TDENGINE_CLIENT_MONITOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos_monitor.h" +#include "thash.h" +#include "query.h" + +typedef enum SQL_RESULT_CODE { + SQL_RESULT_SUCCESS = 0, + SQL_RESULT_FAILED = 1, + SQL_RESULT_CANCEL = 2, +} SQL_RESULT_CODE; + +const char* resultStr(SQL_RESULT_CODE code); + +typedef struct { + char clusterKey[512]; + SEpSet epSet; + void* pTransporter; + taos_collector_registry_t* registry; + taos_collector_t* colector; + SHashObj* counters; +} ClientMonitor; + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter); +void clusterMonitorClose(const char* clusterKey); +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys); +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values); +void cluster_monitor_stop(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENT_MONITOR_H diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 91b3a54ea1..256be26999 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -76,7 +76,8 @@ typedef struct { } SMonBasicInfo; typedef struct { - float uptime; // day + //float uptime; // day + int64_t uptime; // second int8_t has_mnode; int8_t has_qnode; int8_t has_snode; @@ -100,13 +101,15 @@ typedef struct { int32_t mnode_id; char mnode_ep[TSDB_EP_LEN]; char role[MON_ROLE_LEN]; + int32_t syncState; } SMonMnodeDesc; typedef struct { char first_ep[TSDB_EP_LEN]; int32_t first_ep_dnode_id; char version[MON_VER_LEN]; - float master_uptime; // day + //float master_uptime; // day + int64_t master_uptime; //second int32_t monitor_interval; // sec int32_t dbs_total; int32_t stbs_total; @@ -125,6 +128,7 @@ typedef struct { typedef struct { int32_t dnode_id; char vnode_role[MON_ROLE_LEN]; + int32_t syncState; } SMonVnodeDesc; typedef struct { @@ -149,7 +153,7 @@ typedef struct { } SMonStbInfo; typedef struct { - uint32_t expire_time; + int64_t expire_time; int64_t timeseries_used; int64_t timeseries_total; } SMonGrantInfo; @@ -221,7 +225,9 @@ void monSetVmInfo(SMonVmInfo *pInfo); void monSetQmInfo(SMonQmInfo *pInfo); void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); -void monSendReport(); +void monGenAndSendReport(); +void monGenAndSendReportBasic(); +void monSendContent(char *pCont); void tFreeSMonMmInfo(SMonMmInfo *pInfo); void tFreeSMonVmInfo(SMonVmInfo *pInfo); diff --git a/include/libs/monitorfw/taos_alloc.h b/include/libs/monitorfw/taos_alloc.h new file mode 100644 index 0000000000..893ffc7e9b --- /dev/null +++ b/include/libs/monitorfw/taos_alloc.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_alloc.h + * @brief memory management + */ + +#ifndef TAOS_ALLOC_H +#define TAOS_ALLOC_H + +#include +#include + +/** + * @brief Redefine this macro if you wish to override it. The default value is malloc. + */ +#define taos_malloc malloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is realloc. + */ +#define taos_realloc realloc + +/** + * @brief Redefine this macro if you wish to override it. The default value is strdup. + */ +#define taos_strdup strdup + +/** + * @brief Redefine this macro if you wish to override it. The default value is free. + */ +#define taos_free free + +#endif // TAOS_ALLOC_H diff --git a/include/libs/monitorfw/taos_collector.h b/include/libs/monitorfw/taos_collector.h new file mode 100644 index 0000000000..c37cb612d2 --- /dev/null +++ b/include/libs/monitorfw/taos_collector.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_H +#define TAOS_COLLECTOR_H + +#include "taos_map.h" +#include "taos_metric.h" + +/** + * @file taos_collector.h + * @brief A Prometheus collector returns a collection of metrics + */ + +/** + * @brief A prometheus collector calls collect to prepare metrics and return them to the registry to which it is + * registered. + */ +typedef struct taos_collector taos_collector_t; + +/** + * @brief The function responsible for preparing metric data and returning metrics for a given collector. + * + * If you use the default collector registry, this should not concern you. If you are using a custom collector, you may + * set this function on your collector to do additional work before returning the contained metrics. + * + * @param self The target taos_collector_t* + * @return The taos_map_t* containing the collected metrics + */ +typedef taos_map_t *taos_collect_fn(taos_collector_t *self); + +/** + * @brief Create a collector + * @param name The name of the collector. The name MUST NOT be default or process. + * @return The constructed taos_collector_t* + */ +taos_collector_t *taos_collector_new(const char *name); + +/** + * @brief Destroy a collector. You MUST set self to NULL after destruction. + * @param self The target taos_collector_t* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy(taos_collector_t *self); + +/** + * @brief Frees a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + */ +void taos_collector_free_generic(void *gen); + +/** + * @brief Destroys a collector passed as a void pointer. You MUST set self to NULL after destruction. + * @param gen The target taos_collector_t* represented as a void* + * @return A non-zero integer value upon failure. + */ +int taos_collector_destroy_generic(void *gen); + +/** + * @brief Add a metric to a collector + * @param self The target taos_collector_t* + * @param metric the taos_metric_t* to add to the taos_collector_t* passed as self. + * @return A non-zero integer value upon failure. + */ +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric); + +int taos_collector_remove_metric(taos_collector_t *self, const char* key); + +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name); + +/** + * @brief The collect function is responsible for doing any work involving a set of metrics and then returning them + * for metric exposition. + * @param self The target taos_collector_t* + * @param fn The taos_collect_fn* which will be responsible for handling any metric collection operations before + * returning the collected metrics for exposition. + * @return A non-zero integer value upon failure. + */ +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn); + +#endif // TAOS_COLLECTOR_H diff --git a/include/libs/monitorfw/taos_collector_registry.h b/include/libs/monitorfw/taos_collector_registry.h new file mode 100644 index 0000000000..81d1f8050c --- /dev/null +++ b/include/libs/monitorfw/taos_collector_registry.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_collector_registry.h + * @brief The collector registry registers collectors for metric exposition. + */ + +#ifndef TAOS_REGISTRY_H +#define TAOS_REGISTRY_H + +#include "taos_collector.h" +#include "taos_metric.h" + +/** + * @brief A taos_registry_t is responsible for registering metrics and briding them to the string exposition format + */ +typedef struct taos_collector_registry taos_collector_registry_t; + +/** + * @brief Initialize the default registry by calling taos_collector_registry_init within your program. You MUST NOT + * modify this value. + */ +extern taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +/** + * @brief Initializes the default collector registry and enables metric collection on the executing process + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_default_init(void); + +/** + * @brief Constructs a taos_collector_registry_t* + * @param name The name of the collector registry. It MUST NOT be default. + * @return The constructed taos_collector_registry_t* + */ +taos_collector_registry_t *taos_collector_registry_new(const char *name); + +/** + * @brief Destroy a collector registry. You MUST set self to NULL after destruction. + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_destroy(taos_collector_registry_t *self); + +/** + * @brief Enable process metrics on the given collector registry + * @param self The target taos_collector_registry_t* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_enable_process_metrics(taos_collector_registry_t *self); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY + * + * The metric to be registered MUST NOT already be registered with the given . If so, the program will + * halt. It returns a taos_metric_t* to simplify metric creation and registration. Furthermore, + * TAOS_DEFAULT_COLLECTOR_REGISTRY must be registered via taos_collector_registry_default_init() prior to calling this + * function. The metric will be added to the default registry's default collector. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return The registered taos_metric_t* + */ +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric); + +/** + * @brief Registers a metric with the default collector on TAOS_DEFAULT_COLLECTOR_REGISTRY. Returns an non-zero integer + * value on failure. + * + * See taos_collector_registry_must_register_metric. + * + * @param metric The metric to register on TAOS_DEFAULT_COLLECTOR_REGISTRY* + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_metric(taos_metric_t *metric); + +int taos_collector_registry_deregister_metric(const char *key); + +taos_metric_t *taos_collector_registry_get_metric(char* metric_name); + +/** + * @brief Register a collector with the given registry. Returns a non-zero integer value on failure. + * @param self The target taos_collector_registry_t* + * @param collector The taos_collector_t* to register onto the taos_collector_registry_t* as self + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector); + +/** + * @brief Returns a string in the default metric exposition format. The string MUST be freed to avoid unnecessary heap + * memory growth. + * + * Reference: https://prometheus.io/docs/instrumenting/exposition_formats/ + * + * @param self The target taos_collector_registry_t* + * @return The string int he default metric exposition format. + */ +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format); + +int taos_collector_registry_clear_batch(taos_collector_registry_t *self); + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str); + +/** + *@brief Validates that the given metric name complies with the specification: + * + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + * + * Returns a non-zero integer value on failure. + * + * @param self The target taos_collector_registry_t* + * @param metric_name The metric name to validate + * @return A non-zero integer value upon failure + */ +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name); + +#endif // TAOS_H diff --git a/include/libs/monitorfw/taos_counter.h b/include/libs/monitorfw/taos_counter.h new file mode 100644 index 0000000000..28a9eed41c --- /dev/null +++ b/include/libs/monitorfw/taos_counter.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COUNTER_H +#define TAOS_COUNTER_H + +#include + +#include "taos_metric.h" + +/** + * @file taos_counter.h + * @brief https://prometheus.io/docs/concepts/metric_types/#counter + */ + +/** + * @brief A prometheus counter. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#counter + */ +typedef taos_metric_t taos_counter_t; + +/** + * @brief Construct a taos_counter_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_counter_t* + * + * *Example* + * + * // An example with labels + * taos_counter_new("foo", "foo is a counter with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_counter_new("foo", "foo is a counter without labels", 0, NULL); + */ +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_counter_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self A taos_counter_t* + * @return A non-zero integer value upon failure. + */ +int taos_counter_destroy(taos_counter_t *self); + +/** + * @brief Increment the taos_counter_t by 1. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_inc(foo_counter, (const char**) { "bar", "bang" }); + ** + * // An example without labels + * taos_counter_inc(foo_counter, NULL); + */ +int taos_counter_inc(taos_counter_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_counter_t*. A non-zero integer value will be returned on failure. + * @param self The target taos_counter_t* + * @param r_value The double to add to the taos_counter_t passed as self. The value MUST be greater than or equal to 0. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_counter_add(foo_counter, 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_counter_add(foo_counter, 22, NULL); + */ +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values); + +#endif // TAOS_COUNTER_H diff --git a/include/libs/monitorfw/taos_gauge.h b/include/libs/monitorfw/taos_gauge.h new file mode 100644 index 0000000000..a33661dfe4 --- /dev/null +++ b/include/libs/monitorfw/taos_gauge.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_gauge.h + * @brief https://prometheus.io/docs/concepts/metric_types/#gauge + */ + +#ifndef TAOS_GAUGE_H +#define TAOS_GAUGE_H + +#include + +#include "taos_metric.h" + +/** + * @brief A prometheus gauge. + * + * References + * * See https://prometheus.io/docs/concepts/metric_types/#gauge + */ +typedef taos_metric_t taos_gauge_t; + +/** + * @brief Constructs a taos_gauge_t* + * @param name The name of the metric + * @param help The metric description + * @param label_key_count The number of labels associated with the given metric. Pass 0 if the metric does not + * require labels. + * @param label_keys A collection of label keys. The number of keys MUST match the value passed as label_key_count. If + * no labels are required, pass NULL. Otherwise, it may be convenient to pass this value as a + * literal. + * @return The constructed taos_guage_t* + * + * // An example with labels + * taos_gauge_new("foo", "foo is a gauge with labels", 2, (const char**) { "one", "two" }); + * + * // An example without labels + * taos_gauge_new("foo", "foo is a gauge without labels", 0, NULL); + */ +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys); + +/** + * @brief Destroys a taos_gauge_t*. You must set self to NULL after destruction. A non-zero integer value will be + * returned on failure. + * @param self The target taos_gauge_t* + * @return A non-zero integer value upon failure + */ +int taos_gauge_destroy(taos_gauge_t *self); + +/** + * @brief Increment the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure + * *Example* + * + * // An example with labels + * taos_gauge_inc(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_inc(foo_gauge, NULL); + */ +int taos_gauge_inc(taos_gauge_t *self, const char **label_values); + +/** + * @brief Decrement the taos_gauge_t* by 1. + * @param self The target taos_gauger_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * *Example* + * + * // An example with labels + * taos_gauge_dec(foo_gauge, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_dec(foo_gauge, NULL); + */ +int taos_gauge_dec(taos_gauge_t *self, const char **label_values); + +/** + * @brief Add the value to the taos_gauge_t*. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_add(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_add(foo_gauge, 22, NULL); + */ +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Subtract the value to the taos_gauge. A non-zero integer value will be returned on failure. + * @param self The target taos_gauge_t* + * @param r_value The double to add to the taos_gauge_t passed as self. + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_sub(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_sub(foo_gauge, 22, NULL); + */ +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values); + +/** + * @brief Set the value for the taos_gauge_t* + * @param self The target taos_gauge_t* + * @param r_value The double to which the taos_gauge_t* passed as self will be set + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the gauge's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A non-zero integer value upon failure. + * + * *Example* + * + * // An example with labels + * taos_gauge_set(foo_gauge 22, (const char**) { "bar", "bang" }); + * + * // An example without labels + * taos_gauge_set(foo_gauge, 22, NULL); + */ +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values); + +#endif // TAOS_GAUGE_H diff --git a/include/libs/monitorfw/taos_linked_list.h b/include/libs/monitorfw/taos_linked_list.h new file mode 100644 index 0000000000..1fff5e65a6 --- /dev/null +++ b/include/libs/monitorfw/taos_linked_list.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_H +#define TAOS_LIST_H + +#include + +struct taos_linked_list; +/** + * @brief Provides a generic linked list + */ +typedef struct taos_linked_list taos_linked_list_t; + +#endif // TAOS_LIST_H diff --git a/include/libs/monitorfw/taos_map.h b/include/libs/monitorfw/taos_map.h new file mode 100644 index 0000000000..fa554d569c --- /dev/null +++ b/include/libs/monitorfw/taos_map.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_H +#define TAOS_MAP_H + +struct taos_map; +typedef struct taos_map taos_map_t; + +struct taos_map_node; +typedef struct taos_map_node taos_map_node_t; + +#endif // TAOS_MAP_H diff --git a/include/libs/monitorfw/taos_metric.h b/include/libs/monitorfw/taos_metric.h new file mode 100644 index 0000000000..a377311fa0 --- /dev/null +++ b/include/libs/monitorfw/taos_metric.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_METRIC_H +#define TAOS_METRIC_H + +#include "taos_metric_sample.h" + +struct taos_metric; +/** + * @brief A prometheus metric. + * + * Reference: https://prometheus.io/docs/concepts/data_model + */ +typedef struct taos_metric taos_metric_t; + +/** + * @brief Returns a taos_metric_sample_t*. The order of label_values is significant. + * + * You may use this function to cache metric samples to avoid sample lookup. Metric samples are stored in a hash map + * with O(1) lookups in average case; nonethless, caching metric samples and updating them directly might be + * preferrable in performance-sensitive situations. + * + * @param self The target taos_metric_t* + * @param label_values The label values associated with the metric sample being updated. The number of labels must + * match the value passed to label_key_count in the counter's constructor. If no label values are + * necessary, pass NULL. Otherwise, It may be convenient to pass this value as a literal. + * @return A taos_metric_sample_t* + */ +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values); + +#endif // TAOS_METRIC_H diff --git a/include/libs/monitorfw/taos_metric_sample.h b/include/libs/monitorfw/taos_metric_sample.h new file mode 100644 index 0000000000..0283d56ad9 --- /dev/null +++ b/include/libs/monitorfw/taos_metric_sample.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_metric_sample.h + * @brief Functions for interfacting with metric samples directly + */ + +#ifndef TAOS_METRIC_SAMPLE_H +#define TAOS_METRIC_SAMPLE_H + +struct taos_metric_sample; +/** + * @brief Contains the specific metric and value given the name and label set + * Reference: https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + */ +typedef struct taos_metric_sample taos_metric_sample_t; + +/** + * @brief Add the r_value to the sample. The value must be greater than or equal to zero. + * @param self The target taos_metric_sample_t* + * @param r_value The double to add to taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value); + +/** + * @brief Subtract the r_value from the sample. + * + * This operation MUST be called a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double to subtract from the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value); + +/** + * @brief Set the r_value of the sample. + * + * This operation MUST be called on a sample derived from a gauge metric. + * @param self The target taos_metric_sample_t* + * @param r_value The double which will be set to the taos_metric_sample_t* provided by self + * @return Non-zero integer value upon failure + */ +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value); + +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value); + +#endif // TAOS_METRIC_SAMPLE_H diff --git a/include/libs/monitorfw/taos_monitor.h b/include/libs/monitorfw/taos_monitor.h new file mode 100644 index 0000000000..c8569dd0a0 --- /dev/null +++ b/include/libs/monitorfw/taos_monitor.h @@ -0,0 +1,132 @@ +/* +Copyright 2019 DigitalOcean Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * @file taos_monitor.h + * @brief Include taos_monitor.h to include the entire public API + * @mainpage Welcome to the documentation site for prometheus-client-c! + * @tableofcontents + * @section Introduction + * + * prometheus-client-c is a small suite of Prometheus client libraries targeted for the C programming language. + * In this brief tutorial you will learn how to create and register metrics, update metric samples, and expose metrics + * over HTTP. + * + * @section Creating-and-Registering-Metrics Creating and Registering Metrics + * + * prometheus-client-c supports the following metric types: + * + * * [Counter](https://prometheus.io/docs/concepts/metric_types/#counter) + * * [Gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) + * * [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) + * + * To get started using one of the metric types, declare the metric at file scope. For example: + * + * @code{.c} + * + * #incldue "taos_monitor.h" + * + * taos_counter_t *my_counter; + * + * @endcode + * + * Next, create a metric initialization function. You can create the metric and register it with the default metric + * collector registry in one chain of functions. A metric collector is responsible for collecting metrics and returning + * them. A metric collector registry is declared in global scope and contains metric collectors. More on this later... + * + * To create a metric and register it with the default metric collector registry in one shot, you may chain the metric + * constructor into the taos_collector_registry_must_register_metric function. For example: + * + * @code{.c} + * + * void foo_metric_init(void) { + * my_counter = taos_collector_registry_must_register_metric(taos_counter_new("my_counter", "counts things", 0, NULL)); + * } + * + * @endcode + * + * The first argument to taos_counter_new is the counter name. The second argument is the counter description. The third + * argument is the number of metric labels. In this case, we will only have one metric sample for this metric so we pass + * 0 to specify that no labels will be used. The 4th argument is an array of strings storing the metric labels. Since we + * have none, we pass NULL. A call to foo_metric_init within the program's main function will initialize the metrics + * for the file we just created to the default prometheus metric collector registery called + * TAOS_COLLECTOR_REGISTRY_DEFAULT + * + * @section Updating-Metric-Sample-Values Updating Metric Sample Values + * + * Now that we have a metric configured for creation and registration, we can update our metric within any of the + * functions of the file in which it was declared. For example: + * + * @code{.c} + * + * void my_lib_do_something(void) { + * printf("I did a really important thing!\n"); + * taos_counter_inc(my_counter, NULL); + * } + * @endcode + * + * This function will increment the default metric sample for my_counter. Since we are not using metric labels, we pass + * NULL as the second argument. + * + * @section Program-Initialization Program Initialization + * + * At the start of the program's main function you need to do two things: + * + * * Initialize the default metric collector registry: + * + * @code{.c} + * + * taos_collector_registry_default_init(); + * + * @endcode + * + * * For each file containing prometheus metrics, call its corresponding metric initialization function + * + * @code{.c} + * + * foo_metric_init() + * + * @endcode + * + * After initialization is complete, you may proceed to do work and update your metrics. + * + * @section Metric-Exposition-Over-HTTP Metric Exposition Over HTTP + * + * @todo Describe how to use libpromhttp to expose metrics over HTTP + * + * @section Where-To-Go-From-Here Where to Go From Here? + * + * Take a look at the [Files](https://github.internal.digitalocean.com/pages/timeseries/prometheus-client-c/files.html) + * tab in this documentation site for more information about the public API available to you. Also, you can take a look + * at the examples directory at the + * [Github repository](https://github.internal.digitalocean.com/timeseries/prometheus-client-c) for inspiration. + */ + +#ifndef TAOS_INCLUDED +#define TAOS_INCLUDED + +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" +#include "taos_counter.h" +#include "taos_linked_list.h" +#include "taos_map.h" +#include "taos_metric.h" +#include "taos_metric_sample.h" +#include "taos_monitor_util.h" +#include "taos_gauge.h" + +#endif // TAOS_INCLUDED \ No newline at end of file diff --git a/include/libs/monitorfw/taos_monitor_util.h b/include/libs/monitorfw/taos_monitor_util.h new file mode 100644 index 0000000000..a42d1ddcef --- /dev/null +++ b/include/libs/monitorfw/taos_monitor_util.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * @file taos_monitor_util.h + * @brief Functions for retrieving metric samples from metrics given an ordered set of labels + */ + +#ifndef TAOS_MONITOR_UTIL_H +#define TAOS_MONITOR_UTIL_H + +#include "taos_metric.h" +#include "tjson.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del); +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf); +char* taos_monitor_get_metric_name(taos_metric_t* metric); + +#endif // TAOS_MONITOR_UTIL_H \ No newline at end of file diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 9fd00cefb4..48b7b8c56f 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -48,11 +48,13 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val); +double atomic_store_double(double volatile *ptr, double val); void atomic_store_ptr(void *ptr, void *val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); +double atomic_exchange_double(double volatile *ptr, int64_t val); void *atomic_exchange_ptr(void *ptr, void *val); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); @@ -68,6 +70,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_add_double(double volatile *ptr, double val); void *atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); @@ -78,6 +81,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); +double atomic_fetch_sub_double(double volatile *ptr, double val); void *atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c0c20a0fde..3a689fc63d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -433,7 +433,7 @@ int32_t* taosGetErrno(); //mnode-compact #define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1) - +#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2) // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x @@ -757,6 +757,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COL_QUERY_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x266D) #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E) #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F) +#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tdef.h b/include/util/tdef.h index 3fe3214ac4..e2d1beb5a5 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -187,6 +187,8 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define TSDB_INT32_ID_LEN 11 + #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -196,6 +198,8 @@ typedef enum ELogicConditionType { // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 +#define TSDB_NODE_ID_LEN 11 +#define TSDB_VGROUP_ID_LEN 11 #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns @@ -547,6 +551,10 @@ enum { #define VNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60 +#define MONITOR_TABLENAME_LEN 200 +#define MONITOR_TAG_NAME_LEN 100 +#define MONITOR_TAG_VALUE_LEN 300 +#define MONITOR_METRIC_NAME_LEN 100 #ifdef __cplusplus } #endif diff --git a/include/util/tjson.h b/include/util/tjson.h index 6922930c13..b9ea72b4bb 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -103,6 +103,7 @@ char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName); #ifdef __cplusplus } diff --git a/include/util/tlog.h b/include/util/tlog.h index 11ac0e1fae..af92e1864e 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -120,6 +120,7 @@ void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); #define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} #define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} #define uDebugL(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLongString("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uInfoL(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLongString("UTL ", DEBUG_INFO, uDebugFlag, __VA_ARGS__); }} #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index 2cf170cf04..d3ce2eb23b 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -26,6 +26,8 @@ typedef struct SScalableBf { SArray *bfArray; // array of bloom filters uint32_t growth; uint64_t numBits; + uint32_t maxBloomFilters; + int8_t status; _hash_fn_t hashFn1; _hash_fn_t hashFn2; } SScalableBf; diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index f089247859..a17c27c297 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -20,7 +20,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(TD_DARWIN_ARM64) @@ -61,7 +61,7 @@ target_include_directories( target_link_libraries( taos_static INTERFACE api - PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom geometry + PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index f73d121b15..989c6614a6 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -396,6 +396,8 @@ void hbRemoveAppHbMrg(SAppHbMgr** pAppHbMgr); void destroyAllRequests(SHashObj* pRequests); void stopAllRequests(SHashObj* pRequests); +SAppInstInfo* getAppInstInfo(const char* clusterKey); + // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(STscObj* pTscObj, SClientHbKey connKey); @@ -432,6 +434,21 @@ void freeQueryParam(SSyncQueryParam* param); int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes); #endif +void clientSlowQueryMonitorInit(const char* clusterKey); +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost); + +void clientSQLReqMonitorInit(const char* clusterKey); + +enum { + MONITORSQLTYPESELECT = 0, + MONITORSQLTYPEINSERT = 1, + MONITORSQLTYPEDELETE = 2 +}; + +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type); + +void clientMonitorClose(const char* clusterKey); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b732abffb1..122914fd34 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -80,7 +80,7 @@ extern "C" { #define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0) #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE)) + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) #define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 87f8c6be67..1df50a51da 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,6 +18,7 @@ #include "clientLog.h" #include "functionMgt.h" #include "os.h" +#include "osSleep.h" #include "query.h" #include "qworker.h" #include "scheduler.h" @@ -105,7 +106,15 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); reqType = SLOW_LOG_TYPE_QUERY; - } + } + } + + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); + } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPESELECT); + } else if (QUERY_NODE_DELETE_STMT == pRequest->stmtType) { + sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEDELETE); } if (duration >= (tsSlowLogThreshold * 1000000UL)) { @@ -114,6 +123,7 @@ static void deregisterRequest(SRequestObj *pRequest) { taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + SlowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); } } @@ -224,6 +234,7 @@ void destroyAppInst(SAppInstInfo *pAppInfo) { taosThreadMutexLock(&appInfo.mutex); + clientMonitorClose(pAppInfo->instKey); hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 68eb569dfc..de4f9f16e4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -15,6 +15,7 @@ #include "cJSON.h" #include "clientInt.h" +#include "clientMonitor.h" #include "clientLog.h" #include "command.h" #include "scheduler.h" @@ -157,6 +158,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port); pInst = &p; + + clientSlowQueryMonitorInit(p->instKey); + clientSQLReqMonitorInit(p->instKey); } else { ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL); // reset to 0 in case of conn with duplicated user key but its user has ever been dropped. @@ -166,9 +170,19 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas taosThreadMutexUnlock(&appInfo.mutex); taosMemoryFreeClear(key); + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType); } +SAppInstInfo* getAppInstInfo(const char* clusterKey) { + SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); + if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { + return *ppAppInstInfo; + } else { + return NULL; + } +} + void freeQueryParam(SSyncQueryParam* param) { if (param == NULL) return; tsem_destroy(¶m->sem); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f6aef5aa26..275ca0d2aa 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -279,6 +279,7 @@ void taos_close_internal(void *taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + // clientMonitorClose(pTscObj->pAppInfo->instKey); taosRemoveRef(clientConnRefPool, pTscObj->id); } diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 0c610a4611..7535cbfd0c 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -20,14 +20,14 @@ #include "clientSml.h" -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +#define IS_COMMA(sql,escapeChar) (*(sql) == COMMA && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) +#define IS_SPACE(sql,escapeChar) (*(sql) == SPACE && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) +#define IS_EQUAL(sql,escapeChar) (*(sql) == EQUAL && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar))) #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH)) #define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \ - (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL)) + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == SLASH)) #define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen) \ for (int i = 1; i < keyLen; ++i) { \ @@ -198,7 +198,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ int cnt = 0; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql))) { + if (unlikely(IS_SPACE(*sql,NULL))) { break; } @@ -207,18 +207,21 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ size_t keyLen = 0; bool keyEscaped = false; size_t keyLenEscaped = 0; + const char *escapeChar = NULL; + while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); terrno = TSDB_CODE_SML_INVALID_DATA; return -1; } - if (unlikely(IS_EQUAL(*sql))) { + if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; (*sql)++; break; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; keyLenEscaped++; keyEscaped = true; } @@ -238,15 +241,16 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ size_t valueLenEscaped = 0; while (*sql < sqlEnd) { // parse value - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { break; - } else if (unlikely(IS_EQUAL(*sql))) { + } else if (unlikely(IS_EQUAL(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); terrno = TSDB_CODE_SML_INVALID_DATA; return -1; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; valueLenEscaped++; valueEscaped = true; } @@ -293,7 +297,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){ } cnt++; - if (IS_SPACE(*sql)) { + if (IS_SPACE(*sql,escapeChar)) { break; } (*sql)++; @@ -326,7 +330,7 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) { int cnt = 0; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql))) { + if (unlikely(IS_SPACE(*sql,NULL))) { break; } @@ -335,17 +339,19 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL size_t keyLen = 0; bool keyEscaped = false; size_t keyLenEscaped = 0; + const char *escapeChar = NULL; while (*sql < sqlEnd) { - if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); return TSDB_CODE_SML_INVALID_DATA; } - if (unlikely(IS_EQUAL(*sql))) { + if (unlikely(IS_EQUAL(*sql,escapeChar))) { keyLen = *sql - key; (*sql)++; break; } if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { + escapeChar = *sql; keyLenEscaped++; keyEscaped = true; } @@ -363,7 +369,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL bool valueEscaped = false; size_t valueLenEscaped = 0; int quoteNum = 0; - const char *escapeChar = NULL; while (*sql < sqlEnd) { // parse value if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) { @@ -374,7 +379,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } continue; } - if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql)))) { + if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar)))) { break; } if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) { @@ -437,7 +442,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL } cnt++; - if (IS_SPACE(*sql)) { + if (IS_SPACE(*sql,escapeChar)) { break; } (*sql)++; @@ -453,19 +458,18 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine elements->measure = sql; // parse measure size_t measureLenEscaped = 0; + const char *escapeChar = NULL; while (sql < sqlEnd) { - if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { - elements->measureEscaped = true; - measureLenEscaped++; - sql++; - continue; - } - if (unlikely(IS_COMMA(sql))) { + if (unlikely(IS_COMMA(sql,escapeChar) || IS_SPACE(sql,escapeChar))) { break; } - if (unlikely(IS_SPACE(sql))) { - break; + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) { + elements->measureEscaped = true; + escapeChar = sql; + measureLenEscaped++; + sql++; + continue; } sql++; } @@ -478,9 +482,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine // to get measureTagsLen before const char *tmp = sql; while (tmp < sqlEnd) { - if (unlikely(IS_SPACE(tmp))) { + if (unlikely(IS_SPACE(tmp,escapeChar))) { break; } + if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(tmp))){ + escapeChar = tmp; + } tmp++; } elements->measureTagsLen = tmp - elements->measure; diff --git a/source/client/src/clientSqlMonitor.c b/source/client/src/clientSqlMonitor.c new file mode 100644 index 0000000000..572af7ff55 --- /dev/null +++ b/source/client/src/clientSqlMonitor.c @@ -0,0 +1,79 @@ + /* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" +#include "tglobal.h" + +const char* selectMonitorName = "taos_sql_req:count"; +const char* selectMonitorHelp = "count for select sql"; +const int selectMonitorLabelCount = 4; +const char* selectMonitorLabels[] = {"cluster_id", "sql_type", "username", "result"}; + +static const char* defaultClusterID = ""; + +void clientSQLReqMonitorInit(const char* clusterKey) { + if (!tsEnableMonitor) return; + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, selectMonitorName, selectMonitorHelp, selectMonitorLabelCount, selectMonitorLabels); +} + +void clientSQLReqLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int8_t type) { + const char* typeStr; + switch (type) { + case MONITORSQLTYPEDELETE: + typeStr = "delete"; + break; + case MONITORSQLTYPEINSERT: + typeStr = "insert"; + break; + default: + typeStr = "select"; + break; + } + const char* selectMonitorLabelValues[] = {defaultClusterID, typeStr, user, resultStr(result)}; + taosClusterCounterInc(clusterKey, selectMonitorName, selectMonitorLabelValues); +} + +void sqlReqLog(int64_t rid, bool killed, int32_t code, int8_t type) { + if (!tsEnableMonitor) return; + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + + STscObj* pTscObj = acquireTscObj(rid); + if (pTscObj != NULL) { + if (pTscObj->pAppInfo == NULL) { + tscLog("sqlReqLog, not found pAppInfo"); + } else { + clientSQLReqLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, type); + } + releaseTscObj(rid); + } else { + tscLog("sqlReqLog, not found rid"); + } +} + +void clientMonitorClose(const char* clusterKey) { + tscLog("clientMonitorClose, key:%s", clusterKey); + clusterMonitorClose(clusterKey); +} diff --git a/source/client/src/slowQueryMonitor.c b/source/client/src/slowQueryMonitor.c new file mode 100644 index 0000000000..b41343443d --- /dev/null +++ b/source/client/src/slowQueryMonitor.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "clientInt.h" +#include "clientMonitor.h" +#include "clientLog.h" +#include "tglobal.h" + +const char* slowQueryName = "taos_slow_sql:count"; +const char* slowQueryHelp = "slow query log when cost over than config duration"; +const int slowQueryLabelCount = 4; +const char* slowQueryLabels[] = {"cluster_id", "username", "result", "duration"}; +static const char* defaultClusterID = ""; + +const int64_t usInSeconds = 1000 * 1000; +const int64_t msInMinutes = 60 * 1000; + +static const char* getSlowQueryLableCostDesc(int64_t cost) { + if (cost >= 1000 * usInSeconds) { + return "1000s-"; + } else if (cost >= 100 * usInSeconds) { + return "100-1000s"; + } else if (cost >= 10 * usInSeconds) { + return "10-100s"; + } else if (cost >= 3 * usInSeconds) { + return "3-10s"; + } + return "0-3s"; +} + +void clientSlowQueryMonitorInit(const char* clusterKey) { + if (!tsEnableMonitor) return; + SAppInstInfo* pAppInstInfo = getAppInstInfo(clusterKey); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + clusterMonitorInit(clusterKey, epSet, pAppInstInfo->pTransporter); + createClusterCounter(clusterKey, slowQueryName, slowQueryHelp, slowQueryLabelCount, slowQueryLabels); +} + +void clientSlowQueryLog(const char* clusterKey, const char* user, SQL_RESULT_CODE result, int32_t cost) { + const char* slowQueryLabelValues[] = {defaultClusterID, user, resultStr(result), getSlowQueryLableCostDesc(cost)}; + taosClusterCounterInc(clusterKey, slowQueryName, slowQueryLabelValues); +} + +void SlowQueryLog(int64_t rid, bool killed, int32_t code, int32_t cost) { + if (!tsEnableMonitor) return; + SQL_RESULT_CODE result = SQL_RESULT_SUCCESS; + if (TSDB_CODE_SUCCESS != code) { + result = SQL_RESULT_FAILED; + } + // to do Distinguish active Kill events + // else if (killed) { + // result = SQL_RESULT_CANCEL; + // } + + STscObj* pTscObj = acquireTscObj(rid); + if (pTscObj != NULL) { + if(pTscObj->pAppInfo == NULL) { + tscLog("SlowQueryLog, not found pAppInfo"); + } else { + clientSlowQueryLog(pTscObj->pAppInfo->instKey, pTscObj->user, result, cost); + } + releaseTscObj(rid); + } else { + tscLog("SlowQueryLog, not found rid"); + } +} diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 91f0d1eef8..7c3847e4a1 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -23,6 +23,12 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry ) +ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp) +TARGET_LINK_LIBRARIES( + clientMonitorTest + PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor +) + TARGET_INCLUDE_DIRECTORIES( clientTest PUBLIC "${TD_SOURCE_DIR}/include/client/" @@ -41,7 +47,18 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) +TARGET_INCLUDE_DIRECTORIES( + clientMonitorTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + add_test( NAME smlTest COMMAND smlTest ) + +# add_test( +# NAME clientMonitorTest +# COMMAND clientMonitorTest +# ) diff --git a/source/client/test/clientMonitorTests.cpp b/source/client/test/clientMonitorTests.cpp new file mode 100644 index 0000000000..f7ddc1a5cd --- /dev/null +++ b/source/client/test/clientMonitorTests.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "clientInt.h" +#include "clientMonitor.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "executor.h" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(clientMonitorTest, monitorTest) { + const char* cluster1 = "cluster1"; + const char* cluster2 = "cluster2"; + SEpSet epSet; + clusterMonitorInit(cluster1, epSet, NULL); + const char* counterName1 = "slow_query"; + const char* counterName2 = "select_count"; + const char* help1 = "test for slowQuery"; + const char* help2 = "test for selectSQL"; + const char* lables[] = {"lable1"}; + taos_counter_t* c1 = createClusterCounter(cluster1, counterName1, help1, 1, lables); + ASSERT_TRUE(c1 != NULL); + taos_counter_t* c2 = createClusterCounter(cluster1, counterName2, help2, 1, lables); + ASSERT_TRUE(c2 != NULL); + ASSERT_TRUE(c1 != c2); + taos_counter_t* c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 == NULL); + clusterMonitorInit(cluster2, epSet, NULL); + c21 = createClusterCounter(cluster2, counterName1, help2, 1, lables); + ASSERT_TRUE(c21 != NULL); + int i = 0; + while (i < 12) { + taosMsleep(10); + ++i; + } + clusterMonitorClose(cluster1); + clusterMonitorClose(cluster2); +} + +TEST(clientMonitorTest, sendTest) { + TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + ASSERT_TRUE(taos != NULL); + printf("connect taosd sucessfully.\n"); + + int64_t rid = *(int64_t *)taos; + SlowQueryLog(rid, false, -1, 1000); + int i = 0; + while (i < 20) { + SlowQueryLog(rid, false, 0, i * 1000); + taosMsleep(10); + ++i; + } + + taos_close(taos); +} diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 47eac317ec..25cc5d7c79 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -223,12 +223,12 @@ static const SSysDbTableSchema userTblDistSchema[] = { }; static const SSysDbTableSchema userUsersSchema[] = { - {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, - {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, + {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; GRANTS_SCHEMA; @@ -276,7 +276,7 @@ static const SSysDbTableSchema configSchema[] = { }; static const SSysDbTableSchema variablesSchema[] = { - {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -304,7 +304,7 @@ static const SSysDbTableSchema subscriptionSchema[] = { static const SSysDbTableSchema vnodesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = true}, {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, @@ -312,12 +312,12 @@ static const SSysDbTableSchema vnodesSchema[] = { }; static const SSysDbTableSchema userUserPrivilegesSchema[] = { - {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userViewsSchema[] = { @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema useMachinesSchema[] = { - {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysTableMeta infosMeta[] = { @@ -395,9 +395,9 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false}, {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false}, {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false}, - {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false}, - {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false}, - {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false}, + {TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), true}, + {TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), true}, + {TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9f55b67ea3..f4455be206 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -452,20 +452,21 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + int32_t newLen = pSource->varmeta.length; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (pColumnInfoData->varmeta.allocLen < newLen) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + pColumnInfoData->varmeta.allocLen = newLen; } - pColumnInfoData->varmeta.length = pSource->varmeta.length; + pColumnInfoData->varmeta.length = newLen; if (pColumnInfoData->pData != NULL && pSource->pData != NULL) { - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); + memcpy(pColumnInfoData->pData, pSource->pData, newLen); } } else { memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); @@ -1687,7 +1688,29 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { } static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { + if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.length != 0) { + int32_t newLen = pColInfoData->varmeta.offset[n]; + if (-1 == newLen) { + for (int i = n - 1; i >= 0; --i) { + newLen = pColInfoData->varmeta.offset[i]; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + break; + } + } + } + if (newLen <= -1) { + uFatal("colDataKeepFirstNRows: newLen:%d old:%d", newLen, pColInfoData->varmeta.length); + } else { + pColInfoData->varmeta.length = newLen; + } + } // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7a5d554b97..c5a26c5c10 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -94,6 +94,9 @@ char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; uint16_t tsMonitorPort = 6043; int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; +bool tsMonitorLogProtocol = false; +int32_t tsMonitorIntervalForBasic = 30; +bool tsMonitorForceV2 = true; // audit bool tsEnableAudit = true; @@ -535,6 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { return -1; if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; return 0; } @@ -693,6 +698,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1081,6 +1090,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; tsMetaCacheMaxSize = cfgGetItem(pCfg, "metaCacheMaxSize")->i32; tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } @@ -1155,6 +1166,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; + tsMonitorLogProtocol = cfgGetItem(pCfg, "monitorLogProtocol")->bval; + tsMonitorIntervalForBasic = cfgGetItem(pCfg, "monitorIntervalForBasic")->i32; + tsMonitorForceV2 = cfgGetItem(pCfg, "monitorForceV2")->i32; tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; @@ -1244,6 +1258,8 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); @@ -1288,7 +1304,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetServerLogCfg(pCfg); } - taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32); if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -1366,7 +1382,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); if (taosSetFileHandlesLimit() != 0) return -1; - taosSetAllDebugFlag(cfgGetItem(tsCfg, "debugFlag")->i32); + taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); cfgDumpCfg(tsCfg, tsc, false); @@ -1459,7 +1475,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { } if (strncasecmp(name, "debugFlag", 9) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); return 0; } @@ -1533,7 +1549,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { switch (lowcaseName[0]) { case 'd': { if (strcasecmp("debugFlag", name) == 0) { - taosSetAllDebugFlag(pItem->i32); + taosSetAllDebugFlag(pCfg, pItem->i32); matched = true; } break; @@ -1757,11 +1773,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag taosSetDebugFlag(pFlagPtr, name, flag); } -void taosSetAllDebugFlag(int32_t flag) { +void taosSetGlobalDebugFlag(int32_t flag) { taosSetAllDebugFlag(tsCfg, flag); } + +static void taosSetAllDebugFlag(SConfig *pCfg, int32_t flag) { if (flag <= 0) return; SArray *noNeedToSetVars = NULL; - SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); + SConfigItem *pItem = cfgGetItem(pCfg, "debugFlag"); if (pItem != NULL) { pItem->i32 = flag; noNeedToSetVars = pItem->array; @@ -1811,4 +1829,4 @@ int8_t taosGranted(int8_t type) { break; } return 0; -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 85f5d462c7..f8df2edd61 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR tDecoderClear(&decoder); return 0; } -int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - tEndEncode(&encoder); +// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); +// if (tStartEncode(&encoder) < 0) return -1; +// tEndEncode(&encoder); - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq * return 0; } + int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) { // impl later return; } -int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} -int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// tEndEncode(&encoder); +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } +// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { SEncoder encoder = {0}; @@ -1436,19 +1437,15 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { void tFreeSStatusRsp(SStatusRsp *pRsp) { taosArrayDestroy(pRsp->pDnodeEps); } -int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +int32_t tSerializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; - if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; - if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->contLen) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->pCont) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1456,25 +1453,67 @@ int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq return tlen; } -int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +int32_t tDeserializeSStatisReq(void *buf, int32_t bufLen, SStatisReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; - tEndDecode(&decoder); + if (tDecodeI32(&decoder, &pReq->contLen) < 0) return -1; + if (pReq->contLen > 0) { + pReq->pCont = taosMemoryMalloc(pReq->contLen + 1); + if (pReq->pCont == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->pCont) < 0) return -1; + } + + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } +void tFreeSStatisReq(SStatisReq *pReq) { + taosMemoryFreeClear(pReq->pCont); +} + +// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); + +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; +// if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; +// tEndEncode(&encoder); + +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } + +// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); + +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; +// tEndDecode(&decoder); + +// tDecoderClear(&decoder); +// return 0; +// } + int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3280,18 +3319,6 @@ int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) return tlen; } -int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3305,17 +3332,17 @@ int32_t tSerializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) return tlen; } -int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSServerVerReq(void *buf, int32_t bufLen, SServerVerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp) { SEncoder encoder = {0}; @@ -4233,17 +4260,17 @@ int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq return tlen; } -int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; @@ -4325,24 +4352,24 @@ int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return tlen; } -int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; - if (pReq->payloadLen > 0) { - pReq->payload = taosMemoryMalloc(pReq->payloadLen); - if (pReq->payload == NULL) return -1; - if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; - } +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; +// if (pReq->payloadLen > 0) { +// pReq->payload = taosMemoryMalloc(pReq->payloadLen); +// if (pReq->payload == NULL) return -1; +// if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; +// } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } @@ -4595,34 +4622,34 @@ void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { taosArrayDestroy(pRsp->pIndexRsp); } -int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; - if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; +// if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; - if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; +// if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } -void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } +// void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -4930,17 +4957,17 @@ int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { return tlen; } -int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { SEncoder encoder = {0}; @@ -4955,17 +4982,17 @@ int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pR return tlen; } -int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; @@ -5212,11 +5239,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4; + if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5; + if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -5830,17 +5857,17 @@ int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecom return tlen; } -int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { SEncoder encoder = {0}; @@ -5889,38 +5916,38 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq return 0; } -int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; - if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; - if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; +// if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSServerStatusRsp(void *buf, int32_t bufLen, SServerStatusRsp *pRsp) { SEncoder encoder = {0}; @@ -6958,30 +6985,30 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } -int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - // SEncoder encoder = {0}; - // tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +// // SEncoder encoder = {0}; +// // tEncoderInit(&encoder, buf, bufLen); - // if (tStartEncode(&encoder) < 0) return -1; - // if (pRsp->rspList) { - // int32_t num = taosArrayGetSize(pRsp->rspList); - // if (tEncodeI32(&encoder, num) < 0) return -1; - // for (int32_t i = 0; i < num; ++i) { - // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; - // } - // } else { - // if (tEncodeI32(&encoder, 0) < 0) return -1; - // } - // tEndEncode(&encoder); +// // if (tStartEncode(&encoder) < 0) return -1; +// // if (pRsp->rspList) { +// // int32_t num = taosArrayGetSize(pRsp->rspList); +// // if (tEncodeI32(&encoder, num) < 0) return -1; +// // for (int32_t i = 0; i < num; ++i) { +// // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); +// // if (tEncodeI32(&encoder, rsp->code) < 0) return -1; +// // } +// // } else { +// // if (tEncodeI32(&encoder, 0) < 0) return -1; +// // } +// // tEndEncode(&encoder); - // int32_t tlen = encoder.pos; - // tEncoderClear(&encoder); - // reture tlen; - return 0; -} +// // int32_t tlen = encoder.pos; +// // tEncoderClear(&encoder); +// // reture tlen; +// return 0; +// } -int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { +//int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { // SDecoder decoder = {0}; // int32_t num = 0; // tDecoderInit(&decoder, buf, bufLen); @@ -7002,8 +7029,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc // tEndDecode(&decoder); // tDecoderClear(&decoder); - return 0; -} +// return 0; +//} int tEncodeSVCreateTbBatchRsp(SEncoder *pCoder, const SVCreateTbBatchRsp *pRsp) { int32_t nRsps = taosArrayGetSize(pRsp->pArray); @@ -7138,15 +7165,15 @@ int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) { return 0; } -int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { - if (tStartDecode(pCoder) < 0) return -1; +// int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; - if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; +// if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; +// if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); @@ -7427,34 +7454,34 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); } -int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { if (NULL == pReq) { @@ -7854,37 +7881,37 @@ int32_t tDecodeSVDropStbReq(SDecoder *pCoder, SVDropStbReq *pReq) { return 0; } -static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { - if (tStartEncode(pCoder) < 0) return -1; +// static int32_t tEncodeSVSubmitBlk(SEncoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartEncode(pCoder) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; - if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; - if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; - if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; +// if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; +// if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; +// if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndEncode(pCoder); - return 0; -} +// tEndEncode(pCoder); +// return 0; +// } -static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { - if (tStartDecode(pCoder) < 0) return -1; +// static int32_t tDecodeSVSubmitBlk(SDecoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { +// if (tStartDecode(pCoder) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; - if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; - if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; - if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; +// if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; +// if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; +// if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; - if (flags & TD_AUTO_CREATE_TABLE) { - if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; - } +// if (flags & TD_AUTO_CREATE_TABLE) { +// if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; +// } - tEndDecode(pCoder); - return 0; -} +// tEndDecode(pCoder); +// return 0; +// } static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) { if (tStartEncode(pEncoder) < 0) return -1; @@ -7908,78 +7935,78 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl return 0; } -static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { - if (tStartDecode(pDecoder) < 0) return -1; +// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; - if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; +// if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; +// if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; +// pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); +// if (NULL == pBlock->tblFName) return -1; +// if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; +// if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; - int32_t meta = 0; - if (tDecodeI32(pDecoder, &meta) < 0) return -1; - if (meta) { - pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pBlock->pMeta) return -1; - if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; - } else { - pBlock->pMeta = NULL; - } +// int32_t meta = 0; +// if (tDecodeI32(pDecoder, &meta) < 0) return -1; +// if (meta) { +// pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pBlock->pMeta) return -1; +// if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; +// } else { +// pBlock->pMeta = NULL; +// } - tEndDecode(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// return 0; +// } -int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { - int32_t nBlocks = taosArrayGetSize(pRsp->pArray); +// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { +// int32_t nBlocks = taosArrayGetSize(pRsp->pArray); - if (tStartEncode(pEncoder) < 0) return -1; +// if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; - if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; - for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { - if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; - } +// if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; +// for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { +// if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; +// } - tEndEncode(pEncoder); - return 0; -} +// tEndEncode(pEncoder); +// return 0; +// } -int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { - if (tStartDecode(pDecoder) < 0) return -1; +// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; - pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); - if (pRsp->pBlocks == NULL) return -1; - for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { - if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; - } +// if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; +// pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); +// if (pRsp->pBlocks == NULL) return -1; +// for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { +// if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; +// } - tEndDecode(pDecoder); - tDecoderClear(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// tDecoderClear(pDecoder); +// return 0; +// } -void tFreeSSubmitBlkRsp(void *param) { - if (NULL == param) { - return; - } +// void tFreeSSubmitBlkRsp(void *param) { +// if (NULL == param) { +// return; +// } - SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; +// SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param; - taosMemoryFree(pRsp->tblFName); - if (pRsp->pMeta) { - taosMemoryFree(pRsp->pMeta->pSchemas); - taosMemoryFree(pRsp->pMeta); - } -} +// taosMemoryFree(pRsp->tblFName); +// if (pRsp->pMeta) { +// taosMemoryFree(pRsp->pMeta->pSchemas); +// taosMemoryFree(pRsp->pMeta); +// } +// } void tFreeSSubmitRsp(SSubmitRsp *pRsp) { if (NULL == pRsp) return; @@ -8153,23 +8180,23 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { return 0; } -int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) { if (tStartEncode(pEncoder) < 0) return -1; @@ -8194,22 +8221,22 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { @@ -8245,22 +8272,22 @@ int32_t tDecodeSMCreateStbRsp(SDecoder *pDecoder, SMCreateStbRsp *pRsp) { return 0; } -int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { - int32_t meta = 0; - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSMCreateStbRsp(void *buf, int32_t bufLen, SMCreateStbRsp *pRsp) { +// int32_t meta = 0; +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &meta) < 0) return -1; - if (meta) { - pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pRsp->pMeta) return -1; - if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; - } - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeI32(&decoder, &meta) < 0) return -1; +// if (meta) { +// pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pRsp->pMeta) return -1; +// if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; +// } +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) { if (NULL == pRsp) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b91562be7a..6e9278c630 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -137,6 +137,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa return TSDB_CODE_SUCCESS; } +#if 0 int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -177,16 +178,19 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { switch (bitmapMode) { case 0: tdGetBitmapValTypeII(pBitmap, colIdx, pValType); break; +#if 0 case -1: case 1: tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; @@ -231,7 +235,7 @@ bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCe return true; } -#ifdef BUILD_NO_CALL +// #ifdef BUILD_NO_CALL const uint8_t tdVTypeByte[2][3] = {{ // 2 bits TD_VTYPE_NORM_BYTE_II, @@ -439,13 +443,15 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { +int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t rowType) { STColumn *pTColumn; SColVal *pColVal; int32_t nColVal = taosArrayGetSize(pArray); int32_t varDataLen = 0; + int32_t nonVarDataLen = 0; int32_t maxVarDataLen = 0; int32_t iColVal = 0; + int32_t nBound = 0; void *varBuf = NULL; bool isAlloc = false; @@ -458,6 +464,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } else { pColVal = NULL; } + if (pColVal && !COL_VAL_IS_NONE(pColVal)) { + ++nBound; + } if (iColumn == 0) { ASSERT(pColVal && pColVal->cid == pTColumn->colId); @@ -484,16 +493,26 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } } + } else { + if(pColVal && COL_VAL_IS_VALUE(pColVal)) { + nonVarDataLen += TYPE_BYTES[pTColumn->type]; + } } } ++iColVal; } - if (!(*ppRow)) { - *ppRow = (STSRow *)taosMemoryCalloc( - 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); - isAlloc = true; + int32_t rowTotalLen = 0; + if (rowType == TD_ROW_TP) { + rowTotalLen = sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1); + } else { + rowTotalLen = sizeof(STSRow) + sizeof(col_id_t) + varDataLen + nonVarDataLen + (nBound - 1) * sizeof(SKvRowIdx) + + TD_BITMAP_BYTES(nBound - 1); + } + if (!(*ppRow)) { + *ppRow = (STSRow *)taosMemoryCalloc(1, rowTotalLen); + isAlloc = true; } if (!(*ppRow)) { @@ -512,10 +531,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } - SRowBuilder rb = {0}; + SRowBuilder rb = {.rowType = rowType}; tdSRowInit(&rb, pTSchema->version); - tdSRowSetInfo(&rb, pTSchema->numOfCols, pTSchema->numOfCols, pTSchema->flen); + tdSRowSetInfo(&rb, pTSchema->numOfCols, nBound, pTSchema->flen); tdSRowResetBuf(&rb, *ppRow); + int32_t iBound = 0; iColVal = 0; for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; ++iColumn) { @@ -529,21 +549,28 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { valType = TD_VTYPE_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { valType = TD_VTYPE_NULL; + ++iBound; } else if (IS_VAR_DATA_TYPE(pTColumn->type)) { varDataSetLen(varBuf, pColVal->value.nData); if (pColVal->value.nData != 0) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); } val = varBuf; + ++iBound; } else { val = (const void *)&pColVal->value.val; + ++iBound; } } else { // pColVal = NULL; valType = TD_VTYPE_NONE; } - tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + if (TD_IS_TP_ROW(rb.pBuf)) { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, pTColumn->offset, iColVal); + } else { + tdAppendColValToRow(&rb, pTColumn->colId, pTColumn->type, valType, val, true, rb.offset, iBound - 1); + } ++iColVal; } @@ -605,7 +632,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } - +#if 0 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -655,6 +682,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { } return TSDB_CODE_SUCCESS; } +#endif int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP @@ -761,11 +789,11 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + if (colIdx < 1) { terrno = TSDB_CODE_INVALID_PARA; + ASSERTS(0, "colIdx is %" PRIi64, colIdx); return terrno; } - offset -= sizeof(SKvRowIdx); --colIdx; #ifdef TD_SUPPORT_BITMAP @@ -779,6 +807,7 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + pBuilder->offset += sizeof(SKvRowIdx); if (valType == TD_VTYPE_NORM) { char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { @@ -830,7 +859,7 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo return 0; } - +#if 0 int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, int32_t allNullLen, int32_t boundNullLen) { if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { @@ -866,6 +895,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou #endif return TSDB_CODE_SUCCESS; } +#endif int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; @@ -904,6 +934,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_LEN(pBuilder->pBuf, len); TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + pBuilder->offset = 0; break; default: terrno = TSDB_CODE_INVALID_PARA; @@ -1040,10 +1071,12 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int case 0: tdSetBitmapValTypeII(pBitmap, colIdx, valType); break; +#if 0 case -1: case 1: tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; +#endif default: terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; @@ -1056,7 +1089,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; SCellVal cv = {0}; - SValue value = {0}; ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); @@ -1084,5 +1116,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index eb79e79afa..51c733e796 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -35,6 +35,10 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PUBLIC "${TD_SOURCE_DIR}/include/util" ) +add_test( + NAME dataformatTest + COMMAND dataformatTest +) # tmsg test # add_executable(tmsgTest "") diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b1083e4590..8bc7d47ebf 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#if 0 #include #include @@ -59,7 +58,6 @@ STSchema *genSTSchema(int16_t nCols) { case 1: { pSchema[i].type = TSDB_DATA_TYPE_INT; pSchema[i].bytes = TYPE_BYTES[pSchema[i].type]; - ; } break; case 2: { pSchema[i].type = TSDB_DATA_TYPE_BIGINT; @@ -123,7 +121,8 @@ STSchema *genSTSchema(int16_t nCols) { return pResult; } -// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, c9 bool +// ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, +// c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { if (!(*pArray)) { *pArray = taosArrayInit(nCols, sizeof(SColVal)); @@ -142,59 +141,76 @@ static int32_t genTestData(const char **data, int16_t nCols, SArray **pArray) { taosArrayPush(*pArray, &colVal); continue; } - switch (i) { case 0: - sscanf(data[i], "%" PRIi64, &colVal.value.ts); + colVal.type = TSDB_DATA_TYPE_TIMESTAMP; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 1: - sscanf(data[i], "%" PRIi32, &colVal.value.i32); + colVal.type = TSDB_DATA_TYPE_INT; + sscanf(data[i], "%" PRIi32, (int32_t *)&colVal.value.val); break; case 2: - sscanf(data[i], "%" PRIi64, &colVal.value.i64); + colVal.type = TSDB_DATA_TYPE_BIGINT; + sscanf(data[i], "%" PRIi64, &colVal.value.val); break; case 3: - sscanf(data[i], "%f", &colVal.value.f); + colVal.type = TSDB_DATA_TYPE_FLOAT; + sscanf(data[i], "%f", (float *)&colVal.value.val); break; case 4: - sscanf(data[i], "%lf", &colVal.value.d); + colVal.type = TSDB_DATA_TYPE_DOUBLE; + sscanf(data[i], "%lf", (double *)&colVal.value.val); break; case 5: { + colVal.type = TSDB_DATA_TYPE_BINARY; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 10 ? dataLen : 10; colVal.value.pData = (uint8_t *)data[i]; } break; case 6: { + colVal.type = TSDB_DATA_TYPE_NCHAR; int16_t dataLen = strlen(data[i]) + 1; colVal.value.nData = dataLen < 40 ? dataLen : 40; colVal.value.pData = (uint8_t *)data[i]; // just for test, not real nchar } break; - case 7: - case 9: { + case 7: { + colVal.type = TSDB_DATA_TYPE_TINYINT; int32_t d8; sscanf(data[i], "%" PRId32, &d8); - colVal.value.i8 = (int8_t)d8; - } break; + colVal.value.val = (int8_t)d8; + } case 8: { + colVal.type = TSDB_DATA_TYPE_SMALLINT; int32_t d16; sscanf(data[i], "%" PRId32, &d16); - colVal.value.i16 = (int16_t)d16; + colVal.value.val = (int16_t)d16; + } break; + case 9: { + colVal.type = TSDB_DATA_TYPE_BOOL; + int32_t d8; + sscanf(data[i], "%" PRId32, &d8); + colVal.value.val = (int8_t)d8; } break; case 10: { + colVal.type = TSDB_DATA_TYPE_UTINYINT; uint32_t u8; sscanf(data[i], "%" PRId32, &u8); - colVal.value.u8 = (uint8_t)u8; + colVal.value.val = (uint8_t)u8; } break; case 11: { + colVal.type = TSDB_DATA_TYPE_USMALLINT; uint32_t u16; sscanf(data[i], "%" PRId32, &u16); - colVal.value.u16 = (uint16_t)u16; + colVal.value.val = (uint16_t)u16; } break; case 12: { - sscanf(data[i], "%" PRIu32, &colVal.value.u32); + colVal.type = TSDB_DATA_TYPE_UINT; + sscanf(data[i], "%" PRIu32, (uint32_t *)&colVal.value.val); } break; case 13: { - sscanf(data[i], "%" PRIu64, &colVal.value.u64); + colVal.type = TSDB_DATA_TYPE_UBIGINT; + sscanf(data[i], "%" PRIu64, (uint64_t *)&colVal.value.val); } break; default: ASSERT(0); @@ -215,25 +231,25 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { } switch (type) { case TSDB_DATA_TYPE_BOOL: - printf("%s ", cv->value.i8 == 0 ? "false" : "true"); + printf("%s ", cv->value.val == 0 ? "false" : "true"); break; case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", cv->value.i8); + printf("%" PRIi8 " ", *(int8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", cv->value.i16); + printf("%" PRIi16 " ", *(int16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", cv->value.i32); + printf("%" PRIi32 " ", *(int32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f ", cv->value.f); + printf("%f ", *(float *)&cv->value.val); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", cv->value.d); + printf("%lf ", *(double *)&cv->value.val); break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { @@ -242,7 +258,7 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", cv->value.i64); + printf("%" PRIi64 " ", cv->value.val); break; case TSDB_DATA_TYPE_NCHAR: { char tv[15] = {0}; @@ -250,16 +266,16 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { printf("%s ", tv); } break; case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", cv->value.u8); + printf("%" PRIu8 " ", *(uint8_t *)&cv->value.val); break; case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", cv->value.u16); + printf("%" PRIu16 " ", *(uint16_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", cv->value.u32); + printf("%" PRIu32 " ", *(uint32_t *)&cv->value.val); break; case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", cv->value.u64); + printf("%" PRIu64 " ", *(uint64_t *)&cv->value.val); break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -286,11 +302,11 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) { return 0; } -void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) { - // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData); +void debugPrintTSRow(STSRow *row, STSchema *pTSchema, const char *tags, int32_t ln) { + printf("%s:%d %s:v%d:len-%u ", tags, ln, (row->type) ? "KV" : "TP", row->sver, row->len); for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); debugPrintSColVal(&cv, pTSchema->columns[i].type); } printf("\n"); @@ -315,36 +331,36 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_TINYINT: { int32_t d8; sscanf(rawVal, "%" PRId32, &d8); - EXPECT_EQ(cv->value.i8, (int8_t)d8); + EXPECT_EQ((int8_t)cv->value.val, (int8_t)d8); } break; case TSDB_DATA_TYPE_SMALLINT: { int32_t d16; sscanf(rawVal, "%" PRId32, &d16); - EXPECT_EQ(cv->value.i16, (int16_t)d16); + EXPECT_EQ((int16_t)cv->value.val, (int16_t)d16); } break; case TSDB_DATA_TYPE_INT: { - sscanf(rawVal, "%" PRId32, &rawSVal.i32); - EXPECT_EQ(cv->value.i32, rawSVal.i32); + sscanf(rawVal, "%" PRId32, (int32_t *)&rawSVal.val); + EXPECT_EQ((int32_t)cv->value.val, (int32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_BIGINT: { - sscanf(rawVal, "%" PRIi64, &rawSVal.i64); - EXPECT_EQ(cv->value.i64, rawSVal.i64); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_FLOAT: { - sscanf(rawVal, "%f", &rawSVal.f); - EXPECT_FLOAT_EQ(cv->value.f, rawSVal.f); + sscanf(rawVal, "%f", (float *)&rawSVal.val); + EXPECT_FLOAT_EQ((float)cv->value.val, (float)rawSVal.val); } break; case TSDB_DATA_TYPE_DOUBLE: { - sscanf(rawVal, "%lf", &rawSVal.d); - EXPECT_DOUBLE_EQ(cv->value.d, rawSVal.d); + sscanf(rawVal, "%lf", (double *)&rawSVal.val); + EXPECT_DOUBLE_EQ((double)cv->value.val, (double)rawSVal.val); } break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_GEOMETRY: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); } break; case TSDB_DATA_TYPE_TIMESTAMP: { - sscanf(rawVal, "%" PRIi64, &rawSVal.ts); - EXPECT_DOUBLE_EQ(cv->value.ts, rawSVal.ts); + sscanf(rawVal, "%" PRIi64, &rawSVal.val); + EXPECT_DOUBLE_EQ(cv->value.val, rawSVal.val); } break; case TSDB_DATA_TYPE_NCHAR: { EXPECT_STRCASEEQ(rawVal, (const char *)cv->value.pData); // informal nchar comparsion @@ -352,20 +368,20 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { case TSDB_DATA_TYPE_UTINYINT: { uint32_t u8; sscanf(rawVal, "%" PRIu32, &u8); - EXPECT_EQ(cv->value.u8, (uint8_t)u8); + EXPECT_EQ((uint8_t)cv->value.val, (uint8_t)u8); } break; case TSDB_DATA_TYPE_USMALLINT: { uint32_t u16; sscanf(rawVal, "%" PRIu32, &u16); - EXPECT_EQ(cv->value.u16, (uint16_t)u16); + EXPECT_EQ((uint16_t)cv->value.val, (uint16_t)u16); } break; case TSDB_DATA_TYPE_UINT: { - sscanf(rawVal, "%" PRIu32, &rawSVal.u32); - EXPECT_EQ(cv->value.u32, rawSVal.u32); + sscanf(rawVal, "%" PRIu32, (uint32_t *)&rawSVal.val); + EXPECT_EQ((uint32_t)cv->value.val, (uint32_t)rawSVal.val); } break; case TSDB_DATA_TYPE_UBIGINT: { - sscanf(rawVal, "%" PRIu64, &rawSVal.u64); - EXPECT_EQ(cv->value.u64, rawSVal.u64); + sscanf(rawVal, "%" PRIu64, (uint64_t *)&rawSVal.val); + EXPECT_EQ((uint64_t)cv->value.val, (uint64_t)rawSVal.val); } break; case TSDB_DATA_TYPE_JSON: printf("JSON "); @@ -395,36 +411,66 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) { return 0; } -static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) { +static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { for (int16_t i = 0; i < pTSchema->numOfCols; ++i) { SColVal cv = {0}; - tTSRowGet(row, pTSchema, i, &cv); + tTSRowGetVal(row, pTSchema, i, &cv); checkSColVal(data[i], &cv, pTSchema->columns[i].type); } + + STSRowIter rowIter = {0}; + rowIter.pSchema = pTSchema; + tdSTSRowIterReset(&rowIter, row); + for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { + STColumn *pCol = pTSchema->columns + i; + SColVal colVal = {0}; + SCellVal cv = {0}; + if (!tdSTSRowIterFetch(&rowIter, pCol->colId, pCol->type, &cv)) { + break; + } + + colVal.cid = pCol->colId; + colVal.type = pCol->type; + if (tdValTypeIsNone(cv.valType)) { + colVal.flag = CV_FLAG_NONE; + } else if (tdValTypeIsNull(cv.valType)) { + colVal.flag = CV_FLAG_NULL; + } else { + colVal.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pCol->type)) { + colVal.value.nData = varDataLen(cv.val); + colVal.value.pData = (uint8_t *)varDataVal(cv.val); + } else { + memcpy(&colVal.value.val, cv.val, tDataTypes[pCol->type].bytes); + } + } + checkSColVal(data[i], &colVal, pCol->type); + } } TEST(testCase, AllNormTest) { - int16_t nCols = 14; - STSRowBuilder rb = {0}; - STSRow2 *row = nullptr; - SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); + int16_t nCols = 14; + STSRow *row = nullptr; + SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); STSchema *pTSchema = genSTSchema(nCols); EXPECT_NE(pTSchema, nullptr); - // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, - // c9 bool - char *data[14] = {"1653694220000", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}; + // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, + // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned + char *data[14] = { + "1653694220000", "10", "20", "10.1", "10.1", "binary10", "nchar10", "10", "10", "10", "10", "20", "30", "40"}; genTestData((const char **)&data, nCols, &pArray); - tTSRowNew(&rb, pArray, pTSchema, &row); - + tdSTSRowNew(pArray, pTSchema, &row, TD_ROW_TP); debugPrintTSRow(row, pTSchema, __func__, __LINE__); + tdSRowPrint(row, pTSchema, __func__); checkTSRow((const char **)&data, row, pTSchema); - tsRowBuilderClear(&rb); + taosMemoryFreeClear(row); taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } @@ -433,7 +479,7 @@ TEST(testCase, AllNormTest) { TEST(testCase, NoneTest) { const static int nCols = 14; const static int nRows = 20; - STSRow2 *row = nullptr; + STSRow *row = nullptr; SArray *pArray = taosArrayInit(nCols, sizeof(SColVal)); EXPECT_NE(pArray, nullptr); @@ -442,42 +488,46 @@ TEST(testCase, NoneTest) { // ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(10), c6 nchar(10), c7 tinyint, c8 smallint, // c9 bool c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned - const char *data[nRows][nCols] = { - {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, - {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, - {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, - {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, - {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, - {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, - {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, - {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, - {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, - {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, - {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", - "18446744073709551615"}, - {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, - {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, - {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", - "4294967295", "100"}, - {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", - "no"}}; + const int8_t rowType[nRows] = {TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP, TD_ROW_KV, TD_ROW_KV, + TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_KV, TD_ROW_TP}; - for (int r = 0; r < nRows; ++r) { - genTestData((const char **)&data[r], nCols, &pArray); - tTSRowNew(NULL, pArray, pTSchema, &row); - debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print - checkTSRow((const char **)&data[r], row, pTSchema); // check - tTSRowFree(row); - taosArrayClear(pArray); +const char *data[nRows][nCols] = { + {"1653694220000", "no", "20", "10.1", "10.1", "binary10", "no", "10", "10", "nu", "10", "20", "30", "40"}, + {"1653694220001", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220002", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220003", "10", "10", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220004", "no", "20", "no", "no", "no", "nchar10", "no", "no", "no", "no", "no", "no", "no"}, + {"1653694220005", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220006", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "nu"}, + {"1653694220007", "no", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no", "no", "no", "no", "no"}, + {"1653694220008", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "nu", "nu", "nu", "nu", "nu", "no"}, + {"1653694220009", "no", "nu", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220010", "-1", "-1", "-1", "-1", "binary10", "nu", "-1", "0", "0", "0", "0", "0", "0"}, + {"1653694220011", "-2147483648", "nu", "nu", "nu", "biy10", "nu", "nu", "32767", "no", "nu", "nu", "nu", "100"}, + {"1653694220012", "2147483647", "nu", "nu", "nu", "ary10", "nu", "nu", "-32768", "no", "nu", "nu", "nu", "100"}, + {"1653694220013", "no", "-9223372036854775818", "nu", "nu", "b1", "nu", "nu", "10", "no", "nu", "nu", "nu", "nu"}, + {"1653694220014", "no", "nu", "nu", "nu", "b0", "nu", "nu", "10", "no", "nu", "nu", "nu", "9223372036854775808"}, + {"1653694220015", "no", "nu", "nu", "nu", "binary30", "char4", "nu", "10", "no", "nu", "nu", "nu", + "18446744073709551615"}, + {"1653694220016", "2147483647", "nu", "nu", "nu", "bin50", "nu", "nu", "10", "no", "nu", "nu", "nu", "100"}, + {"1653694220017", "2147483646", "0", "0", "0", "binary10", "0", "0", "0", "0", "255", "0", "0", "0"}, + {"1653694220018", "no", "-9223372036854775808", "nu", "nu", "binary10", "nu", "nu", "10", "no", "nu", "nu", + "4294967295", "100"}, + {"1653694220019", "no", "9223372036854775807", "nu", "nu", "bin10", "nu", "nu", "10", "no", "254", "nu", "nu", + "no"}}; + +for (int r = 0; r < nRows; ++r) { + genTestData((const char **)&data[r], nCols, &pArray); + tdSTSRowNew(pArray, pTSchema, &row, rowType[r]); + debugPrintTSRow(row, pTSchema, __func__, __LINE__); // debug print + tdSRowPrint(row, pTSchema, __func__); + checkTSRow((const char **)&data[r], row, pTSchema); // check + taosMemoryFreeClear(row); + taosArrayClear(pArray); } taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } -#endif #endif \ No newline at end of file diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1508d88def..f9456f1729 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -68,7 +68,7 @@ static struct { int64_t startTime; } global = {0}; -static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetGlobalDebugFlag(143); } static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } static void dmStopDnode(int signum, void *sigInfo, void *context) { diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 80502e2662..975246a10f 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -43,6 +43,7 @@ typedef struct SDnodeMgmt { GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; int32_t statusSeq; + SendMonitorReportFp sendMonitorReportFpBasic; } SDnodeMgmt; // dmHandle.c diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index fa59f56496..aa5e87e8b1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -360,12 +360,12 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - +#if 0 if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } - +#endif if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { terrno = TSDB_CODE_INVALID_MSG; return -1; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b9dd45f1c0..a651fbf060 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -65,6 +65,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp; + pMgmt->sendMonitorReportFpBasic = pInput->sendMonitorReportFpBasic; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7ca19d7725..cecb3d1581 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -75,6 +75,7 @@ static void *dmNotifyThreadFp(void *param) { static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); + int64_t lastTimeForBasic = taosGetTimestampMs(); setThreadName("dnode-monitor"); static int32_t TRIM_FREQ = 20; @@ -85,6 +86,7 @@ static void *dmMonitorThreadFp(void *param) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); + if (curTime < lastTime) lastTime = curTime; float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { @@ -96,6 +98,15 @@ static void *dmMonitorThreadFp(void *param) { taosMemoryTrim(0); } } + + if(tsMonitorForceV2){ + if (curTime < lastTimeForBasic) lastTimeForBasic = curTime; + float intervalForBasic = (curTime - lastTimeForBasic) / 1000.0f; + if (intervalForBasic >= tsMonitorIntervalForBasic) { + (*pMgmt->sendMonitorReportFpBasic)(); + lastTimeForBasic = curTime; + } + } } return NULL; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 531e6f4b3d..a3a5f1d971 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -191,6 +191,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATIS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 6b875db860..0cdc68345a 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw ) IF (TD_STORAGE) @@ -14,6 +14,10 @@ IF (TD_STORAGE) ENDIF () +IF (DEFINED GRANT_CFG_INCLUDE_DIR) + add_definitions(-DGRANTS_CFG) +ENDIF() + target_include_directories( dnode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index bc6a4652e7..90e44e5acc 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -128,6 +128,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); // dmMonitor.c void dmSendMonitorReport(); void dmSendAuditRecords(); +void dmSendMonitorReportBasic(); void dmGetVnodeLoads(SMonVloadInfo *pInfo); void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d173c83e97..12712d6585 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -394,6 +394,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .sendAuditRecordFp = auditSendRecordsInBatch, + .sendMonitorReportFpBasic = dmSendMonitorReportBasic, .getVnodeLoadsFp = dmGetVnodeLoads, .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite, .getMnodeLoadsFp = dmGetMnodeLoads, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index c42aa6a1ae..21e25f5535 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -25,8 +25,16 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); } +static void dmGetMonitorBasicInfoBasic(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 2; + pInfo->dnode_id = pDnode->data.dnodeId; + pInfo->cluster_id = pDnode->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + //pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) /1000.0f; pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; @@ -44,6 +52,12 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) { monSetDmInfo(&dmInfo); } +static void dmGetDmMonitorInfoBasic(SDnode *pDnode) { + SMonDmInfo dmInfo = {0}; + dmGetMonitorBasicInfoBasic(pDnode, &dmInfo.basic); + monSetDmInfo(&dmInfo); +} + static void dmGetMmMonitorInfo(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; if (dmMarkWrapper(pWrapper) == 0) { @@ -106,7 +120,17 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - monSendReport(); + monGenAndSendReport(); +} + +void dmSendMonitorReportBasic() { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfoBasic(pDnode); + dmGetMmMonitorInfo(pDnode); + monGenAndSendReportBasic(); } //Todo: put this in seperate file in the future diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1a31f08801..77760b16f4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5c670cbdd3..d882d784de 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor monitorfw ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 504c405506..2175824acd 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -123,6 +123,7 @@ typedef struct { ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; SendAuditRecordsFp sendAuditRecordFp; + SendMonitorReportFp sendMonitorReportFpBasic; GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsLiteFp; GetMnodeLoadsFp getMnodeLoadsFp; diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 0e77ae9743..ceaf086dc1 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -23,9 +23,13 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser audit monitorfw ) +IF (DEFINED GRANT_CFG_INCLUDE_DIR) + add_definitions(-DGRANTS_CFG) +ENDIF() + IF (TD_GRANT) TARGET_LINK_LIBRARIES(mnode grant) ADD_DEFINITIONS(-D_GRANT) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 4e71684372..75b4531dbb 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -240,7 +240,7 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompac SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact); if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) { sdbFreeRaw(pVgRaw); return -1; } @@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p } static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact"); if (pTrans == NULL) { mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr()); return -1; } mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); @@ -378,7 +380,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - void *pIter = NULL; + void *pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail); @@ -452,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[MND_COMPACT_ID_LEN] = {0}; + char obj[TSDB_INT32_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); @@ -488,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c sdbRelease(pMnode->pSdb, pDetail); } - return -1; + return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST; } int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ SQueryCompactProgressRsp req = {0}; - if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) { + int32_t code = 0; + code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); + if (code != 0) { terrno = TSDB_CODE_INVALID_MSG; + mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", + code, pReq->pCont, pReq->contLen); return -1; } @@ -502,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){ req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); SMnode *pMnode = pReq->info.node; - int32_t code = -1; - - if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){ + code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req); + if(code != 0){ + terrno = code; mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished); return -1; @@ -612,15 +618,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { return 0; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress"); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress"); if (pTrans == NULL) { mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr()); return -1; } mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); - + SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); + mndTransSetDbName(pTrans, pCompact->dbname, NULL); + pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c65c228224..79a5f5fd83 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,6 +14,8 @@ */ #define _DEFAULT_SOURCE +#include +#include "tjson.h" #include "mndDnode.h" #include "audit.h" #include "mndCluster.h" @@ -28,6 +30,7 @@ #include "mndVgroup.h" #include "tmisce.h" #include "tunit.h" +#include "taos_monitor.h" #define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -74,6 +77,7 @@ static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); +static int32_t mndProcessStatisReq(SRpcMsg *pReq); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); @@ -109,6 +113,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_STATIS, mndProcessStatisReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -499,6 +504,207 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +static int32_t mndProcessStatisReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStatisReq statisReq = {0}; + int32_t code = -1; + + char strClusterId[TSDB_CLUSTER_ID_LEN] = {0}; + sprintf(strClusterId, "%"PRId64, pMnode->clusterId); + + if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return code; + } + + if(tsMonitorLogProtocol){ + mInfo("process statis req,\n %s", statisReq.pCont); + } + + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + char tagName[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", tagName); + + if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + tjsonDeleteItemFromObject(item, "value"); + tjsonAddStringToObject(item, "value", strClusterId); + } + } + } + } + } + + char *pCont = tjsonToString(pJson); + monSendContent(pCont); + + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + + if(pCont != NULL){ + taosMemoryFree(pCont); + pCont = NULL; + } + + tFreeSStatisReq(&statisReq); + return 0; + +/* + SJson* pJson = tjsonParse(statisReq.pCont); + + int32_t ts_size = tjsonGetArraySize(pJson); + + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); + + SJson* tables = tjsonGetObjectItem(item, "tables"); + + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); + + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + + int32_t tagSize = tjsonGetArraySize(arrayTag); + + char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); + char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); + + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); + + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + tjsonGetStringValue(item, "name", *(labels + j)); + + *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); + tjsonGetStringValue(item, "value", *(sample_labels + j)); + if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); + } + } + + SJson* metrics = tjsonGetObjectItem(item, "metrics"); + + int32_t metricLen = tjsonGetArraySize(metrics); + for(int32_t j = 0; j < metricLen; j++){ + SJson *item = tjsonGetArrayItem(metrics, j); + + char name[MONITOR_METRIC_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", name); + + double value = 0; + tjsonGetDoubleValue(item, "value", &value); + + double type = 0; + tjsonGetDoubleValue(item, "type", &type); + + int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; + char* metricName = taosMemoryMalloc(metricNameLen); + memset(metricName, 0, metricNameLen); + sprintf(metricName, "%s:%s", tableName, name); + + taos_metric_t* metric = taos_collector_registry_get_metric(metricName); + if(metric == NULL){ + if(type == 0){ + metric = taos_counter_new(metricName, "", tagSize, (const char**)labels); + } + if(type == 1){ + metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels); + } + mTrace("fail to get metric from registry, new one metric:%p", metric); + + if(taos_collector_registry_register_metric(metric) == 1){ + if(type == 0){ + taos_counter_destroy(metric); + } + if(type == 1){ + taos_gauge_destroy(metric); + } + + metric = taos_collector_registry_get_metric(metricName); + + mTrace("fail to register metric, get metric from registry:%p", metric); + } + else{ + mTrace("succeed to register metric:%p", metric); + } + } + else{ + mTrace("get metric from registry:%p", metric); + } + + if(type == 0){ + taos_counter_add(metric, value, (const char**)sample_labels); + } + if(type == 1){ + taos_gauge_set(metric, value, (const char**)sample_labels); + } + + taosMemoryFreeClear(metricName); + } + + for(int32_t j = 0; j < tagSize; j++){ + taosMemoryFreeClear(*(labels + j)); + taosMemoryFreeClear(*(sample_labels + j)); + } + + taosMemoryFreeClear(sample_labels); + taosMemoryFreeClear(labels); + } + } + + } + + code = 0; + +_OVER: + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } + + tFreeSStatisReq(&statisReq); + return code; + */ +} + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a39e9e93c6..69a0bd477d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { int32_t code = syncProcessMsg(pMgmt->sync, pMsg); if (code != 0) { - mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), + terrstr(), code); } return code; @@ -911,11 +912,14 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pObj->id == pMnode->selfDnodeId) { pClusterInfo->first_ep_dnode_id = pObj->id; tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); - pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + //pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; + pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); + desc.syncState = TAOS_SYNC_STATE_LEADER; } else { tstrncpy(desc.role, syncStr(pObj->syncState), sizeof(desc.role)); + desc.syncState = pObj->syncState; } taosArrayPush(pClusterInfo->mnodes, &desc); sdbRelease(pSdb, pObj); @@ -946,6 +950,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; pVnDesc->dnode_id = pVgid->dnodeId; tstrncpy(pVnDesc->vnode_role, syncStr(pVgid->syncState), sizeof(pVnDesc->vnode_role)); + pVnDesc->syncState = pVgid->syncState; if (pVgid->syncState == TAOS_SYNC_STATE_LEADER) { tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; @@ -985,8 +990,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; if (pMnode->grant.expireTimeMS == 0) { - pGrantInfo->expire_time = INT32_MAX; - pGrantInfo->timeseries_total = INT32_MAX; + pGrantInfo->expire_time = 0; + pGrantInfo->timeseries_total = 0; } mndReleaseRpc(pMnode); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5c76baff08..04d74112d4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -751,8 +751,8 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { pTrans->createdTime); int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id); if (code != 0) { - mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), - tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId); + mError("trans:%d, failed to sync, errno:%s code:0x%x createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(), + code, pTrans->createdTime, pMnode->syncMgmt.transId); sdbFreeRaw(pRaw); return -1; } @@ -1100,7 +1100,10 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; - if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; + if (topHalf) { + terrno = TSDB_CODE_MND_TRANS_CTX_SWITCH; + return TSDB_CODE_MND_TRANS_CTX_SWITCH; + } int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1214,7 +1217,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA if (numOfActions == 0) return 0; if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) { - return -1; + return code; } int32_t numOfExecuted = 0; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 06051ee5c8..e610161544 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -47,6 +47,8 @@ #include "vnode.h" +#include "taos_monitor.h" + #ifdef __cplusplus extern "C" { #endif @@ -104,6 +106,20 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" +#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count" + +#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" +#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" +#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" +#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" +#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" +#define VNODE_METRIC_TAG_NAME_USERNAME "username" +#define VNODE_METRIC_TAG_NAME_RESULT "result" + +#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" +//#define VNODE_METRIC_TAG_VALUE_INSERT "insert" +//#define VNODE_METRIC_TAG_VALUE_DELETE "delete" + // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { @@ -445,6 +461,13 @@ typedef struct SVCommitSched { int64_t maxWaitMs; } SVCommitSched; +typedef struct SVMonitorObj{ + char strClusterId[TSDB_CLUSTER_ID_LEN]; + char strDnodeId[TSDB_NODE_ID_LEN]; + char strVgId[TSDB_VGROUP_ID_LEN]; + taos_counter_t *insertCounter; +}SVMonitorObj; + struct SVnode { char* path; SVnodeCfg config; @@ -483,6 +506,7 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; + SVMonitorObj monitor; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index a2c7a04ab3..4a978c8f41 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1008,11 +1008,11 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { } tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); } - + /* if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); } - + */ metaDebug("batch drop table:%" PRId64, uid); } metaULock(pMeta); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 1ef2a451a7..0848fd0076 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_ return code; } -static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { - SLastCol *pLastCol = NULL; - - char *err = NULL; - size_t vlen = 0; - SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; - size_t klen = ROCKS_KEY_LEN; - char *value = NULL; - value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); - if (NULL != err) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); - rocksdb_free(err); - } - - pLastCol = tsdbCacheDeserialize(value); - - return pLastCol; -} - typedef struct { int idx; SLastKey key; @@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); #ifdef BUILD_NO_CALL +static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) { + SLastCol *pLastCol = NULL; + + char *err = NULL; + size_t vlen = 0; + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}; + size_t klen = ROCKS_KEY_LEN; + char *value = NULL; + value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err); + if (NULL != err) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err); + rocksdb_free(err); + } + + pLastCol = tsdbCacheDeserialize(value); + + return pLastCol; +} + int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { rocksdb_writebatch_t *wb = NULL; int32_t code = 0; @@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); - SArray* lastTmpColArray = NULL; - SArray* lastTmpIndexArray = NULL; - SArray* lastrowTmpColArray = NULL; - SArray* lastrowTmpIndexArray = NULL; + SArray *lastTmpColArray = NULL; + SArray *lastTmpIndexArray = NULL; + SArray *lastrowTmpColArray = NULL; + SArray *lastrowTmpIndexArray = NULL; int lastIndex = 0; int lastrowIndex = 0; @@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr SIdxKey *idxKey = taosArrayGet(remainCols, i); slotIds[i] = pr->pSlotIds[idxKey->idx]; if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) { - if(NULL == lastTmpIndexArray) { + if (NULL == lastTmpIndexArray) { lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastTmpIndexArray, &(i)); @@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx]; lastIndex++; } else { - if(NULL == lastrowTmpIndexArray) { + if (NULL == lastrowTmpIndexArray) { lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); } taosArrayPush(lastrowTmpIndexArray, &(i)); @@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); - if(lastTmpIndexArray != NULL) { + if (lastTmpIndexArray != NULL) { mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); - for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i)); } } - if(lastrowTmpIndexArray != NULL) { + if (lastrowTmpIndexArray != NULL) { mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); - for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { - taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); + for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { + taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i), + taosArrayGet(lastrowTmpColArray, i)); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 6aff1c2930..81a136a0e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -448,8 +448,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), - sver, ever, type); + tsdbInfo("vgId:%d, tsdb snapshot incremental reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", + TD_VID(tsdb->pVnode), sver, ever, type); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index b7c22aa0e9..ae840eb6d2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -69,7 +69,8 @@ _exit: taosMemoryFree(reader[0]); reader[0] = NULL; } else { - tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type); + tsdbInfo("vgId:%d, tsdb snapshot raw reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, + type); } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 946ce9d278..63ca7251f5 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -480,6 +480,32 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC vnodeRollback(pVnode); } + snprintf(pVnode->monitor.strClusterId, TSDB_CLUSTER_ID_LEN, "%"PRId64, pVnode->config.syncCfg.nodeInfo[0].clusterId); + snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%"PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); + snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%"PRId32, pVnode->config.vgId); + + if(tsEnableMonitor && pVnode->monitor.insertCounter == NULL){ + taos_counter_t *counter = NULL; + counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); + if(counter == NULL){ + int32_t label_count = 7; + const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID, + VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP, + VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME, + VNODE_METRIC_TAG_NAME_RESULT}; + counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", + label_count, sample_labels); + vInfo("vgId:%d, new metric:%p",TD_VID(pVnode), counter); + if(taos_collector_registry_register_metric(counter) == 1){ + taos_counter_destroy(counter); + counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT); + vInfo("vgId:%d, get metric from registry:%p",TD_VID(pVnode), counter); + } + } + pVnode->monitor.insertCounter = counter; + vInfo("vgId:%d, succeed to set metric:%p",TD_VID(pVnode), counter); + } + return pVnode; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ed1dcc64c9..b4c8a26f5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -172,6 +172,23 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader goto _err; } + // open tsdb snapshot raw reader + if (!pReader->tsdbRAWDone) { + ASSERT(pReader->sver == 0); + code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); + if (code) goto _err; + } + + // check snapshot ever + SSnapshot snapshot = {0}; + vnodeGetSnapshot(pVnode, &snapshot); + if (ever != snapshot.lastApplyIndex) { + vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64, + TD_VID(pVnode), ever, snapshot.lastApplyIndex); + code = TSDB_CODE_SYN_INTERNAL_ERROR; + goto _err; + } + vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 84cbbfd4b2..e32d4b70e0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,6 +16,7 @@ #include "audit.h" #include "cos.h" #include "tencode.h" +#include "tglobal.h" #include "tmsg.h" #include "tstrbuild.h" #include "vnd.h" @@ -30,13 +31,15 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, SRpcMsg *pOriginRpc); -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -542,10 +545,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_DELETE: - if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp, pMsg) < 0) goto _err; break; case TDMT_VND_BATCH_DEL: if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err; @@ -1486,7 +1489,8 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { return code; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; terrno = 0; @@ -1704,10 +1708,35 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + + if(tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){ + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); + } + if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); } + /* + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len); + + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + */ // clear taosArrayDestroy(newTbUids); @@ -1963,7 +1992,8 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe return 0; } -static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, + SRpcMsg *pOriginalMsg) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; @@ -2006,6 +2036,21 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in return code; _err: + /* + if(code == TSDB_CODE_SUCCESS){ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Success"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + else{ + const char *batch_sample_labels[] = {VNODE_METRIC_TAG_VALUE_DELETE, pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, "Failed"}; + taos_counter_inc(pVnode->monitor.insertCounter, batch_sample_labels); + } + */ + return code; } static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f4b7b8442..f9f2ae6b21 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t code = syncProcessMsg(pVnode->sync, pMsg); if (code != 0) { - vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg, - TMSG_INFO(pMsg->msgType), terrstr()); + vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), terrstr(), code); } return code; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 9f812517c1..6196d30a58 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(qcom) add_subdirectory(nodes) add_subdirectory(catalog) add_subdirectory(audit) +add_subdirectory(monitorfw) add_subdirectory(scalar) add_subdirectory(function) diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index 3980e5ae4d..1f38264644 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -94,10 +94,10 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pBlock->info.rows;) { - int32_t step = pInfo->windowSliding; SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow); int32_t prevRows = pBuffInfo->winRows; int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows); + int32_t step = num; if (prevRows == 0) { pInfo->pRow->win.skey = tsCols[i]; } @@ -118,6 +118,8 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (prevRows <= pInfo->windowSliding) { if (pBuffInfo->winRows > pInfo->windowSliding) { step = pInfo->windowSliding - prevRows; + } else { + step = pInfo->windowSliding; } } else { step = 0; @@ -129,7 +131,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { return code; } -static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock) { +static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, SFilterInfo* pFilterInfo, SSDataBlock* pBlock) { SResultRow* pResultRow = NULL; for (int32_t i = 0; i < taosArrayGetSize(pCountSup->pWinStates); i++) { SCountWindowResult* pBuff = setCountWindowOutputBuff(pExprSup, pCountSup, &pResultRow); @@ -143,6 +145,7 @@ static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, S clearWinStateBuff(pBuff); clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs); } + doFilter(pBlock, pFilterInfo, NULL); } static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { @@ -177,7 +180,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { if (pInfo->groupId == 0) { pInfo->groupId = pBlock->info.id.groupId; } else if (pInfo->groupId != pBlock->info.id.groupId) { - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); pInfo->groupId = pBlock->info.id.groupId; } @@ -187,7 +190,7 @@ static SSDataBlock* countWindowAggregate(SOperatorInfo* pOperator) { } } - buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pRes); + buildCountResult(pExprSup, &pInfo->countSup, pTaskInfo, pOperator->exprSupp.pFilterInfo, pRes); return pRes->info.rows == 0 ? NULL : pRes; } @@ -246,6 +249,11 @@ SOperatorInfo* createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo pInfo->countSup.stateIndex = 0; + code = filterInitFromNode((SNode*)pCountWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); setOperatorInfo(pOperator, "CountWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, true, OP_NOT_OPENED, pInfo, diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2534c5e9f0..831fd4e883 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1009,6 +1009,22 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { pSup->deleteMark = INT64_MAX; pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; pInfo->ignoreExpiredData = false; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) { + SStreamCountAggOperatorInfo* pInfo = pOperator->info; + STimeWindowAggSupp* pSup = &pInfo->twAggSup; + + ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0); + + qInfo("save stream param for state: %d, %" PRId64, pSup->calTrigger, pSup->deleteMark); + + pSup->calTriggerSaved = pSup->calTrigger; + pSup->deleteMarkSaved = pSup->deleteMark; + pSup->calTrigger = STREAM_TRIGGER_AT_ONCE; + pSup->deleteMark = INT64_MAX; + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; + qInfo("save stream task:%s, param for state: %d", GET_TASKID(pTaskInfo), pInfo->ignoreExpiredData); } // iterate operator tree diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index cc83ecd84e..691210e5ba 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -461,7 +461,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy _error: destroyIndefinitOperatorInfo(pInfo); taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8cc2f72adb..9024f7a341 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3763,8 +3763,9 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) { taosMemoryFree(pSubTblsInfo); pInfo->pSubTablesMergeInfo = NULL; + + taosMemoryTrim(0); } - taosMemoryTrim(0); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 294c2730df..f9c7b51316 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -287,6 +287,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); @@ -344,7 +345,7 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); + void* key = tSimpleHashGetKey(pIte, &keyLen); tlen += encodeSSessionKey(buf, key); tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize); } @@ -437,7 +438,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, &key); } while (1) { - SSessionKey tmpKey = {0}; + SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { pAggSup->stateStore.streamStateFreeCur(pCur); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 1313221952..b7b9f1cc9f 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -76,8 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) { return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || - type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || + type == TSDB_DATA_TYPE_GEOMETRY) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); @@ -108,8 +108,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { - if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || - dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) { + if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY || + dtype == TSDB_DATA_TYPE_GEOMETRY) { return tDoCompare(func, cmptype, a, b); } #if 1 @@ -290,6 +290,7 @@ int idxUidCompare(const void* a, const void* b) { uint64_t r = *(uint64_t*)b; return l - r; } +#ifdef BUILD_NO_CALL int32_t idxConvertData(void* src, int8_t type, void** dst) { int tlen = -1; switch (type) { @@ -372,6 +373,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { // indexMayFillNumbericData(*dst, tlen); return tlen; } +#endif + int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); diff --git a/source/libs/monitor/CMakeLists.txt b/source/libs/monitor/CMakeLists.txt index 30dce7aaef..cc8f40fa4c 100644 --- a/source/libs/monitor/CMakeLists.txt +++ b/source/libs/monitor/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(monitor os util common transport) +target_link_libraries(monitor os util common qcom transport monitorfw) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index ae1af4ba62..7fc718393b 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -17,8 +17,10 @@ #define _TD_MONITOR_INT_H_ #include "monitor.h" +#include "query.h" #include "tjson.h" +#include "thash.h" typedef struct { int64_t curTime; @@ -44,8 +46,22 @@ typedef struct { SMonSmInfo smInfo; SMonQmInfo qmInfo; SMonBmInfo bmInfo; + SHashObj *metrics; } SMonitor; +void monGenClusterInfoTable(SMonInfo *pMonitor); +void monGenVgroupInfoTable(SMonInfo *pMonitor); +void monGenDnodeInfoTable(SMonInfo *pMonitor); +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor); +void monGenDataDiskTable(SMonInfo *pMonitor); +void monGenLogDiskTable(SMonInfo *pMonitor); +void monGenMnodeRoleTable(SMonInfo *pMonitor); +void monGenVnodeRoleTable(SMonInfo *pMonitor); + +void monSendPromReport(); +void monInitMonitorFW(); +void monCleanupMonitorFW(); + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/clientMonitor.c b/source/libs/monitor/src/clientMonitor.c new file mode 100644 index 0000000000..0891932583 --- /dev/null +++ b/source/libs/monitor/src/clientMonitor.c @@ -0,0 +1,210 @@ +#include "clientMonitor.h" +#include "os.h" +#include "tmisce.h" +#include "ttime.h" +#include "ttimer.h" +#include "tglobal.h" + +SRWLatch monitorLock; +void* tmrClientMonitor; +tmr_h tmrStartHandle; +SHashObj* clusterMonitorInfoTable; + +static int interval = 30 * 1000; +static int sendBathchSize = 1; + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont); +void generateClusterReport(ClientMonitor* pMonitor, bool send) { + char ts[50]; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + char* pCont = (char*)taos_collector_registry_bridge_new(pMonitor->registry, ts, "%" PRId64, NULL); + if(NULL == pCont) { + uError("generateClusterReport failed, get null content."); + return; + } + if (send && strlen(pCont) != 0) { + if (sendReport(pMonitor, pCont) == 0) { + taos_collector_registry_clear_batch(pMonitor->registry); + } + } + taosMemoryFreeClear(pCont); +} + +void reportSendProcess(void* param, void* tmrId) { + taosTmrReset(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor, &tmrStartHandle); + taosRLockLatch(&monitorLock); + + static int index = 0; + index++; + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashIterate(clusterMonitorInfoTable, NULL); + while (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + generateClusterReport(*ppMonitor, index == sendBathchSize); + ppMonitor = taosHashIterate(clusterMonitorInfoTable, ppMonitor); + } + + if (index == sendBathchSize) index = 0; + taosRUnLockLatch(&monitorLock); +} + +void monitorClientInitOnce() { + static int8_t init = 0; + if (atomic_exchange_8(&init, 1) == 0) { + uInfo("tscMonitorInit once."); + clusterMonitorInfoTable = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + tmrClientMonitor = taosTmrInit(0, 0, 0, "MONITOR"); + tmrStartHandle = taosTmrStart(reportSendProcess, tsMonitorInterval * 1000, NULL, tmrClientMonitor); + if(tsMonitorInterval < 1){ + interval = 30 * 1000; + } else { + interval = tsMonitorInterval * 1000; + } + if (tsMonitorInterval < 10) { + sendBathchSize = (10 / sendBathchSize) + 1; + } + taosInitRWLatch(&monitorLock); + } +} + +void createMonitorClient(const char* clusterKey, SEpSet epSet, void* pTransporter) { + if (clusterKey == NULL || strlen(clusterKey) == 0) { + uError("createMonitorClient failed, clusterKey is NULL"); + return; + } + taosWLockLatch(&monitorLock); + if (taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)) == NULL) { + uInfo("createMonitorClient for %s.", clusterKey); + ClientMonitor* pMonitor = taosMemoryCalloc(1, sizeof(ClientMonitor)); + snprintf(pMonitor->clusterKey, sizeof(pMonitor->clusterKey), "%s", clusterKey); + pMonitor->registry = taos_collector_registry_new(clusterKey); + pMonitor->colector = taos_collector_new(clusterKey); + epsetAssign(&pMonitor->epSet, &epSet); + pMonitor->pTransporter = pTransporter; + + taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); + pMonitor->counters = + (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosHashPut(clusterMonitorInfoTable, clusterKey, strlen(clusterKey), &pMonitor, sizeof(ClientMonitor*)); + uInfo("createMonitorClient for %s finished %p.", clusterKey, pMonitor); + } + taosWUnLockLatch(&monitorLock); +} + +static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) { + static int32_t emptyRspNum = 0; + if (TSDB_CODE_SUCCESS != code) { + uError("found error in monitorReport send callback, code:%d, please check the network.", code); + } + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + return code; +} + +int32_t sendReport(ClientMonitor* pMonitor, char* pCont) { + SStatisReq sStatisReq; + sStatisReq.pCont = pCont; + sStatisReq.contLen = strlen(pCont); + + int tlen = tSerializeSStatisReq(NULL, 0, &sStatisReq); + if (tlen < 0) return 0; + void* buf = taosMemoryMalloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + tSerializeSStatisReq(buf, tlen, &sStatisReq); + + SMsgSendInfo* pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pInfo->fp = monitorReportAsyncCB; + pInfo->msgInfo.pData = buf; + pInfo->msgInfo.len = tlen; + pInfo->msgType = TDMT_MND_STATIS; + // pInfo->param = taosMemoryMalloc(sizeof(int32_t)); + // *(int32_t*)pInfo->param = i; + pInfo->paramFreeFp = taosMemoryFree; + pInfo->requestId = tGenIdPI64(); + pInfo->requestObjRefId = 0; + + int64_t transporterId = 0; + return asyncSendMsgToServer(pMonitor->pTransporter, &pMonitor->epSet, &transporterId, pInfo); +} + +void clusterMonitorInit(const char* clusterKey, SEpSet epSet, void* pTransporter) { + monitorClientInitOnce(); + createMonitorClient(clusterKey, epSet, pTransporter); +} + +taos_counter_t* createClusterCounter(const char* clusterKey, const char* name, const char* help, size_t label_key_count, + const char** label_keys) { + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, name, strlen(name)); + if (ppCounter != NULL && *ppCounter != NULL) { + taosHashRemove(pMonitor->counters, name, strlen(name)); + uInfo("createClusterCounter remove old counter: %s.", name); + } + + taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys); + if (newCounter != NULL) { + taos_collector_add_metric(pMonitor->colector, newCounter); + taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, sizeof(taos_counter_t*)); + uInfo("createClusterCounter %s(%p):%s : %p.", pMonitor->clusterKey, pMonitor, name, newCounter); + return newCounter; + } else { + return NULL; + } + } else { + return NULL; + } + return NULL; +} + +int taosClusterCounterInc(const char* clusterKey, const char* counterName, const char** label_values) { + taosRLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); + if (ppCounter != NULL && *ppCounter != NULL) { + taos_counter_inc(*ppCounter, label_values); + } else { + uError("taosClusterCounterInc not found pCounter %s:%s.", clusterKey, counterName); + } + } else { + uError("taosClusterCounterInc not found pMonitor %s.", clusterKey); + } + taosRUnLockLatch(&monitorLock); + return 0; +} + +void clusterMonitorClose(const char* clusterKey) { + taosWLockLatch(&monitorLock); + ClientMonitor** ppMonitor = (ClientMonitor**)taosHashGet(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + + if (ppMonitor != NULL && *ppMonitor != NULL) { + ClientMonitor* pMonitor = *ppMonitor; + uInfo("clusterMonitorClose valule:%p clusterKey:%s.", pMonitor, pMonitor->clusterKey); + taosHashCleanup(pMonitor->counters); + taos_collector_registry_destroy(pMonitor->registry); + taosMemoryFree(pMonitor); + taosHashRemove(clusterMonitorInfoTable, clusterKey, strlen(clusterKey)); + } + taosWUnLockLatch(&monitorLock); +} + +const char* resultStr(SQL_RESULT_CODE code) { + static const char* result_state[] = {"Success", "Failed", "Cancel"}; + return result_state[code]; +} diff --git a/source/libs/monitor/src/monFramework.c b/source/libs/monitor/src/monFramework.c new file mode 100644 index 0000000000..488b4d1d58 --- /dev/null +++ b/source/libs/monitor/src/monFramework.c @@ -0,0 +1,763 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "monitor.h" +#include "monInt.h" + +#include "thash.h" +#include "taos_monitor.h" +#include "thttp.h" +#include "ttime.h" +#include "tglobal.h" + +extern SMonitor tsMonitor; +extern char* tsMonUri; +extern char* tsMonFwUri; + +#define LEVEL_LEN 11 + +#define CLUSTER_TABLE "taosd_cluster_info" + +#define MASTER_UPTIME CLUSTER_TABLE":cluster_uptime" +#define DBS_TOTAL CLUSTER_TABLE":dbs_total" +#define TBS_TOTAL CLUSTER_TABLE":tbs_total" +#define STBS_TOTAL CLUSTER_TABLE":stbs_total" +#define VGROUPS_TOTAL CLUSTER_TABLE":vgroups_total" +#define VGROUPS_ALIVE CLUSTER_TABLE":vgroups_alive" +#define VNODES_TOTAL CLUSTER_TABLE":vnodes_total" +#define VNODES_ALIVE CLUSTER_TABLE":vnodes_alive" +#define DNODES_TOTAL CLUSTER_TABLE":dnodes_total" +#define DNODES_ALIVE CLUSTER_TABLE":dnodes_alive" +#define MNODES_TOTAL CLUSTER_TABLE":mnodes_total" +#define MNODES_ALIVE CLUSTER_TABLE":mnodes_alive" +#define CONNECTIONS_TOTAL CLUSTER_TABLE":connections_total" +#define TOPICS_TOTAL CLUSTER_TABLE":topics_total" +#define STREAMS_TOTAL CLUSTER_TABLE":streams_total" +#define EXPIRE_TIME CLUSTER_TABLE":grants_expire_time" +#define TIMESERIES_USED CLUSTER_TABLE":grants_timeseries_used" +#define TIMESERIES_TOTAL CLUSTER_TABLE":grants_timeseries_total" + +#define VGROUP_TABLE "taosd_vgroups_info" + +#define TABLES_NUM VGROUP_TABLE":tables_num" +#define STATUS VGROUP_TABLE":status" + +#define DNODE_TABLE "taosd_dnodes_info" + +#define UPTIME DNODE_TABLE":uptime" +#define CPU_ENGINE DNODE_TABLE":cpu_engine" +#define CPU_SYSTEM DNODE_TABLE":cpu_system" +#define CPU_CORE DNODE_TABLE":cpu_cores" +#define MEM_ENGINE DNODE_TABLE":mem_engine" +#define MEM_SYSTEM DNODE_TABLE":mem_free" +#define MEM_TOTAL DNODE_TABLE":mem_total" +#define DISK_ENGINE DNODE_TABLE":disk_engine" +#define DISK_USED DNODE_TABLE":disk_used" +#define DISK_TOTAL DNODE_TABLE":disk_total" +#define NET_IN DNODE_TABLE":system_net_in" +#define NET_OUT DNODE_TABLE":system_net_out" +#define IO_READ DNODE_TABLE":io_read" +#define IO_WRITE DNODE_TABLE":io_write" +#define IO_READ_DISK DNODE_TABLE":io_read_disk" +#define IO_WRITE_DISK DNODE_TABLE":io_write_disk" +//#define ERRORS DNODE_TABLE":errors" +#define VNODES_NUM DNODE_TABLE":vnodes_num" +#define MASTERS DNODE_TABLE":masters" +#define HAS_MNODE DNODE_TABLE":has_mnode" +#define HAS_QNODE DNODE_TABLE":has_qnode" +#define HAS_SNODE DNODE_TABLE":has_snode" +#define DNODE_LOG_ERROR DNODE_TABLE":error_log_count" +#define DNODE_LOG_INFO DNODE_TABLE":info_log_count" +#define DNODE_LOG_DEBUG DNODE_TABLE":debug_log_count" +#define DNODE_LOG_TRACE DNODE_TABLE":trace_log_count" + +#define DNODE_STATUS "taosd_dnodes_status:status" + +#define DATADIR_TABLE "taosd_dnodes_data_dirs" + +#define DNODE_DATA_AVAIL DATADIR_TABLE":avail" +#define DNODE_DATA_USED DATADIR_TABLE":used" +#define DNODE_DATA_TOTAL DATADIR_TABLE":total" + +#define LOGDIR_TABLE "taosd_dnodes_log_dirs" + +#define DNODE_LOG_AVAIL LOGDIR_TABLE":avail" +#define DNODE_LOG_USED LOGDIR_TABLE":used" +#define DNODE_LOG_TOTAL LOGDIR_TABLE":total" + +#define MNODE_ROLE "taosd_mnodes_info:role" +#define VNODE_ROLE "taosd_vnodes_info:role" + +void monInitMonitorFW(){ + taos_collector_registry_default_init(); + + tsMonitor.metrics = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + taos_gauge_t *gauge = NULL; + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + char *dnodes_gauges[] = {UPTIME, CPU_ENGINE, CPU_SYSTEM, CPU_CORE, MEM_ENGINE, MEM_SYSTEM, + MEM_TOTAL, DISK_ENGINE, DISK_USED, DISK_TOTAL, NET_IN, + NET_OUT, IO_READ, IO_WRITE, IO_READ_DISK, IO_WRITE_DISK, /*ERRORS,*/ + VNODES_NUM, MASTERS, HAS_MNODE, HAS_QNODE, HAS_SNODE, + DNODE_LOG_ERROR, DNODE_LOG_INFO, DNODE_LOG_DEBUG, DNODE_LOG_TRACE}; + for(int32_t i = 0; i < 25; i++){ + gauge= taos_gauge_new(dnodes_gauges[i], "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_gauges[i], strlen(dnodes_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_data_label_count = 5; + const char *dnodes_data_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name", "data_dir_level"}; + char *dnodes_data_gauges[] = {DNODE_DATA_AVAIL, DNODE_DATA_USED, DNODE_DATA_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_data_gauges[i], "", dnodes_data_label_count, dnodes_data_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_data_gauges[i], strlen(dnodes_data_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + int32_t dnodes_log_label_count = 4; + const char *dnodes_log_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep", "data_dir_name"}; + char *dnodes_log_gauges[] = {DNODE_LOG_AVAIL, DNODE_LOG_USED, DNODE_LOG_TOTAL}; + for(int32_t i = 0; i < 3; i++){ + gauge= taos_gauge_new(dnodes_log_gauges[i], "", dnodes_log_label_count, dnodes_log_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, dnodes_log_gauges[i], strlen(dnodes_log_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } +} + +void monCleanupMonitorFW(){ + taosHashCleanup(tsMonitor.metrics); + taos_collector_registry_destroy(TAOS_COLLECTOR_REGISTRY_DEFAULT); + TAOS_COLLECTOR_REGISTRY_DEFAULT = NULL; +} + +void monGenClusterInfoTable(SMonInfo *pMonitor){ + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + SMonGrantInfo *pGrantInfo = &pMonitor->mmInfo.grant; + + char *metric_names[] = {MASTER_UPTIME, DBS_TOTAL, TBS_TOTAL, STBS_TOTAL, VGROUPS_TOTAL, + VGROUPS_ALIVE, VNODES_TOTAL, VNODES_ALIVE, MNODES_TOTAL, MNODES_ALIVE, + CONNECTIONS_TOTAL, TOPICS_TOTAL, STREAMS_TOTAL, + DNODES_TOTAL, DNODES_ALIVE, EXPIRE_TIME, TIMESERIES_USED, + TIMESERIES_TOTAL}; + + for(int32_t i = 0; i < 18; i++){ + if(taos_collector_registry_deregister_metric(metric_names[i]) != 0){ + uError("failed to delete metric %s", metric_names[i]); + } + + taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])); + } + + if(pBasicInfo->cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + //cluster info + taos_gauge_t *gauge = NULL; + int32_t label_count =1; + const char *sample_labels1[] = {"cluster_id"}; + + for(int32_t i = 0; i < 18; i++){ + gauge= taos_gauge_new(metric_names[i], "", label_count, sample_labels1); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, metric_names[i], strlen(metric_names[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%"PRId64, pBasicInfo->cluster_id); + const char *sample_label_values[] = {buf}; + + taos_gauge_t **metric = NULL; + + metric = taosHashGet(tsMonitor.metrics, MASTER_UPTIME, strlen(MASTER_UPTIME)); + taos_gauge_set(*metric, pInfo->master_uptime, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, DBS_TOTAL, strlen(DBS_TOTAL)); + taos_gauge_set(*metric, pInfo->dbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TBS_TOTAL, strlen(TBS_TOTAL)); + taos_gauge_set(*metric, pInfo->tbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, STBS_TOTAL, strlen(STBS_TOTAL)); + taos_gauge_set(*metric, pInfo->stbs_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_TOTAL, strlen(VGROUPS_TOTAL)); + taos_gauge_set(*metric, pInfo->vgroups_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VGROUPS_ALIVE, strlen(VGROUPS_ALIVE)); + taos_gauge_set(*metric, pInfo->vgroups_alive, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VNODES_TOTAL, strlen(VNODES_TOTAL)); + taos_gauge_set(*metric, pInfo->vnodes_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, VNODES_ALIVE, strlen(VNODES_ALIVE)); + taos_gauge_set(*metric, pInfo->vnodes_alive, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, CONNECTIONS_TOTAL, strlen(CONNECTIONS_TOTAL)); + taos_gauge_set(*metric, pInfo->connections_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TOPICS_TOTAL, strlen(TOPICS_TOTAL)); + taos_gauge_set(*metric, pInfo->topics_toal, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, STREAMS_TOTAL, strlen(STREAMS_TOTAL)); + taos_gauge_set(*metric, pInfo->streams_total, sample_label_values); + + //dnodes number + int32_t dnode_total = taosArrayGetSize(pInfo->dnodes); + int32_t dnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnode_alive++; + } + } + + metric = taosHashGet(tsMonitor.metrics, DNODES_TOTAL, strlen(DNODES_TOTAL)); + taos_gauge_set(*metric, dnode_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, DNODES_ALIVE, strlen(DNODES_ALIVE)); + taos_gauge_set(*metric, dnode_alive, sample_label_values); + + //mnodes number + int32_t mnode_total = taosArrayGetSize(pInfo->mnodes); + int32_t mnode_alive = 0; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + + if(dnodeIsOnline){ + if(pMnodeDesc->syncState != 0){ + mnode_alive++; + } + } + } + + metric = taosHashGet(tsMonitor.metrics, MNODES_TOTAL, strlen(MNODES_TOTAL)); + taos_gauge_set(*metric, mnode_total, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, MNODES_ALIVE, strlen(MNODES_ALIVE)); + taos_gauge_set(*metric, mnode_alive, sample_label_values); + + //grant info + metric = taosHashGet(tsMonitor.metrics, EXPIRE_TIME, strlen(EXPIRE_TIME)); + taos_gauge_set(*metric, pGrantInfo->expire_time, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_USED, strlen(TIMESERIES_USED)); + taos_gauge_set(*metric, pGrantInfo->timeseries_used, sample_label_values); + + metric = taosHashGet(tsMonitor.metrics, TIMESERIES_TOTAL, strlen(TIMESERIES_TOTAL)); + taos_gauge_set(*metric, pGrantInfo->timeseries_total, sample_label_values); +} + +void monGenVgroupInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(TABLES_NUM) != 0){ + uError("failed to delete metric "TABLES_NUM); + } + + if(taos_collector_registry_deregister_metric(STATUS) != 0){ + uError("failed to delete metric "STATUS); + } + + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + int32_t vgroup_label_count = 3; + const char *vgroup_sample_labels[] = {"cluster_id", "vgroup_id", "database_name"}; + taos_gauge_t *tableNumGauge = taos_gauge_new(TABLES_NUM, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(tableNumGauge) == 1){ + taos_counter_destroy(tableNumGauge); + } + taos_gauge_t *statusGauge = taos_gauge_new(STATUS, "", vgroup_label_count, vgroup_sample_labels); + if(taos_collector_registry_register_metric(statusGauge) == 1){ + taos_counter_destroy(statusGauge); + } + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_NODE_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + const char *sample_labels[] = {cluster_id, vgroup_id, pVgroupDesc->database_name}; + + taos_gauge_t **metric = NULL; + + taos_gauge_set(tableNumGauge, pVgroupDesc->tables_num, sample_labels); + + int32_t status = 0; + if(strcmp(pVgroupDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(statusGauge, status, sample_labels); + } +} + +void monGenDnodeInfoTable(SMonInfo *pMonitor) { + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep}; + + taos_gauge_t **metric = NULL; + + //dnode info + SMonDnodeInfo *pInfo = &pMonitor->dmInfo.dnode; + SMonSysInfo *pSys = &pMonitor->dmInfo.sys; + SVnodesStat *pStat = &pMonitor->vmInfo.vstat; + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + double interval = (pMonitor->curTime - pMonitor->lastTime) / 1000.0; + if (pMonitor->curTime - pMonitor->lastTime == 0) { + interval = 1; + } + + double cpu_engine = 0; + double mem_engine = 0; + double net_in = 0; + double net_out = 0; + double io_read = 0; + double io_write = 0; + double io_read_disk = 0; + double io_write_disk = 0; + + SMonSysInfo *sysArrays[6]; + sysArrays[0] = &pMonitor->dmInfo.sys; + sysArrays[1] = &pMonitor->mmInfo.sys; + sysArrays[2] = &pMonitor->vmInfo.sys; + sysArrays[3] = &pMonitor->qmInfo.sys; + sysArrays[4] = &pMonitor->smInfo.sys; + sysArrays[5] = &pMonitor->bmInfo.sys; + for (int32_t i = 0; i < 6; ++i) { + cpu_engine += sysArrays[i]->cpu_engine; + mem_engine += sysArrays[i]->mem_engine; + net_in += sysArrays[i]->net_in; + net_out += sysArrays[i]->net_out; + io_read += sysArrays[i]->io_read; + io_write += sysArrays[i]->io_write; + io_read_disk += sysArrays[i]->io_read_disk; + io_write_disk += sysArrays[i]->io_write_disk; + } + + double req_select_rate = pStat->numOfSelectReqs / interval; + double req_insert_rate = pStat->numOfInsertReqs / interval; + double req_insert_batch_rate = pStat->numOfBatchInsertReqs / interval; + double net_in_rate = net_in / interval; + double net_out_rate = net_out / interval; + double io_read_rate = io_read / interval; + double io_write_rate = io_write / interval; + double io_read_disk_rate = io_read_disk / interval; + double io_write_disk_rate = io_write_disk / interval; + + metric = taosHashGet(tsMonitor.metrics, UPTIME, strlen(UPTIME)); + taos_gauge_set(*metric, pInfo->uptime, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_ENGINE, strlen(CPU_ENGINE)); + taos_gauge_set(*metric, cpu_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_SYSTEM, strlen(CPU_SYSTEM)); + taos_gauge_set(*metric, pSys->cpu_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, CPU_CORE, strlen(CPU_CORE)); + taos_gauge_set(*metric, pSys->cpu_cores, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_ENGINE, strlen(MEM_ENGINE)); + taos_gauge_set(*metric, mem_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_SYSTEM, strlen(MEM_SYSTEM)); + taos_gauge_set(*metric, pSys->mem_system, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MEM_TOTAL, strlen(MEM_TOTAL)); + taos_gauge_set(*metric, pSys->mem_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_ENGINE, strlen(DISK_ENGINE)); + taos_gauge_set(*metric, pSys->disk_engine, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_USED, strlen(DISK_USED)); + taos_gauge_set(*metric, pSys->disk_used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DISK_TOTAL, strlen(DISK_TOTAL)); + taos_gauge_set(*metric, pSys->disk_total, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_IN, strlen(NET_IN)); + taos_gauge_set(*metric, net_in_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, NET_OUT, strlen(NET_OUT)); + taos_gauge_set(*metric, net_out_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ, strlen(IO_READ)); + taos_gauge_set(*metric, io_read_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE, strlen(IO_WRITE)); + taos_gauge_set(*metric, io_write_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_READ_DISK, strlen(IO_READ_DISK)); + taos_gauge_set(*metric, io_read_disk_rate, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, IO_WRITE_DISK, strlen(IO_WRITE_DISK)); + taos_gauge_set(*metric, io_write_disk_rate, sample_labels); + + //metric = taosHashGet(tsMonitor.metrics, ERRORS, strlen(ERRORS)); + //taos_gauge_set(*metric, pStat->errors, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, VNODES_NUM, strlen(VNODES_NUM)); + taos_gauge_set(*metric, pStat->totalVnodes, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, MASTERS, strlen(MASTERS)); + taos_gauge_set(*metric, pStat->masterNum, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_MNODE, strlen(HAS_MNODE)); + taos_gauge_set(*metric, pInfo->has_mnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_QNODE, strlen(HAS_QNODE)); + taos_gauge_set(*metric, pInfo->has_qnode, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, HAS_SNODE, strlen(HAS_SNODE)); + taos_gauge_set(*metric, pInfo->has_snode, sample_labels); + + //log number + SMonLogs *logs[6]; + logs[0] = &pMonitor->log; + logs[1] = &pMonitor->mmInfo.log; + logs[2] = &pMonitor->vmInfo.log; + logs[3] = &pMonitor->smInfo.log; + logs[4] = &pMonitor->qmInfo.log; + logs[5] = &pMonitor->bmInfo.log; + + int32_t numOfErrorLogs = 0; + int32_t numOfInfoLogs = 0; + int32_t numOfDebugLogs = 0; + int32_t numOfTraceLogs = 0; + + for (int32_t j = 0; j < 6; j++) { + SMonLogs *pLog = logs[j]; + numOfErrorLogs += pLog->numOfErrorLogs; + numOfInfoLogs += pLog->numOfInfoLogs; + numOfDebugLogs += pLog->numOfDebugLogs; + numOfTraceLogs += pLog->numOfTraceLogs; + } + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_ERROR, strlen(DNODE_LOG_ERROR)); + taos_gauge_set(*metric, numOfErrorLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_INFO, strlen(DNODE_LOG_INFO)); + taos_gauge_set(*metric, numOfInfoLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_DEBUG, strlen(DNODE_LOG_DEBUG)); + taos_gauge_set(*metric, numOfDebugLogs, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TRACE, strlen(DNODE_LOG_TRACE)); + taos_gauge_set(*metric, numOfTraceLogs, sample_labels); +} + +void monGenDnodeStatusInfoTable(SMonInfo *pMonitor){ + if(taos_collector_registry_deregister_metric(DNODE_STATUS) != 0){ + uError("failed to delete metric "DNODE_STATUS); + } + + if(pMonitor->dmInfo.basic.cluster_id == 0) { + uError("failed to generate dnode info table since cluster_id is 0"); + return; + } + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + taos_gauge_t *gauge = NULL; + + int32_t dnodes_label_count = 3; + const char *dnodes_sample_labels[] = {"cluster_id", "dnode_id", "dnode_ep"}; + + gauge= taos_gauge_new(DNODE_STATUS, "", dnodes_label_count, dnodes_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + + char cluster_id[TSDB_CLUSTER_ID_LEN]; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%"PRId64, pMonitor->dmInfo.basic.cluster_id); + + //dnodes status + + SMonClusterInfo *pClusterInfo = &pMonitor->mmInfo.cluster; + + for (int32_t i = 0; i < taosArrayGetSize(pClusterInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pClusterInfo->dnodes, i); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pDnodeDesc->dnode_id); + + const char *sample_labels[] = {cluster_id, dnode_id, pDnodeDesc->dnode_ep}; + + int32_t status = 0; + if(strcmp(pDnodeDesc->status, "ready") == 0){ + status = 1; + } + taos_gauge_set(gauge, status, sample_labels); + } +} + +void monGenDataDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskInfo *pInfo = &pMonitor->vmInfo.tfs; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { + SMonDiskDesc *pDatadirDesc = taosArrayGet(pInfo->datadirs, i); + + char level[LEVEL_LEN] = {0}; + snprintf(level, LEVEL_LEN, "%"PRId32, pDatadirDesc->level); + + const char *sample_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pDatadirDesc->name, level}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_AVAIL, strlen(DNODE_DATA_AVAIL)); + taos_gauge_set(*metric, pDatadirDesc->size.avail, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_USED, strlen(DNODE_DATA_USED)); + taos_gauge_set(*metric, pDatadirDesc->size.used, sample_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_DATA_TOTAL, strlen(DNODE_DATA_TOTAL)); + taos_gauge_set(*metric, pDatadirDesc->size.total, sample_labels); + } +} + +void monGenLogDiskTable(SMonInfo *pMonitor){ + if(pMonitor->dmInfo.basic.cluster_id == 0) return; + + SMonDiskDesc *pLogDesc = &pMonitor->dmInfo.dnode.logdir; + SMonDiskDesc *pTempDesc = &pMonitor->dmInfo.dnode.tempdir; + + char cluster_id[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(cluster_id, TSDB_CLUSTER_ID_LEN, "%" PRId64, pMonitor->dmInfo.basic.cluster_id); + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMonitor->dmInfo.basic.dnode_id); + + taos_gauge_t **metric = NULL; + + const char *sample_log_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pLogDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pLogDesc->size.avail, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pLogDesc->size.used, sample_log_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pLogDesc->size.total, sample_log_labels); + + const char *sample_temp_labels[] = {cluster_id, dnode_id, pMonitor->dmInfo.basic.dnode_ep, pTempDesc->name}; + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_AVAIL, strlen(DNODE_LOG_AVAIL)); + taos_gauge_set(*metric, pTempDesc->size.avail, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_USED, strlen(DNODE_LOG_USED)); + taos_gauge_set(*metric, pTempDesc->size.used, sample_temp_labels); + + metric = taosHashGet(tsMonitor.metrics, DNODE_LOG_TOTAL, strlen(DNODE_LOG_TOTAL)); + taos_gauge_set(*metric, pTempDesc->size.total, sample_temp_labels); +} + +void monGenMnodeRoleTable(SMonInfo *pMonitor){ + char *mnodes_role_gauges[] = {MNODE_ROLE}; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(mnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", mnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])); + } + + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + taos_gauge_t *gauge = NULL; + int32_t mnodes_role_label_count = 3; + const char *mnodes_role_sample_labels[] = {"cluster_id", "mnode_id", "mnode_ep"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(mnodes_role_gauges[i], "", mnodes_role_label_count, mnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); + + char mnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(mnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pMnodeDesc->mnode_id); + + const char *sample_labels[] = {buf, mnode_id, pMnodeDesc->mnode_ep}; + + bool dnodeIsOnline = false; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDnodeDesc = taosArrayGet(pInfo->dnodes, i); + + if(pDnodeDesc->dnode_id == pMnodeDesc->mnode_id){ + if(strcmp(pDnodeDesc->status, "ready") == 0){ + dnodeIsOnline = true; + } + } + } + + metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + + if(dnodeIsOnline){ + taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } + else{ + taos_gauge_set(*metric, 0, sample_labels); + } + //metric = taosHashGet(tsMonitor.metrics, MNODE_ROLE, strlen(MNODE_ROLE)); + //taos_gauge_set(*metric, pMnodeDesc->syncState, sample_labels); + } +} + +void monGenVnodeRoleTable(SMonInfo *pMonitor){ + char *vnodes_role_gauges[] = {VNODE_ROLE}; + taos_gauge_t *gauge = NULL; + + for(int32_t i = 0; i < 1; i++){ + if(taos_collector_registry_deregister_metric(vnodes_role_gauges[i]) != 0){ + uError("failed to delete metric %s", vnodes_role_gauges[i]); + } + + taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])); + } + + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SMonBasicInfo *pBasicInfo = &pMonitor->dmInfo.basic; + if(pBasicInfo->cluster_id == 0) return; + + int32_t vnodes_role_label_count = 4; + const char *vnodes_role_sample_labels[] = {"cluster_id", "vgroup_id", "database_name", "dnode_id"}; + for(int32_t i = 0; i < 1; i++){ + gauge= taos_gauge_new(vnodes_role_gauges[i], "", vnodes_role_label_count, vnodes_role_sample_labels); + if(taos_collector_registry_register_metric(gauge) == 1){ + taos_counter_destroy(gauge); + } + taosHashPut(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i]), &gauge, sizeof(taos_gauge_t *)); + } + + char buf[TSDB_CLUSTER_ID_LEN] = {0}; + snprintf(buf, TSDB_CLUSTER_ID_LEN, "%" PRId64, pBasicInfo->cluster_id); + + taos_gauge_t **metric = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pVgroupDesc = taosArrayGet(pInfo->vgroups, i); + + char vgroup_id[TSDB_VGROUP_ID_LEN] = {0}; + snprintf(vgroup_id, TSDB_VGROUP_ID_LEN, "%"PRId32, pVgroupDesc->vgroup_id); + + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc *pVnodeDesc = &pVgroupDesc->vnodes[j]; + if (pVnodeDesc->dnode_id <= 0) continue; + + char dnode_id[TSDB_NODE_ID_LEN] = {0}; + snprintf(dnode_id, TSDB_NODE_ID_LEN, "%"PRId32, pVnodeDesc->dnode_id); + + const char *sample_labels[] = {buf, vgroup_id, pVgroupDesc->database_name, dnode_id}; + + metric = taosHashGet(tsMonitor.metrics, VNODE_ROLE, strlen(VNODE_ROLE)); + taos_gauge_set(*metric, pVnodeDesc->syncState, sample_labels); + } + } +} + +void monSendPromReport() { + char ts[50] = {0}; + sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + + char* promStr = NULL; + char* pCont = (char *)taos_collector_registry_bridge_new(TAOS_COLLECTOR_REGISTRY_DEFAULT, ts, "%" PRId64, &promStr); + if(tsMonitorLogProtocol){ + if(pCont != NULL){ + uInfoL("report cont:\n%s", pCont); + uDebugL("report cont prom:\n%s", promStr); + } + else{ + uInfo("report cont is null"); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + }else{ + taos_collector_registry_clear_batch(TAOS_COLLECTOR_REGISTRY_DEFAULT); + } + taosMemoryFreeClear(pCont); + } + if(promStr != NULL){ + taosMemoryFreeClear(promStr); + } +} \ No newline at end of file diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 56cf0a2b51..4ecc6f1261 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -18,9 +18,13 @@ #include "taoserror.h" #include "thttp.h" #include "ttime.h" +#include "taos_monitor.h" +#include "tglobal.h" -static SMonitor tsMonitor = {0}; -static char* tsMonUri = "/report"; +SMonitor tsMonitor = {0}; +char* tsMonUri = "/report"; +char* tsMonFwUri = "/general-metric"; +char* tsMonFwBasicUri = "/taosd-cluster-basic"; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); @@ -108,6 +112,9 @@ int32_t monInit(const SMonCfg *pCfg) { tsLogFp = monRecordLog; tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); + + monInitMonitorFW(); + return 0; } @@ -121,6 +128,8 @@ void monCleanup() { tFreeSMonQmInfo(&tsMonitor.qmInfo); tFreeSMonBmInfo(&tsMonitor.bmInfo); taosThreadMutexDestroy(&tsMonitor.lock); + + monCleanupMonitorFW(); } static void monCleanupMonitorInfo(SMonInfo *pMonitor) { @@ -186,6 +195,22 @@ static void monGenBasicJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); } +static void monGenBasicJsonBasic(SMonInfo *pMonitor) { + SMonBasicInfo *pInfo = &pMonitor->dmInfo.basic; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SJson *pJson = pMonitor->pJson; + char buf[40] = {0}; + + sprintf(buf, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + tjsonAddStringToObject(pJson, "ts", buf); + tjsonAddDoubleToObject(pJson, "dnode_id", pInfo->dnode_id); + tjsonAddStringToObject(pJson, "dnode_ep", pInfo->dnode_ep); + snprintf(buf, sizeof(buf), "%" PRId64, pInfo->cluster_id); + tjsonAddStringToObject(pJson, "cluster_id", buf); + tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); +} + static void monGenClusterJson(SMonInfo *pMonitor) { SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -244,6 +269,17 @@ static void monGenClusterJson(SMonInfo *pMonitor) { } } +static void monGenClusterJsonBasic(SMonInfo *pMonitor) { + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + //tjsonAddStringToObject(pMonitor->pJson, "first_ep", pInfo->first_ep); + tjsonAddStringToObject(pMonitor->pJson, "first_ep", tsFirst); + tjsonAddDoubleToObject(pMonitor->pJson, "first_ep_dnode_id", pInfo->first_ep_dnode_id); + tjsonAddStringToObject(pMonitor->pJson, "cluster_version", pInfo->version); + //tjsonAddDoubleToObject(pMonitor->pJson, "monitor_interval", pInfo->monitor_interval); +} + static void monGenVgroupJson(SMonInfo *pMonitor) { SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -517,21 +553,11 @@ static void monGenLogJson(SMonInfo *pMonitor) { if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } -void monSendReport() { - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - monGenBasicJson(pMonitor); - monGenClusterJson(pMonitor); - monGenVgroupJson(pMonitor); - monGenStbJson(pMonitor); - monGenGrantJson(pMonitor); - monGenDnodeJson(pMonitor); - monGenDiskJson(pMonitor); - monGenLogJson(pMonitor); - +void monSendReport(SMonInfo *pMonitor){ char *pCont = tjsonToString(pMonitor->pJson); - // uDebugL("report cont:%s\n", pCont); + if(tsMonitorLogProtocol){ + uInfoL("report cont:\n%s", pCont); + } if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { @@ -539,6 +565,83 @@ void monSendReport() { } taosMemoryFree(pCont); } +} + +void monGenAndSendReport() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + if(!tsMonitorForceV2){ + monGenBasicJson(pMonitor); + monGenClusterJson(pMonitor); + monGenVgroupJson(pMonitor); + monGenStbJson(pMonitor); + monGenGrantJson(pMonitor); + monGenDnodeJson(pMonitor); + monGenDiskJson(pMonitor); + monGenLogJson(pMonitor); + + monSendReport(pMonitor); + } + else{ + monGenClusterInfoTable(pMonitor); + monGenVgroupInfoTable(pMonitor); + monGenDnodeInfoTable(pMonitor); + monGenDnodeStatusInfoTable(pMonitor); + monGenDataDiskTable(pMonitor); + monGenLogDiskTable(pMonitor); + monGenMnodeRoleTable(pMonitor); + monGenVnodeRoleTable(pMonitor); + + monSendPromReport(); + } monCleanupMonitorInfo(pMonitor); } + +void monSendReportBasic(SMonInfo *pMonitor){ + char *pCont = tjsonToString(pMonitor->pJson); + if(tsMonitorLogProtocol){ + if(pCont != NULL){ + uInfoL("report cont basic:\n%s", pCont); + } + else{ + uInfo("report cont basic is null"); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwBasicUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + taosMemoryFree(pCont); + } +} + +void monGenAndSendReportBasic() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + + monGenBasicJsonBasic(pMonitor); + monGenClusterJsonBasic(pMonitor); + + if (pMonitor->mmInfo.cluster.first_ep_dnode_id != 0) { + monSendReportBasic(pMonitor); + } + + monCleanupMonitorInfo(pMonitor); +} + +void monSendContent(char *pCont) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + if(tsMonitorLogProtocol){ + if (pCont != NULL){ + uInfoL("report client cont:\n%s\n", pCont); + } + } + if (pCont != NULL) { + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonFwUri, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg"); + } + } +} \ No newline at end of file diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 3f7b1b51da..2660cff216 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -283,10 +283,10 @@ TEST_F(MonitorTest, 01_Full) { tFreeSMonSmInfo(&smInfo); tFreeSMonQmInfo(&qmInfo); tFreeSMonBmInfo(&bmInfo); - monSendReport(); + monGenAndSendReport(); } TEST_F(MonitorTest, 02_Log) { AddLogInfo2(); - monSendReport(); + monGenAndSendReport(); } diff --git a/source/libs/monitorfw/CMakeLists.txt b/source/libs/monitorfw/CMakeLists.txt new file mode 100644 index 0000000000..610cd63985 --- /dev/null +++ b/source/libs/monitorfw/CMakeLists.txt @@ -0,0 +1,9 @@ +aux_source_directory(src MONITOR2_SRC) +add_library(monitorfw STATIC ${MONITOR2_SRC}) +target_include_directories( + monitorfw + PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(monitorfw os util common transport) diff --git a/source/libs/monitorfw/inc/taos_assert.h b/source/libs/monitorfw/inc/taos_assert.h new file mode 100644 index 0000000000..d3226eed26 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_assert.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_ASSERT_H +#define TAOS_ASSERT_H + +#ifdef TAOS_ASSERT_ENABLE +#define TAOS_ASSERT(i) assert(i); +#else +#define TAOS_ASSERT(i) +#endif // TAOS_TEST + +#endif // TAOS_ASSERT_H diff --git a/source/libs/monitorfw/inc/taos_collector_registry_i.h b/source/libs/monitorfw/inc/taos_collector_registry_i.h new file mode 100644 index 0000000000..ed5bb1fe19 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_i.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_collector_registry_t.h" + +#ifndef TAOS_COLLECTOR_REGISTRY_I_INCLUDED +#define TAOS_COLLECTOR_REGISTRY_I_INCLUDED + +int taos_collector_registry_enable_custom_process_metrics(taos_collector_registry_t *self, + const char *process_limits_path, + const char *process_stats_path); + +#endif // TAOS_COLLECTOR_REGISTRY_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_collector_registry_t.h b/source/libs/monitorfw/inc/taos_collector_registry_t.h new file mode 100644 index 0000000000..8e8a881fca --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_registry_t.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_REGISTRY_T_H +#define TAOS_REGISTRY_T_H + +#include +#include + +// Public +#include "taos_collector_registry.h" + +// Private +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector_registry { + const char *name; + bool disable_process_metrics; /**< Disables the collection of process metrics */ + taos_map_t *collectors; /**< Map of collectors keyed by name */ + taos_string_builder_t *string_builder; /**< Enables string building */ + taos_metric_formatter_t *metric_formatter; /**< metric formatter for metric exposition on bridge call */ + pthread_rwlock_t *lock; /**< mutex for safety against concurrent registration */ + taos_string_builder_t *string_builder_batch; +}; + +#endif // TAOS_REGISTRY_T_H diff --git a/source/libs/monitorfw/inc/taos_collector_t.h b/source/libs/monitorfw/inc/taos_collector_t.h new file mode 100644 index 0000000000..264e8e9ad9 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_collector_t.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_COLLECTOR_T_H +#define TAOS_COLLECTOR_T_H + +#include "taos_collector.h" +#include "taos_map_t.h" +#include "taos_string_builder_t.h" + +struct taos_collector { + const char *name; + taos_map_t *metrics; + taos_collect_fn *collect_fn; + taos_string_builder_t *string_builder; + const char *proc_limits_file_path; + const char *proc_stat_file_path; +}; + +#endif // TAOS_COLLECTOR_T_H diff --git a/source/libs/monitorfw/inc/taos_errors.h b/source/libs/monitorfw/inc/taos_errors.h new file mode 100644 index 0000000000..ee2a894df3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_errors.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_STDIO_CLOSE_DIR_ERROR "failed to close dir" +#define TAOS_STDIO_OPEN_DIR_ERROR "failed to open dir" +#define TAOS_METRIC_INCORRECT_TYPE "incorrect metric type" +#define TAOS_METRIC_INVALID_LABEL_NAME "invalid label name" +#define TAOS_PTHREAD_RWLOCK_DESTROY_ERROR "failed to destroy the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_INIT_ERROR "failed to initialize the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_LOCK_ERROR "failed to lock the pthread_rwlock_t*" +#define TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR "failed to unlock the pthread_rwlock_t*" +#define TAOS_REGEX_REGCOMP_ERROR "failed to compile the regular expression" +#define TAOS_REGEX_REGEXEC_ERROR "failed to execute the regular expression" diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h new file mode 100644 index 0000000000..ed11a76427 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_I_INCLUDED +#define TAOS_LIST_I_INCLUDED + +// Private +#include "taos_linked_list_t.h" + +/** + * @brief API PRIVATE Returns a pointer to a taos_linked_list + */ +taos_linked_list_t *taos_linked_list_new(void); + +/** + * @brief API PRIVATE removes all nodes from the given taos_linked_list * + */ +int taos_linked_list_purge(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Destroys a taos_linked_list + */ +int taos_linked_list_destroy(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Append an item to the back of the list + */ +int taos_linked_list_append(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Push an item onto the front of the list + */ +int taos_linked_list_push(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Pop the first item off of the list + */ +void *taos_linked_list_pop(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the head of the list or NULL if not present + */ +void *taos_linked_list_first(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Returns the item at the tail of the list or NULL if not present + */ +void *taos_linked_list_last(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Removes an item from the linked list + */ +int taos_linked_list_remove(taos_linked_list_t *self, void *item); + +/** + * @brief API PRIVATE Compares two items within a linked list + */ +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *node_b); + +/** + * @brief API PRIVATE Get the size + */ +size_t taos_linked_list_size(taos_linked_list_t *self); + +/** + * @brief API PRIVATE Set the free_fn member on taos_linked_list + */ +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn); + +/** + * @brief API PRIVATE Set the compare_fn member on the taos_linked_list + */ +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn); + +/** + * API PRIVATE + * @brief does nothing + */ +void taos_linked_list_no_op_free(void *item); + +#endif // TAOS_LIST_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_linked_list_t.h b/source/libs/monitorfw/inc/taos_linked_list_t.h new file mode 100644 index 0000000000..ccd82ffb61 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_linked_list_t.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_LIST_T_H +#define TAOS_LIST_T_H + +#include "taos_linked_list.h" + +typedef enum { TAOS_LESS = -1, TAOS_EQUAL = 0, TAOS_GREATER = 1 } taos_linked_list_compare_t; + +/** + * @brief API PRIVATE Frees an item in a taos_linked_list_node + */ +typedef void (*taos_linked_list_free_item_fn)(void *); + +/** + * @brief API PRIVATE Compares two items within a taos_linked_list + */ +typedef taos_linked_list_compare_t (*taos_linked_list_compare_item_fn)(void *item_a, void *item_b); + +/** + * @brief API PRIVATE A struct containing a generic item, represented as a void pointer, and next, a pointer to the + * next taos_linked_list_node* + */ +typedef struct taos_linked_list_node { + struct taos_linked_list_node *next; + void *item; +} taos_linked_list_node_t; + +/** + * @brief API PRIVATE A linked list comprised of taos_linked_list_node* instances + */ +struct taos_linked_list { + taos_linked_list_node_t *head; + taos_linked_list_node_t *tail; + size_t size; + taos_linked_list_free_item_fn free_fn; + taos_linked_list_compare_item_fn compare_fn; +}; + +#endif // TAOS_LIST_T_H diff --git a/source/libs/monitorfw/inc/taos_log.h b/source/libs/monitorfw/inc/taos_log.h new file mode 100644 index 0000000000..fecbda59e5 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_log.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#ifndef TAOS_LOG_H +#define TAOS_LOG_H + +//#define TAOS_LOG_ENABLE + +#ifdef TAOS_LOG_ENABLE +#define TAOS_LOG(msg) printf("monitor_log %s %s %s %s %d %s\n", __DATE__, __TIME__, __FILE__, __FUNCTION__, __LINE__, msg); +#else +#define TAOS_LOG(msg) +#endif // TAOS_LOG_ENABLE + +#endif // TAOS_LOG_H diff --git a/source/libs/monitorfw/inc/taos_map_i.h b/source/libs/monitorfw/inc/taos_map_i.h new file mode 100644 index 0000000000..55f248b96a --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_i.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_I_INCLUDED +#define TAOS_MAP_I_INCLUDED + +#include "taos_map_t.h" + +taos_map_t *taos_map_new(void); + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn); + +void *taos_map_get(taos_map_t *self, const char *key); + +void *taos_map_get_withoutlock(taos_map_t *self, const char *key); + +int taos_map_set(taos_map_t *self, const char *key, void *value); + +int taos_map_delete(taos_map_t *self, const char *key); + +int taos_map_destroy(taos_map_t *self); + +size_t taos_map_size(taos_map_t *self); + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn); + +#endif // TAOS_MAP_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_map_t.h b/source/libs/monitorfw/inc/taos_map_t.h new file mode 100644 index 0000000000..6fcbda1366 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_map_t.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MAP_T_H +#define TAOS_MAP_T_H + +#include + +// Public +#include "taos_map.h" + +// Private +#include "taos_linked_list_t.h" + +typedef void (*taos_map_node_free_value_fn)(void *); + +struct taos_map_node { + const char *key; + void *value; + taos_map_node_free_value_fn free_value_fn; +}; + +struct taos_map { + size_t size; /**< contains the size of the map */ + size_t max_size; /**< stores the current max_size */ + taos_linked_list_t *keys; /**< linked list containing containing all keys present */ + taos_linked_list_t **addrs; /**< Sequence of linked lists. Each list contains nodes with the same index */ + pthread_rwlock_t *rwlock; + taos_map_node_free_value_fn free_value_fn; +}; + +#endif // TAOS_MAP_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h new file mode 100644 index 0000000000..f3e4ebae75 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_custom_i.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_CUSTOMV2_I_H +#define TAOS_METRIC_FORMATTER_CUSTOMV2_I_H + +#define ALLOW_FORBID_FUNC + +#include +#include "tjson.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups); +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray); +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray); +#endif // TAOS_METRIC_FORMATTER_CUSTOMV2_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_i.h b/source/libs/monitorfw/inc/taos_metric_formatter_i.h new file mode 100644 index 0000000000..fba30d9eeb --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_i.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_I_H +#define TAOS_METRIC_FORMATTER_I_H + +// Private +#include "taos_metric_formatter_t.h" +#include "taos_metric_t.h" + +/** + * @brief API PRIVATE taos_metric_formatter constructor + */ +taos_metric_formatter_t *taos_metric_formatter_new(); + +/** + * @brief API PRIVATE taos_metric_formatter destructor + */ +int taos_metric_formatter_destroy(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Loads the help text + */ +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help); + +/** + * @brief API PRIVATE Loads the type text + */ +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample L-value + * @param name The metric name + * @param suffix The metric suffix. This is applicable to Summary and Histogram metric types. + * @param label_count The number of labels for the given metric. + * @param label_keys An array of constant strings. + * @param label_values An array of constant strings. + * + * The number of const char **and taos_label_value must be the same. + */ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *metric_formatter, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values); + +/** + * @brief API PRIVATE Loads the formatter with a metric sample + */ +int taos_metric_formatter_load_sample(taos_metric_formatter_t *metric_formatter, taos_metric_sample_t *sample, + char *ts, char *format); + +/** + * @brief API PRIVATE Loads a metric in the string exposition format + */ +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format); + +/** + * @brief API PRIVATE Loads the given metrics + */ +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format); + +/** + * @brief API PRIVATE Clear the underlying string_builder + */ +int taos_metric_formatter_clear(taos_metric_formatter_t *self); + +/** + * @brief API PRIVATE Returns the string built by taos_metric_formatter + */ +char *taos_metric_formatter_dump(taos_metric_formatter_t *metric_formatter); + +#endif // TAOS_METRIC_FORMATTER_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_formatter_t.h b/source/libs/monitorfw/inc/taos_metric_formatter_t.h new file mode 100644 index 0000000000..0d7425aa59 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_formatter_t.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_FORMATTER_T_H +#define TAOS_METRIC_FORMATTER_T_H + +#include "taos_string_builder_t.h" + +typedef struct taos_metric_formatter { + taos_string_builder_t *string_builder; + taos_string_builder_t *err_builder; +} taos_metric_formatter_t; + +#endif // TAOS_METRIC_FORMATTER_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_i.h b/source/libs/monitorfw/inc/taos_metric_i.h new file mode 100644 index 0000000000..e8ae799547 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_i.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Private +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_I_INCLUDED +#define TAOS_METRIC_I_INCLUDED + +/** + * @brief API PRIVATE Returns a *taos_metric + */ +taos_metric_t *taos_metric_new(taos_metric_type_t type, const char *name, const char *help, size_t label_key_count, + const char **label_keys); + +/** + * @brief API PRIVATE Destroys a *taos_metric + */ +int taos_metric_destroy(taos_metric_t *self); + +/** + * @brief API PRIVATE takes a generic item, casts to a *taos_metric_t and destroys it + */ +int taos_metric_destroy_generic(void *item); + +/** + * @brief API Private takes a generic item, casts to a *taos_metric_t and destroys it. Discards any errors. + */ +void taos_metric_free_generic(void *item); + +#endif // TAOS_METRIC_I_INCLUDED diff --git a/source/libs/monitorfw/inc/taos_metric_sample_i.h b/source/libs/monitorfw/inc/taos_metric_sample_i.h new file mode 100644 index 0000000000..b5e90f1933 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_i.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +#ifndef TAOS_METRIC_SAMPLE_I_H +#define TAOS_METRIC_SAMPLE_I_H + +/** + * @brief API PRIVATE Return a taos_metric_sample_t* + * + * @param type The type of metric sample + * @param l_value The entire left value of the metric e.g metric_name{foo="bar"} + * @param r_value A double representing the value of the sample + */ +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value); + +/** + * @brief API PRIVATE Destroy the taos_metric_sample** + */ +int taos_metric_sample_destroy(taos_metric_sample_t *self); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor + */ +int taos_metric_sample_destroy_generic(void *); + +/** + * @brief API PRIVATE A taos_linked_list_free_item_fn to enable item destruction within a linked list's destructor. + * + * This function ignores any errors. + */ +void taos_metric_sample_free_generic(void *gen); + +#endif // TAOS_METRIC_SAMPLE_I_H diff --git a/source/libs/monitorfw/inc/taos_metric_sample_t.h b/source/libs/monitorfw/inc/taos_metric_sample_t.h new file mode 100644 index 0000000000..387512af9d --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_sample_t.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_SAMPLE_T_H +#define TAOS_METRIC_SAMPLE_T_H + +#include "taos_metric_sample.h" +#include "taos_metric_t.h" + +#if !defined(WINDOWS) +#define C11_ATOMIC +#else +#define DOUBLE_ATOMIC +#endif + +#ifdef C11_ATOMIC +#include +#endif + +struct taos_metric_sample { + taos_metric_type_t type; /**< type is the metric type for the sample */ + char *l_value; /**< l_value is the full metric name and label set represeted as a string */ +#ifdef C11_ATOMIC + _Atomic double r_value; /**< r_value is the value of the metric sample */ +#else +#ifdef DOUBLE_ATOMIC + double r_value; /**< r_value is the value of the metric sample */ +#else + int64_t r_value; +#endif +#endif +}; + +#endif // TAOS_METRIC_SAMPLE_T_H diff --git a/source/libs/monitorfw/inc/taos_metric_t.h b/source/libs/monitorfw/inc/taos_metric_t.h new file mode 100644 index 0000000000..da237aa814 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_metric_t.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_METRIC_T_H +#define TAOS_METRIC_T_H + +#include + +// Public +#include "taos_metric.h" + +// Private +#include "taos_map_i.h" +#include "taos_map_t.h" +#include "taos_metric_formatter_t.h" + +/** + * @brief API PRIVATE Contains metric type constants + */ +typedef enum taos_metric_type { TAOS_COUNTER, TAOS_GAUGE, TAOS_HISTOGRAM, TAOS_SUMMARY } taos_metric_type_t; + +/** + * @brief API PRIVATE Maps metric type constants to human readable string values + */ +extern char *taos_metric_type_map[4]; + +/** + * @brief API PRIVATE An opaque struct to users containing metric metadata; one or more metric samples; and a metric + * formatter for locating metric samples and exporting metric data + */ +struct taos_metric { + taos_metric_type_t type; /**< metric_type The type of metric */ + char *name; /**< name The name of the metric */ + const char *help; /**< help The help output for the metric */ + taos_map_t *samples; /**< samples Map comprised of samples for the given metric */ + size_t label_key_count; /**< label_keys_count The count of labe_keys*/ + taos_metric_formatter_t *formatter; /**< formatter The metric formatter */ + pthread_rwlock_t *rwlock; /**< rwlock Required for locking on certain non-atomic operations */ + const char **label_keys; /**< labels Array comprised of const char **/ +}; + +#endif // TAOS_METRIC_T_H diff --git a/source/libs/monitorfw/inc/taos_monitor_util_i.h b/source/libs/monitorfw/inc/taos_monitor_util_i.h new file mode 100644 index 0000000000..fe072204a3 --- /dev/null +++ b/source/libs/monitorfw/inc/taos_monitor_util_i.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_MONITOR_UTIL_I_H +#define TAOS_MONITOR_UTIL_I_H + +#include + +void taos_monitor_split_str(char** arr, char* str, const char* del); +int taos_monitor_count_occurrences(char *str, char *toSearch); +void taos_monitor_strip(char *s); +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count); + +#endif // TAOS_MONITOR_UTIL_I_H \ No newline at end of file diff --git a/source/libs/monitorfw/inc/taos_string_builder_i.h b/source/libs/monitorfw/inc/taos_string_builder_i.h new file mode 100644 index 0000000000..142ca020ba --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_i.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_I_H +#define TAOS_STRING_BUILDER_I_H + +#include + +#include "taos_string_builder_t.h" + +/** + * API PRIVATE + * @brief Constructor for taos_string_builder + */ +taos_string_builder_t *taos_string_builder_new(void); + +/** + * API PRIVATE + * @brief Destroys a taos_string_builder* + */ +int taos_string_builder_destroy(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Adds a string + */ +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str); + +/** + * API PRIVATE + * @brief Adds a char + */ +int taos_string_builder_add_char(taos_string_builder_t *self, char c); + +/** + * API PRIVATE + * @brief Clear the string + */ +int taos_string_builder_clear(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Remove data from the end + */ +int taos_string_buillder_truncate(taos_string_builder_t *self, size_t len); + +/** + * API PRIVATE + * @brief Returns the length of the string + */ +size_t taos_string_builder_len(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Returns a copy of the string. The returned string must be deallocated when no longer needed. + */ +char *taos_string_builder_dump(taos_string_builder_t *self); + +/** + * API PRIVATE + * @brief Getter for str member + */ +char *taos_string_builder_str(taos_string_builder_t *self); + +#endif // TAOS_STRING_BUILDER_I_H diff --git a/source/libs/monitorfw/inc/taos_string_builder_t.h b/source/libs/monitorfw/inc/taos_string_builder_t.h new file mode 100644 index 0000000000..edd3d574fa --- /dev/null +++ b/source/libs/monitorfw/inc/taos_string_builder_t.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TAOS_STRING_BUILDER_T_H +#define TAOS_STRING_BUILDER_T_H + +struct taos_string_builder; +/** + * @brief API PRIVATE A structure with functions responsible for building a string + */ +typedef struct taos_string_builder taos_string_builder_t; + +#endif // TAOS_STRING_BUILDER_T_H diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c new file mode 100644 index 0000000000..17d324462c --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_i.h" +#include "taos_string_builder_i.h" + +taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self->metrics; } + +taos_collector_t *taos_collector_new(const char *name) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + self->name = taos_strdup(name); + self->metrics = taos_map_new(); + if (self->metrics == NULL) { + taos_collector_destroy(self); + return NULL; + } + r = taos_map_set_free_value_fn(self->metrics, &taos_metric_free_generic); + if (r) { + taos_collector_destroy(self); + return NULL; + } + self->collect_fn = &taos_collector_default_collect; + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_collector_destroy(self); + return NULL; + } + self->proc_limits_file_path = NULL; + self->proc_stat_file_path = NULL; + return self; +} + +int taos_collector_destroy(taos_collector_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->metrics); + if (r) ret = r; + self->metrics = NULL; + + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + + taos_free((char *)self->name); + self->name = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_destroy_generic(void *gen) { + int r = 0; + taos_collector_t *self = (taos_collector_t *)gen; + r = taos_collector_destroy(self); + self = NULL; + return r; +} + +void taos_collector_free_generic(void *gen) { + taos_collector_t *self = (taos_collector_t *)gen; + taos_collector_destroy(self); +} + +int taos_collector_set_collect_fn(taos_collector_t *self, taos_collect_fn *fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->collect_fn = fn; + return 0; +} + +int taos_collector_add_metric(taos_collector_t *self, taos_metric_t *metric) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (taos_map_get(self->metrics, metric->name) != NULL) { + TAOS_LOG("metric already found in collector"); + return 1; + } + return taos_map_set(self->metrics, metric->name, metric); +} + +int taos_collector_remove_metric(taos_collector_t *self, const char* key){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + return taos_map_delete(self->metrics, key); +} + +taos_metric_t* taos_collector_get_metric(taos_collector_t *self, char *metric_name){ + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + return taos_map_get(self->metrics, metric_name); +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c new file mode 100644 index 0000000000..c3ed0112c5 --- /dev/null +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +// Public +#include "taos_alloc.h" +#include "taos_collector.h" +#include "taos_collector_registry.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_registry_t.h" +#include "taos_collector_t.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" +#include "taos_metric_formatter_custom_i.h" + +#define ALLOW_FORBID_FUNC +#include "tjson.h" + +taos_collector_registry_t *TAOS_COLLECTOR_REGISTRY_DEFAULT; + +taos_collector_registry_t *taos_collector_registry_new(const char *name) { + int r = 0; + + taos_collector_registry_t *self = (taos_collector_registry_t *)taos_malloc(sizeof(taos_collector_registry_t)); + + self->disable_process_metrics = false; + + self->name = taos_strdup(name); + self->collectors = taos_map_new(); + taos_map_set_free_value_fn(self->collectors, &taos_collector_free_generic); + taos_map_set(self->collectors, "default", taos_collector_new("default")); + + self->metric_formatter = taos_metric_formatter_new(); + self->string_builder = taos_string_builder_new(); + self->string_builder_batch = taos_string_builder_new(); + self->lock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->lock, NULL); + if (r) { + TAOS_LOG("failed to initialize rwlock"); + return NULL; + } + return self; +} + +int taos_collector_registry_default_init(void) { + if (TAOS_COLLECTOR_REGISTRY_DEFAULT != NULL) return 0; + + TAOS_COLLECTOR_REGISTRY_DEFAULT = taos_collector_registry_new("default"); + //if (TAOS_COLLECTOR_REGISTRY_DEFAULT) { + // return taos_collector_registry_enable_process_metrics(TAOS_COLLECTOR_REGISTRY_DEFAULT); + //} + return 1; +} + +int taos_collector_registry_destroy(taos_collector_registry_t *self) { + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->collectors); + self->collectors = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->metric_formatter); + self->metric_formatter = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->string_builder_batch); + self->string_builder_batch = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->lock); + taos_free(self->lock); + self->lock = NULL; + if (r) ret = r; + + taos_free((char *)self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_collector_registry_register_metric(taos_metric_t *metric) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_add_metric(default_collector, metric); +} + +int taos_collector_registry_deregister_metric(const char *key) { + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return 1; + } + + return taos_collector_remove_metric(default_collector, key); +} + +taos_metric_t *taos_collector_registry_get_metric(char* metric_name){ + TAOS_ASSERT(metric != NULL); + + taos_collector_t *default_collector = + (taos_collector_t *)taos_map_get(TAOS_COLLECTOR_REGISTRY_DEFAULT->collectors, "default"); + + if (default_collector == NULL) { + return NULL; + } + + return taos_collector_get_metric(default_collector, metric_name); +} + +taos_metric_t *taos_collector_registry_must_register_metric(taos_metric_t *metric) { + int err = taos_collector_registry_register_metric(metric); + if (err != 0) { + //exit(err); + return NULL; + } + return metric; +} + +int taos_collector_registry_register_collector(taos_collector_registry_t *self, taos_collector_t *collector) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = pthread_rwlock_wrlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return 1; + } + if (taos_map_get(self->collectors, collector->name) != NULL) { + TAOS_LOG("the given taos_collector_t* is already registered"); + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return 1; + } + } + r = taos_map_set(self->collectors, collector->name, collector); + if (r) { + int rr = pthread_rwlock_unlock(self->lock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->lock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return 1; + } + return 0; +} + +int taos_collector_registry_validate_metric_name(taos_collector_registry_t *self, const char *metric_name) { + regex_t r; + int ret = 0; + ret = regcomp(&r, "^[a-zA-Z_:][a-zA-Z0-9_:]*$", REG_EXTENDED); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGCOMP_ERROR); + regfree(&r); + return ret; + } + + ret = regexec(&r, metric_name, 0, NULL, 0); + if (ret) { + TAOS_LOG(TAOS_REGEX_REGEXEC_ERROR); + regfree(&r); + return ret; + } + regfree(&r); + return 0; +} +/* +const char *taos_collector_registry_bridge(taos_collector_registry_t *self, char *ts, char *format) { + taos_metric_formatter_clear(self->metric_formatter); + taos_metric_formatter_load_metrics(self->metric_formatter, self->collectors, ts, format); + char *out = taos_metric_formatter_dump(self->metric_formatter); + + int r = 0; + r = taos_string_builder_add_str(self->string_builder_batch, out); + if (r) return NULL; + taos_free(out); + + return taos_string_builder_str(self->string_builder_batch); +} +*/ +int taos_collector_registry_clear_batch(taos_collector_registry_t *self){ + return taos_string_builder_clear(self->string_builder_batch); +} + +const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, char *ts, char *format, char** prom_str) { + taos_metric_formatter_clear(self->metric_formatter); + + SJson* pJson = tjsonCreateArray(); + SJson* item = tjsonCreateObject(); + tjsonAddItemToArray(pJson, item); + tjsonAddStringToObject(item, "ts", ts); + tjsonAddDoubleToObject(item, "protocol", 2); + SJson* array = tjsonCreateArray(); + tjsonAddItemToObject(item, "tables", array); + + if(taos_metric_formatter_load_metrics_new(self->metric_formatter, self->collectors, ts, format, array) != 0){ + TAOS_LOG("failed to load metrics"); + tjsonDelete(pJson); + return NULL; + } + + if(tjsonGetArraySize(array) == 0){ + tjsonDelete(pJson); + return NULL; + } + + //caller free this + //generate prom protocol for debug + if(prom_str != NULL){ + *prom_str = taos_metric_formatter_dump(self->metric_formatter); + } + + //add this result to batch cache, format in batch cache is {},{} + int r = 0; + char* old_str = taos_string_builder_str(self->string_builder_batch); + if(old_str[0] != '\0'){ + r = taos_string_builder_add_str(self->string_builder_batch, ","); + if (r) goto _OVER; + } + char * item_str = tjsonToString(item); + r = taos_string_builder_add_str(self->string_builder_batch, item_str); + taos_free(item_str); + if (r) goto _OVER;; + + //generate final array format result, ie, add [] to str in batch cache + taos_string_builder_t* tmp_builder = taos_string_builder_new(); + + r = taos_string_builder_add_str(tmp_builder, "["); + if (r) goto _OVER;; + + r = taos_string_builder_add_str(tmp_builder, taos_string_builder_str(self->string_builder_batch)); + if (r) goto _OVER;; + + r = taos_string_builder_add_str(tmp_builder, "]"); + if (r) goto _OVER;; + + //caller free this + char *data = taos_string_builder_dump(tmp_builder); + if (data == NULL) goto _OVER;; + r = taos_string_builder_clear(tmp_builder); + if (r) goto _OVER;; + + r = taos_string_builder_destroy(tmp_builder); + tmp_builder = NULL; + if (r) goto _OVER;; + + tjsonDelete(pJson); + return data; + +_OVER: + tjsonDelete(pJson); + + return NULL; +} diff --git a/source/libs/monitorfw/src/taos_counter.c b/source/libs/monitorfw/src/taos_counter.c new file mode 100644 index 0000000000..d522411b2b --- /dev/null +++ b/source/libs/monitorfw/src/taos_counter.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_counter.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_counter_t *taos_counter_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_counter_t *)taos_metric_new(TAOS_COUNTER, name, help, label_key_count, label_keys); +} + +int taos_counter_destroy(taos_counter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +int taos_counter_inc(taos_counter_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_counter_add(taos_counter_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_gauge.c b/source/libs/monitorfw/src/taos_gauge.c new file mode 100644 index 0000000000..7793f4c464 --- /dev/null +++ b/source/libs/monitorfw/src/taos_gauge.c @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_gauge.h" + +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" + +taos_gauge_t *taos_gauge_new(const char *name, const char *help, size_t label_key_count, const char **label_keys) { + return (taos_gauge_t *)taos_metric_new(TAOS_GAUGE, name, help, label_key_count, label_keys); +} + +int taos_gauge_destroy(taos_gauge_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = taos_metric_destroy(self); + self = NULL; + return r; +} +/* +int taos_gauge_inc(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, 1.0); +} + +int taos_gauge_dec(taos_gauge_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, 1.0); +} + +int taos_gauge_add(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_add(sample, r_value); +} + +int taos_gauge_sub(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_sub(sample, r_value); +} +*/ +int taos_gauge_set(taos_gauge_t *self, double r_value, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + taos_metric_sample_t *sample = taos_metric_sample_from_labels(self, label_values); + if (sample == NULL) return 1; + return taos_metric_sample_set(sample, r_value); +} diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c new file mode 100644 index 0000000000..675400a6fa --- /dev/null +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" + +taos_linked_list_t *taos_linked_list_new(void) { + taos_linked_list_t *self = (taos_linked_list_t *)taos_malloc(sizeof(taos_linked_list_t)); + self->head = NULL; + self->tail = NULL; + self->free_fn = NULL; + self->compare_fn = NULL; + self->size = 0; + return self; +} + +int taos_linked_list_purge(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = self->head; + while (node != NULL) { + taos_linked_list_node_t *next = node->next; + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + taos_free(node); + node = NULL; + node = next; + } + self->head = NULL; + self->tail = NULL; + self->size = 0; + return 0; +} + +int taos_linked_list_destroy(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_purge(self); + if (r) ret = r; + taos_free(self); + self = NULL; + return ret; +} + +void *taos_linked_list_first(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->head) { + return self->head->item; + } else { + return NULL; + } +} + +void *taos_linked_list_last(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self->tail) { + return self->tail->item; + } else { + return NULL; + } +} + +int taos_linked_list_append(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + if (self->tail) { + self->tail->next = node; + } else { + self->head = node; + } + self->tail = node; + node->next = NULL; + self->size++; + return 0; +} + +int taos_linked_list_push(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node = (taos_linked_list_node_t *)taos_malloc(sizeof(taos_linked_list_node_t)); + + node->item = item; + node->next = self->head; + self->head = node; + if (self->tail == NULL) { + self->tail = node; + } + self->size++; + return 0; +} + +void *taos_linked_list_pop(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return NULL; + taos_linked_list_node_t *node = self->head; + void *item = NULL; + if (node != NULL) { + item = node->item; + self->head = node->next; + if (self->tail == node) { + self->tail = NULL; + } + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + node->item = NULL; + node = NULL; + self->size--; + } + return item; +} + +int taos_linked_list_remove(taos_linked_list_t *self, void *item) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + taos_linked_list_node_t *node; + taos_linked_list_node_t *prev_node = NULL; +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; + char tmp[200] = {0}; + + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + + // Locate the node +#ifdef TAOS_LOG_ENABLE + count = 0; +#endif + for (node = self->head; node != NULL; node = node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif + if (self->compare_fn) { + if ((*self->compare_fn)(node->item, item) == TAOS_EQUAL) { + break; + } + } else { + if (node->item == item) { + break; + } + } + prev_node = node; + } + +#ifdef TAOS_LOG_ENABLE + sprintf(tmp, "remove item:%d", count); + TAOS_LOG(tmp); +#endif + + if (node == NULL) return 0; + + if (prev_node) { + prev_node->next = node->next; + } else { + self->head = node->next; + } + if (node->next == NULL) { + self->tail = prev_node; + } + + if (node->item != NULL) { + if (self->free_fn) { + (*self->free_fn)(node->item); + } else { + taos_free(node->item); + } + } + + node->item = NULL; + taos_free(node); + node = NULL; + self->size--; + +#ifdef TAOS_LOG_ENABLE + count = 0; + for (node = self->head; node != NULL; node = node->next) { + count++; + } + + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + + return 0; +} + +taos_linked_list_compare_t taos_linked_list_compare(taos_linked_list_t *self, void *item_a, void *item_b) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (self->compare_fn) { + return (*self->compare_fn)(item_a, item_b); + } else { + return strcmp(item_a, item_b); + } +} + +size_t taos_linked_list_size(taos_linked_list_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} + +int taos_linked_list_set_free_fn(taos_linked_list_t *self, taos_linked_list_free_item_fn free_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->free_fn = free_fn; + return 0; +} + +int taos_linked_list_set_compare_fn(taos_linked_list_t *self, taos_linked_list_compare_item_fn compare_fn) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->compare_fn = compare_fn; + return 0; +} + +void taos_linked_list_no_op_free(void *item) {} diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c new file mode 100644 index 0000000000..8f0b83884e --- /dev/null +++ b/source/libs/monitorfw/src/taos_map.c @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_linked_list_i.h" +#include "taos_linked_list_t.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_map_t.h" + +#define TAOS_MAP_INITIAL_SIZE 32 + +static void destroy_map_node_value_no_op(void *value) {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map_node +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_node_t *taos_map_node_new(const char *key, void *value, taos_map_node_free_value_fn free_value_fn) { + taos_map_node_t *self = taos_malloc(sizeof(taos_map_node_t)); + self->key = taos_strdup(key); + self->value = value; + self->free_value_fn = free_value_fn; + return self; +} + +int taos_map_node_destroy(taos_map_node_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->key); + self->key = NULL; + if (self->value != NULL) (*self->free_value_fn)(self->value); + self->value = NULL; + taos_free(self); + self = NULL; + return 0; +} + +void taos_map_node_free(void *item) { + taos_map_node_t *map_node = (taos_map_node_t *)item; + taos_map_node_destroy(map_node); +} + +taos_linked_list_compare_t taos_map_node_compare(void *item_a, void *item_b) { + taos_map_node_t *map_node_a = (taos_map_node_t *)item_a; + taos_map_node_t *map_node_b = (taos_map_node_t *)item_b; + + return strcmp(map_node_a->key, map_node_b->key); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// taos_map +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +taos_map_t *taos_map_new() { + int r = 0; + + taos_map_t *self = (taos_map_t *)taos_malloc(sizeof(taos_map_t)); + self->size = 0; + self->max_size = TAOS_MAP_INITIAL_SIZE; + + self->keys = taos_linked_list_new(); + if (self->keys == NULL) return NULL; + + // These each key will be allocated once by taos_map_node_new and used here as well to save memory. With that said + // we will only have to deallocate each key once. That will happen on taos_map_node_destroy. + r = taos_linked_list_set_free_fn(self->keys, taos_linked_list_no_op_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + + self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->free_value_fn = destroy_map_node_value_no_op; + + for (int i = 0; i < self->max_size; i++) { + self->addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(self->addrs[i], taos_map_node_free); + if (r) { + taos_map_destroy(self); + return NULL; + } + r = taos_linked_list_set_compare_fn(self->addrs[i], taos_map_node_compare); + if (r) { + taos_map_destroy(self); + return NULL; + } + } + + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_map_destroy(self); + return NULL; + } + + return self; +} + +int taos_map_destroy(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + + r = taos_linked_list_destroy(self->keys); + if (r) ret = r; + self->keys = NULL; + + for (size_t i = 0; i < self->max_size; i++) { + r = taos_linked_list_destroy(self->addrs[i]); + if (r) ret = r; + self->addrs[i] = NULL; + } + taos_free(self->addrs); + self->addrs = NULL; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR) + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + taos_free(self); + self = NULL; + + return ret; +} + +static size_t taos_map_get_index_internal(const char *key, size_t *size, size_t *max_size) { + size_t index; + size_t a = 31415, b = 27183; + for (index = 0; *key != '\0'; key++, a = a * b % (*max_size - 1)) { + index = (a * index + *key) % *max_size; + } + return index; +} + +/** + * @brief API PRIVATE hash function that returns an array index from the given key and taos_map. + * + * The algorithm is based off of Horner's method. In a simpler version, you set the return value to 0. Next, for each + * character in the string, you add the integer value of the current character to the product of the prime number and + * the current return value, set the result to the return value, then finally return the return value. + * + * In this version of the algorithm, we attempt to achieve a probabily of key to index conversion collisions to + * 1/M (with M being the max_size of the map). This optimizes dispersion and consequently, evens out the performance + * for gets and sets for each item. Instead of using a fixed prime number, we generate a coefficient for each iteration + * through the loop. + * + * Reference: + * * Algorithms in C: Third Edition by Robert Sedgewick, p579 + */ +size_t taos_map_get_index(taos_map_t *self, const char *key) { + return taos_map_get_index_internal(key, &self->size, &self->max_size); +} + +static void *taos_map_get_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return current_map_node->value; + } + } + taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return NULL; +} + +void *taos_map_get(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + NULL; + } + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return NULL; + } + return payload; +} + +void *taos_map_get_withoutlock(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + void *payload = + taos_map_get_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + return payload; +} + +static int taos_map_set_internal(const char *key, void *value, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn, + bool destroy_current_value) { + taos_map_node_t *map_node = taos_map_node_new(key, value, free_value_fn); + if (map_node == NULL) return 1; + + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, map_node); + if (result == TAOS_EQUAL) { + if (destroy_current_value) { + free_value_fn(current_map_node->value); + current_map_node->value = NULL; + } + taos_free((char *)current_map_node->key); + current_map_node->key = NULL; + taos_free(current_map_node); + current_map_node = NULL; + current_node->item = map_node; + return 0; + } + } + taos_linked_list_append(list, map_node); + taos_linked_list_append(keys, (char *)map_node->key); + (*size)++; + return 0; +} + +int taos_map_ensure_space(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self->size <= self->max_size / 2) { + return 0; + } + + // Increase the max size + size_t new_max = self->max_size * 2; + size_t new_size = 0; + + // Create a new list of keys + taos_linked_list_t *new_keys = taos_linked_list_new(); + if (new_keys == NULL) return 1; + + r = taos_linked_list_set_free_fn(new_keys, taos_linked_list_no_op_free); + if (r) return r; + + // Create a new array of addrs + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + + // Initialize the new array + for (int i = 0; i < new_max; i++) { + new_addrs[i] = taos_linked_list_new(); + r = taos_linked_list_set_free_fn(new_addrs[i], taos_map_node_free); + if (r) return r; + r = taos_linked_list_set_compare_fn(new_addrs[i], taos_map_node_compare); + if (r) return r; + } + + // Iterate through each linked-list at each memory region in the map's backbone + for (int i = 0; i < self->max_size; i++) { + // Create a new map node for each node in the linked list and insert it into the new map. Afterwards, deallocate + // the old map node + taos_linked_list_t *list = self->addrs[i]; + taos_linked_list_node_t *current_node = list->head; + while (current_node != NULL) { + taos_map_node_t *map_node = (taos_map_node_t *)current_node->item; + r = taos_map_set_internal(map_node->key, map_node->value, &new_size, &new_max, new_keys, new_addrs, + self->free_value_fn, false); + if (r) return r; + + taos_linked_list_node_t *next = current_node->next; + taos_free(current_node); + current_node = NULL; + taos_free((void *)map_node->key); + map_node->key = NULL; + taos_free(map_node); + map_node = NULL; + current_node = next; + } + // We're done deallocating each map node in the linked list, so deallocate the linked-list object + taos_free(self->addrs[i]); + self->addrs[i] = NULL; + } + // Destroy the collection of keys in the map + taos_linked_list_destroy(self->keys); + self->keys = NULL; + + // Deallocate the backbone of the map + taos_free(self->addrs); + self->addrs = NULL; + + // Update the members of the current map + self->size = new_size; + self->max_size = new_max; + self->keys = new_keys; + self->addrs = new_addrs; + + return 0; +} + +int taos_map_set(taos_map_t *self, const char *key, void *value) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return r; + } + + r = taos_map_ensure_space(self); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = taos_map_set_internal(key, value, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn, + true); + if (r) { + int rr = 0; + rr = pthread_rwlock_unlock(self->rwlock); + if (rr) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + return rr; + } else { + return r; + } + } + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + } + return r; +} + +static int taos_map_delete_internal(const char *key, size_t *size, size_t *max_size, taos_linked_list_t *keys, + taos_linked_list_t **addrs, taos_map_node_free_value_fn free_value_fn) { + int r = 0; + size_t index = taos_map_get_index_internal(key, size, max_size); + taos_linked_list_t *list = addrs[index]; + taos_map_node_t *temp_map_node = taos_map_node_new(key, NULL, free_value_fn); + + for (taos_linked_list_node_t *current_node = list->head; current_node != NULL; current_node = current_node->next) { + taos_map_node_t *current_map_node = (taos_map_node_t *)current_node->item; + taos_linked_list_compare_t result = taos_linked_list_compare(list, current_map_node, temp_map_node); + if (result == TAOS_EQUAL) { + r = taos_linked_list_remove(keys, (char*)current_map_node->key); + if (r) return r; + + r = taos_linked_list_remove(list, current_node->item); + if (r) return r; + + (*size)--; + break; + } + } + r = taos_map_node_destroy(temp_map_node); + temp_map_node = NULL; + return r; +} + +int taos_map_delete(taos_map_t *self, const char *key) { + TAOS_ASSERT(self != NULL); + int r = 0; + int ret = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + ret = r; + } + r = taos_map_delete_internal(key, &self->size, &self->max_size, self->keys, self->addrs, self->free_value_fn); + if (r) ret = r; + r = pthread_rwlock_unlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); + ret = r; + } + return ret; +} + +int taos_map_set_free_value_fn(taos_map_t *self, taos_map_node_free_value_fn free_value_fn) { + TAOS_ASSERT(self != NULL); + self->free_value_fn = free_value_fn; + return 0; +} + +size_t taos_map_size(taos_map_t *self) { + TAOS_ASSERT(self != NULL); + return self->size; +} diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c new file mode 100644 index 0000000000..4e9af35f34 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_i.h" +#include "taos_metric_sample_i.h" + +char *taos_metric_type_map[4] = {"counter", "gauge", "histogram", "summary"}; + +taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, const char *help, + size_t label_key_count, const char **label_keys) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + self->type = metric_type; + int len = strlen(name) + 1; + self->name = taos_malloc(len); + memset(self->name, 0, len); + strcpy(self->name, name); + //self->name = name; + self->help = help; + + const char **k = (const char **)taos_malloc(sizeof(const char *) * label_key_count); + + for (int i = 0; i < label_key_count; i++) { + if (strcmp(label_keys[i], "le") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + if (strcmp(label_keys[i], "quantile") == 0) { + TAOS_LOG(TAOS_METRIC_INVALID_LABEL_NAME); + taos_metric_destroy(self); + return NULL; + } + k[i] = taos_strdup(label_keys[i]); + } + self->label_keys = k; + self->label_key_count = label_key_count; + self->samples = taos_map_new(); + + if (metric_type == TAOS_HISTOGRAM) { + + } else { + r = taos_map_set_free_value_fn(self->samples, &taos_metric_sample_free_generic); + if (r) { + taos_metric_destroy(self); + return NULL; + } + } + + self->formatter = taos_metric_formatter_new(); + if (self->formatter == NULL) { + taos_metric_destroy(self); + return NULL; + } + self->rwlock = (pthread_rwlock_t *)taos_malloc(sizeof(pthread_rwlock_t)); + r = pthread_rwlock_init(self->rwlock, NULL); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + return NULL; + } + return self; +} + +int taos_metric_destroy(taos_metric_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + + r = taos_metric_formatter_destroy(self->formatter); + self->formatter = NULL; + if (r) ret = r; + + r = pthread_rwlock_destroy(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_DESTROY_ERROR); + ret = r; + } + + taos_free(self->rwlock); + self->rwlock = NULL; + + for (int i = 0; i < self->label_key_count; i++) { + taos_free((void *)self->label_keys[i]); + self->label_keys[i] = NULL; + } + taos_free(self->label_keys); + self->label_keys = NULL; + + taos_free(self->name); + self->name = NULL; + + taos_free(self); + self = NULL; + + return ret; +} + +int taos_metric_destroy_generic(void *item) { + int r = 0; + taos_metric_t *self = (taos_metric_t *)item; + r = taos_metric_destroy(self); + self = NULL; + return r; +} + +void taos_metric_free_generic(void *item) { + taos_metric_t *self = (taos_metric_t *)item; + taos_metric_destroy(self); +} + +taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const char **label_values) { + TAOS_ASSERT(self != NULL); + int r = 0; + r = pthread_rwlock_wrlock(self->rwlock); + if (r) { + TAOS_LOG(TAOS_PTHREAD_RWLOCK_LOCK_ERROR); + return NULL; + } + +#define TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK() \ + r = pthread_rwlock_unlock(self->rwlock); \ + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); \ + return NULL; + + // Get l_value + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, + label_values); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // This must be freed before returning + const char *l_value = taos_metric_formatter_dump(self->formatter); + if (l_value == NULL) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + + // Get sample + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(self->samples, l_value); + if (sample == NULL) { + sample = taos_metric_sample_new(self->type, l_value, 0.0); + r = taos_map_set(self->samples, l_value, sample); + if (r) { + TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); + } + } + pthread_rwlock_unlock(self->rwlock); + taos_free((void *)l_value); + return sample; +} + diff --git a/source/libs/monitorfw/src/taos_metric_formatter.c b/source/libs/monitorfw/src/taos_metric_formatter.c new file mode 100644 index 0000000000..53012935ba --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_collector_t.h" +#include "taos_linked_list_t.h" +#include "taos_map_i.h" +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "taos_metric_t.h" +#include "taos_string_builder_i.h" + + +taos_metric_formatter_t *taos_metric_formatter_new() { + taos_metric_formatter_t *self = (taos_metric_formatter_t *)taos_malloc(sizeof(taos_metric_formatter_t)); + self->string_builder = taos_string_builder_new(); + if (self->string_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + self->err_builder = taos_string_builder_new(); + if (self->err_builder == NULL) { + taos_metric_formatter_destroy(self); + return NULL; + } + return self; +} + +int taos_metric_formatter_destroy(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + + int r = 0; + int ret = 0; + + r = taos_string_builder_destroy(self->string_builder); + self->string_builder = NULL; + if (r) ret = r; + + r = taos_string_builder_destroy(self->err_builder); + self->err_builder = NULL; + if (r) ret = r; + + taos_free(self); + self = NULL; + return ret; +} +/* +int taos_metric_formatter_load_help(taos_metric_formatter_t *self, const char *name, const char *help) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# HELP "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, help); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_type(taos_metric_formatter_t *self, const char *name, taos_metric_type_t metric_type) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, "# TYPE "); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, taos_metric_type_map[metric_type]); + if (r) return r; + + return taos_string_builder_add_char(self->string_builder, '\n'); +} +*/ +int taos_metric_formatter_load_l_value(taos_metric_formatter_t *self, const char *name, const char *suffix, + size_t label_count, const char **label_keys, const char **label_values) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, name); + if (r) return r; + + if (suffix != NULL) { + r = taos_string_builder_add_char(self->string_builder, '_'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, suffix); + if (r) return r; + } + + if (label_count == 0) return 0; + + for (int i = 0; i < label_count; i++) { + if (i == 0) { + r = taos_string_builder_add_char(self->string_builder, '{'); + if (r) return r; + } + r = taos_string_builder_add_str(self->string_builder, (const char *)label_keys[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '='); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, (const char *)label_values[i]); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, '"'); + if (r) return r; + + if (i == label_count - 1) { + r = taos_string_builder_add_char(self->string_builder, '}'); + if (r) return r; + } else { + r = taos_string_builder_add_char(self->string_builder, ','); + if (r) return r; + } + } + return 0; +} +/* +int taos_metric_formatter_load_sample(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_string_builder_add_str(self->string_builder, sample->l_value); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + char buffer[50]; + sprintf(buffer, format, sample->r_value); + r = taos_string_builder_add_str(self->string_builder, buffer); + if (r) return r; + + r = taos_string_builder_add_char(self->string_builder, ' '); + if (r) return r; + + r = taos_string_builder_add_str(self->string_builder, ts); + if (r) return r; + + //taos_metric_sample_set(sample, 0); + + return taos_string_builder_add_char(self->string_builder, '\n'); +} +*/ +int taos_metric_formatter_clear(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + return taos_string_builder_clear(self->string_builder); +} + +char *taos_metric_formatter_dump(taos_metric_formatter_t *self) { + TAOS_ASSERT(self != NULL); + int r = 0; + if (self == NULL) return NULL; + char *data = taos_string_builder_dump(self->string_builder); + if (data == NULL) return NULL; + r = taos_string_builder_clear(self->string_builder); + if (r) { + taos_free(data); + return NULL; + } + return data; +} +/* +int taos_metric_formatter_load_metric(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + r = taos_metric_formatter_load_help(self, metric->name, metric->help); + if (r) return r; + + r = taos_metric_formatter_load_type(self, metric->name, metric->type); + if (r) return r; + + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample(self, sample, ts, format); + if (r) return r; + } + } + return taos_string_builder_add_char(self->string_builder, '\n'); +} + +int taos_metric_formatter_load_metrics(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, char *format) { + TAOS_ASSERT(self != NULL); + int r = 0; + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts, format); + if (r) return r; + } + } + return r; +} +*/ \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_formatter_custom.c b/source/libs/monitorfw/src/taos_metric_formatter_custom.c new file mode 100644 index 0000000000..3b1318dfc0 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_formatter_custom.c @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include +#include "taos_metric_formatter_i.h" +#include "taos_metric_sample_t.h" +#include "tjson.h" +#include "taos_monitor_util_i.h" +#include "taos_assert.h" +#include "tdef.h" +#include "taos_collector_t.h" +#include "taos_log.h" + +int taos_metric_formatter_load_sample_new(taos_metric_formatter_t *self, taos_metric_sample_t *sample, + char *ts, char *format, char *metricName, int32_t metric_type, + SJson *arrayMetricGroups) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + char* start = strstr(sample->l_value, "{"); + char* end = strstr(sample->l_value, "}"); + + int32_t len = end -start; + + char* keyvalues = taosMemoryMalloc(len); + memset(keyvalues, 0, len); + memcpy(keyvalues, start + 1, len - 1); + + int32_t count = taos_monitor_count_occurrences(keyvalues, ","); + + char** keyvalue = taosMemoryMalloc(sizeof(char*) * (count + 1)); + memset(keyvalue, 0, sizeof(char*) * (count + 1)); + taos_monitor_split_str(keyvalue, keyvalues, ","); + + char** arr = taosMemoryMalloc(sizeof(char*) * (count + 1) * 2); + memset(arr, 0, sizeof(char*) * (count + 1) * 2); + + bool isfound = true; + for(int32_t i = 0; i < count + 1; i++){ + char* str = *(keyvalue + i); + + char** pair = arr + i * 2; + taos_monitor_split_str(pair, str, "="); + + taos_monitor_strip(pair[1]); + } + + int32_t table_size = tjsonGetArraySize(arrayMetricGroups); + + SJson* item = NULL; + for(int32_t i = 0; i < table_size; i++){ + SJson *cur = tjsonGetArrayItem(arrayMetricGroups, i); + + SJson* tag = tjsonGetObjectItem(cur, "tags"); + + if(taos_monitor_is_match(tag, arr, count + 1)) { + item = cur; + break; + } + } + + SJson* metrics = NULL; + if(item == NULL) { + item = tjsonCreateObject(); + + SJson* arrayTag = tjsonCreateArray(); + for(int32_t i = 0; i < count + 1; i++){ + char** pair = arr + i * 2; + + char* key = *pair; + char* value = *(pair + 1); + + SJson* tag = tjsonCreateObject(); + tjsonAddStringToObject(tag, "name", key); + tjsonAddStringToObject(tag, "value", value); + + tjsonAddItemToArray(arrayTag, tag); + } + tjsonAddItemToObject(item, "tags", arrayTag); + + metrics = tjsonCreateArray(); + tjsonAddItemToObject(item, "metrics", metrics); + + tjsonAddItemToArray(arrayMetricGroups, item); + } + else{ + metrics = tjsonGetObjectItem(item, "metrics"); + } + + taosMemoryFreeClear(arr); + taosMemoryFreeClear(keyvalue); + taosMemoryFreeClear(keyvalues); + + SJson* metric = tjsonCreateObject(); + tjsonAddStringToObject(metric, "name", metricName); + + double old_value = 0; +#define USE_EXCHANGE +#ifdef USE_EXCHANGE + taos_metric_sample_exchange(sample, 0, &old_value); +#else + old_value = sample->r_value; + taos_metric_sample_set(sample, 0); +#endif + + tjsonAddDoubleToObject(metric, "value", old_value); + tjsonAddDoubleToObject(metric, "type", metric_type); + tjsonAddItemToArray(metrics, metric); + + return 0; +} + +int taos_metric_formatter_load_metric_new(taos_metric_formatter_t *self, taos_metric_t *metric, char *ts, char *format, + SJson* tableArray) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + + int r = 0; + + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + char* arr[2] = {0}; //arr[0] is table name, arr[1] is metric name + taos_monitor_split_str((char**)&arr, name, ":"); + + bool isFound = false; + SJson* table = NULL; + SJson* arrayMetricGroups = NULL; + + int32_t table_count = tjsonGetArraySize(tableArray); + for(int32_t i = 0; i < table_count; i++){ + SJson* table = tjsonGetArrayItem(tableArray, i); + + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); + if(strcmp(tableName, arr[0]) == 0){ + isFound = true; + arrayMetricGroups = tjsonGetObjectItem(table, "metric_groups"); + break; + } + } + + if(!isFound){ + table = tjsonCreateObject(); + + tjsonAddStringToObject(table, "name", arr[0]); + + arrayMetricGroups = tjsonCreateArray(); + tjsonAddItemToObject(table, "metric_groups", arrayMetricGroups); + } + + int32_t sample_count = 0; + for (taos_linked_list_node_t *current_node = metric->samples->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *key = (const char *)current_node->item; + if (metric->type == TAOS_HISTOGRAM) { + + } else { + taos_metric_sample_t *sample = (taos_metric_sample_t *)taos_map_get(metric->samples, key); + if (sample == NULL) return 1; + r = taos_metric_formatter_load_sample_new(self, sample, ts, format, arr[1], metric->type, arrayMetricGroups); + if (r) return r; + } + sample_count++; + } + + if(!isFound && sample_count > 0){ + tjsonAddItemToArray(tableArray, table); + } + else{ + if(table != NULL) tjsonDelete(table); + } + + taosMemoryFreeClear(name); + return r; +} + +int taos_metric_formatter_load_metrics_new(taos_metric_formatter_t *self, taos_map_t *collectors, char *ts, + char *format, SJson* tableArray) { + TAOS_ASSERT(self != NULL); + int r = 0; + + for (taos_linked_list_node_t *current_node = collectors->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *collector_name = (const char *)current_node->item; + taos_collector_t *collector = (taos_collector_t *)taos_map_get(collectors, collector_name); + if (collector == NULL) return 1; + + taos_map_t *metrics = collector->collect_fn(collector); + if (metrics == NULL) return 1; + + //if(strcmp(collector->name, "custom") != 0 ){ + + r = pthread_rwlock_wrlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to lock"); + return r; + } + +#ifdef TAOS_LOG_ENABLE + int32_t count = 0; +#endif + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { +#ifdef TAOS_LOG_ENABLE + count++; +#endif + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get_withoutlock(metrics, metric_name); + if (metric == NULL) { +#ifdef TAOS_LOG_ENABLE + char tmp[200] = {0}; + sprintf(tmp, "fail to get metric(%d):%s", count, metric_name); + TAOS_LOG(tmp); +#endif + continue;; + } + r = taos_metric_formatter_load_metric_new(self, metric, ts, format, tableArray); + if (r) { + TAOS_LOG("failed to load metric"); + continue; + } + } + +#ifdef TAOS_LOG_ENABLE + char tmp[20] = {0}; + sprintf(tmp, "list count:%d", count); + TAOS_LOG(tmp); +#endif + r = pthread_rwlock_unlock(metrics->rwlock); + if (r) { + TAOS_LOG("failed to unlock"); + return r; + } + //} + //else{ + /* + for (taos_linked_list_node_t *current_node = metrics->keys->head; current_node != NULL; + current_node = current_node->next) { + const char *metric_name = (const char *)current_node->item; + taos_metric_t *metric = (taos_metric_t *)taos_map_get(metrics, metric_name); + if (metric == NULL) return 1; + r = taos_metric_formatter_load_metric(self, metric, ts, format); + if (r) return r; + } + */ + //} + } + return r; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_metric_sample.c b/source/libs/monitorfw/src/taos_metric_sample.c new file mode 100644 index 0000000000..c6d817b513 --- /dev/null +++ b/source/libs/monitorfw/src/taos_metric_sample.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_errors.h" +#include "taos_log.h" +#include "taos_metric_sample_i.h" +#include "taos_metric_sample_t.h" + +#ifdef C11_ATOMIC +#include +#else +#define ALLOW_FORBID_FUNC +#include "tdef.h" +#include "osAtomic.h" +#endif + +taos_metric_sample_t *taos_metric_sample_new(taos_metric_type_t type, const char *l_value, double r_value) { + taos_metric_sample_t *self = (taos_metric_sample_t *)taos_malloc(sizeof(taos_metric_sample_t)); + self->type = type; + self->l_value = taos_strdup(l_value); + self->r_value = 0; + return self; +} + +int taos_metric_sample_destroy(taos_metric_sample_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free((void *)self->l_value); + self->l_value = NULL; + taos_free((void *)self); + self = NULL; + return 0; +} + +int taos_metric_sample_destroy_generic(void *gen) { + int r = 0; + + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + r = taos_metric_sample_destroy(self); + self = NULL; + return r; +} + +void taos_metric_sample_free_generic(void *gen) { + taos_metric_sample_t *self = (taos_metric_sample_t *)gen; + taos_metric_sample_destroy(self); +} + +int taos_metric_sample_add(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (r_value < 0) { + return 1; + } + +#ifdef C11_ATOMIC + /*_Atomic*/ double old = atomic_load(&self->r_value); + + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old + r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + atomic_fetch_add_double(&self->r_value, r_value); +#else + atomic_fetch_add_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} + +/* +int taos_metric_sample_sub(taos_metric_sample_t *self, double r_value) { + TAOS_ASSERT(self != NULL); + if (self->type != TAOS_GAUGE) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + ///_Atomic/ + double old = atomic_load(&self->r_value); + for (;;) { + _Atomic double new = ATOMIC_VAR_INIT(old - r_value); + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + atomic_fetch_sub_double(&self->r_value, r_value); +#else + atomic_fetch_sub_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} +*/ + +int taos_metric_sample_set(taos_metric_sample_t *self, double r_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + atomic_store(&self->r_value, r_value); +#else +#ifdef DOUBLE_ATOMIC + atomic_store_double(&self->r_value, r_value); +#else + atomic_store_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} + +int taos_metric_sample_exchange(taos_metric_sample_t *self, double r_value, double* old_value) { + if (self->type != TAOS_GAUGE && self->type != TAOS_COUNTER) { + TAOS_LOG(TAOS_METRIC_INCORRECT_TYPE); + return 1; + } + +#ifdef C11_ATOMIC + _Atomic double new = ATOMIC_VAR_INIT(r_value); + for (;;) { + /*_Atomic*/ double old = atomic_load(&self->r_value); + *old_value = old; + if (atomic_compare_exchange_weak(&self->r_value, &old, new)) { + return 0; + } + } +#else +#ifdef DOUBLE_ATOMIC + *old_value = atomic_exchange_double(&self->r_value, r_value); +#else + *old_value = atomic_exchange_64(&self->r_value, r_value); +#endif +#endif + + return 0; +} \ No newline at end of file diff --git a/source/libs/monitorfw/src/taos_monitor_util.c b/source/libs/monitorfw/src/taos_monitor_util.c new file mode 100644 index 0000000000..182402b3ff --- /dev/null +++ b/source/libs/monitorfw/src/taos_monitor_util.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#define ALLOW_FORBID_FUNC +#include "tjson.h" +#include +#include +#include +#include "osMemory.h" +#include "tdef.h" + +#include "taos_metric_t.h" + +void taos_monitor_split_str(char** arr, char* str, const char* del) { + char *lasts; + char* s = strsep(&str, del); + while (s != NULL) { + *arr++ = s; + s = strsep(&str, del); + } +} + +void taos_monitor_split_str_metric(char** arr, taos_metric_t* metric, const char* del, char** buf) { + int32_t size = strlen(metric->name); + char* name = taosMemoryMalloc(size + 1); + memset(name, 0, size + 1); + memcpy(name, metric->name, size); + + char* s = strtok(name, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } + + *buf = name; +} + +const char* taos_monitor_get_metric_name(taos_metric_t* metric){ + return metric->name; +} + +int taos_monitor_count_occurrences(char *str, char *toSearch) { + int count = 0; + char *ptr = str; + while ((ptr = strstr(ptr, toSearch)) != NULL) { + count++; + ptr++; + } + return count; +} + +void taos_monitor_strip(char *s) +{ + size_t i; + size_t len = strlen(s); + size_t offset = 0; + for(i = 0; i < len; ++i){ + char c = s[i]; + if(c=='\"') ++offset; + else s[i-offset] = c; + } + s[len-offset] = '\0'; +} + +bool taos_monitor_is_match(const SJson* tags, char** pairs, int32_t count) { + int32_t size = tjsonGetArraySize(tags); + if(size != count) return false; + + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(tags, i); + + char item_name[MONITOR_TAG_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", item_name); + + char item_value[MONITOR_TAG_VALUE_LEN] = {0}; + tjsonGetStringValue(item, "value", item_value); + + bool isfound = false; + for(int32_t j = 0; j < count; j++){ + + char** pair = pairs + j * 2; + + char* key = *pair; + char* value = *(pair + 1); + + + if(strcmp(value, item_value) == 0 && strcmp(key, item_name) == 0){ + isfound = true; + break; + } + } + + if(!isfound) return false; + } + + return true; +} diff --git a/source/libs/monitorfw/src/taos_string_builder.c b/source/libs/monitorfw/src/taos_string_builder.c new file mode 100644 index 0000000000..0f3940cdab --- /dev/null +++ b/source/libs/monitorfw/src/taos_string_builder.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +// Public +#include "taos_alloc.h" + +// Private +#include "taos_assert.h" +#include "taos_string_builder_i.h" +#include "taos_string_builder_t.h" + +// The initial size of a string created via taos_string_builder +#define TAOS_STRING_BUILDER_INIT_SIZE 32 + +// taos_string_builder_init prototype declaration +int taos_string_builder_init(taos_string_builder_t *self); + +struct taos_string_builder { + char *str; /**< the target string */ + size_t allocated; /**< the size allocated to the string in bytes */ + size_t len; /**< the length of str */ + size_t init_size; /**< the initialize size of space to allocate */ +}; + +taos_string_builder_t *taos_string_builder_new(void) { + int r = 0; + + taos_string_builder_t *self = (taos_string_builder_t *)taos_malloc(sizeof(taos_string_builder_t)); + self->init_size = TAOS_STRING_BUILDER_INIT_SIZE; + r = taos_string_builder_init(self); + if (r) { + taos_string_builder_destroy(self); + return NULL; + } + + return self; +} + +int taos_string_builder_init(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + self->str = (char *)taos_malloc(self->init_size); + *self->str = '\0'; + self->allocated = self->init_size; + self->len = 0; + return 0; +} + +int taos_string_builder_destroy(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 0; + taos_free(self->str); + self->str = NULL; + taos_free(self); + self = NULL; + return 0; +} + +/** + * @brief API PRIVATE Grows the size of the string given the value we want to add + * + * The method continuously shifts left until the new size is large enough to accommodate add_len. This private method + * is called in methods that need to add one or more characters to the underlying string. + */ +static int taos_string_builder_ensure_space(taos_string_builder_t *self, size_t add_len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (add_len == 0 || self->allocated >= self->len + add_len + 1) return 0; + while (self->allocated < self->len + add_len + 1) self->allocated <<= 1; + self->str = (char *)taos_realloc(self->str, self->allocated); + return 0; +} + +int taos_string_builder_add_str(taos_string_builder_t *self, const char *str) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + if (str == NULL || *str == '\0') return 0; + + size_t len = strlen(str); + r = taos_string_builder_ensure_space(self, len); + if (r) return r; + + memcpy(self->str + self->len, str, len); + self->len += len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_add_char(taos_string_builder_t *self, char c) { + TAOS_ASSERT(self != NULL); + int r = 0; + + if (self == NULL) return 1; + r = taos_string_builder_ensure_space(self, 1); + if (r) return r; + + self->str[self->len] = c; + self->len++; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_truncate(taos_string_builder_t *self, size_t len) { + TAOS_ASSERT(self != NULL); + if (self == NULL) return 1; + if (len >= self->len) return 0; + + self->len = len; + self->str[self->len] = '\0'; + return 0; +} + +int taos_string_builder_clear(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + taos_free(self->str); + self->str = NULL; + return taos_string_builder_init(self); +} + +size_t taos_string_builder_len(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->len; +} + +char *taos_string_builder_dump(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + // +1 to accommodate \0 + char *out = (char *)taos_malloc((self->len + 1) * sizeof(char)); + memcpy(out, self->str, self->len + 1); + return out; +} + +char *taos_string_builder_str(taos_string_builder_t *self) { + TAOS_ASSERT(self != NULL); + return self->str; +} diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index c2cd4786db..4f523bf4f2 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -347,8 +347,11 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_SCORES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_GRANTS_FULL_STMT: case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index abe50a27da..cc522c9b5f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -373,6 +373,7 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); } +#ifdef BUILD_NO_CALL static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { #ifndef TD_ENTERPRISE return TSDB_CODE_PAR_TABLE_NOT_EXIST; @@ -396,6 +397,7 @@ static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCac } return code; } +#endif int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) { int32_t code = TSDB_CODE_SUCCESS; @@ -774,9 +776,11 @@ static bool isAggFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isSelectFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); @@ -790,9 +794,11 @@ static bool isInterpPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +#ifdef BUILD_NO_CALL static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } +#endif static bool isImplicitTsFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsImplicitTsFunc(((SFunctionNode*)pNode)->funcId)); @@ -1672,11 +1678,11 @@ static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pLeft))->userAlias); return DEAL_RES_ERROR; } if (isMultiResFunc(pOp->pRight)) { - generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, ((SExprNode*)(pOp->pRight))->userAlias); return DEAL_RES_ERROR; } @@ -7738,9 +7744,11 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl return code; } +#ifdef BUILD_NO_CALL static bool isEventWindowQuery(SSelectStmt* pSelect) { return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow); } +#endif static bool hasJsonTypeProjection(SSelectStmt* pSelect) { SNode* pProj = NULL; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..76d8022578 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT: + return "Operator not supported multi result: %s"; default: return "Unknown error"; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 390faab537..d7b31d2ac8 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -381,7 +381,7 @@ TEST_F(ParserSelectTest, semanticCheck) { // TSDB_CODE_PAR_WRONG_VALUE_TYPE run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); - run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE); + run("SELECT LAST(*) + SUM(c1) FROM t1", TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT); run("SELECT CEIL(LAST(ts, c1)) FROM t1", TSDB_CODE_FUNC_FUNTION_PARA_NUM); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c5e84898ed..37cdc31ae0 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -16,6 +16,7 @@ #include "planInt.h" #include "filter.h" #include "functionMgt.h" +#include "tglobal.h" typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1c90e61ea3..1aa21e73a3 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -57,7 +57,7 @@ typedef enum { #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 5000000 +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 30000000 #define SCH_MAX_TASK_TIMEOUT_USEC 300000000 #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 841066a4c9..795c21c234 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -33,7 +33,7 @@ int32_t schedulerInit() { schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables; schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; - schMgmt.cfg.enableReSchedule = true; + schMgmt.cfg.enableReSchedule = false; qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 03f70604b7..1dc1db8e9c 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -17,10 +17,14 @@ #define _STREAM_BACKEDN_ROCKSDB_H_ #include "rocksdb/c.h" -//#include "streamInt.h" +// #include "streamInt.h" #include "streamState.h" #include "tcommon.h" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct SCfComparator { rocksdb_comparator_t** comp; int32_t numOfComp; @@ -244,11 +248,6 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); int32_t taskDbBuildSnap(void* arg, SArray* pSnap); -// int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); - -// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); -// void taskDbDestroy(void* pDb, bool flush); - int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); SBkdMgt* bkdMgtCreate(char* path); @@ -258,4 +257,10 @@ int32_t bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname); void bkdMgtDestroy(SBkdMgt* bm); int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list); -#endif \ No newline at end of file + +uint32_t nextPow2(uint32_t x); +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f173157da6..910fd93989 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2788,7 +2788,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) { STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey); return pCur; } -#ifdef BUILD_NO_CALL SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) { stDebug("streamStateGetCur_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; @@ -2838,7 +2837,6 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) { STREAM_STATE_DEL_ROCKSDB(pState, "func", key); return 0; } -#endif // session cf int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { @@ -3432,7 +3430,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* SSessionKey tmpKey = *key; int32_t valSize = *pVLen; void* tmp = taosMemoryMalloc(valSize); - // tdbRealloc(NULL, valSize); if (!tmp) { return -1; } @@ -3506,13 +3503,11 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi return code; } -#ifdef BUILD_NO_CALL int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } -#endif int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); @@ -3535,10 +3530,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co if (pIter == NULL) { return -1; } - + size_t klen = 0; rocksdb_iter_seek(pIter, start, strlen(start)); while (rocksdb_iter_valid(pIter)) { - const char* key = rocksdb_iter_key(pIter, NULL); + const char* key = rocksdb_iter_key(pIter, &klen); int32_t vlen = 0; const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen); char* val = NULL; @@ -3700,6 +3695,8 @@ uint32_t nextPow2(uint32_t x) { x = x | (x >> 16); return x + 1; } + +#ifdef BUILD_NO_CALL int32_t copyFiles(const char* src, const char* dst) { int32_t code = 0; // opt later, just hard link @@ -3739,6 +3736,7 @@ _err: taosCloseDir(&pDir); return code >= 0 ? 0 : -1; } +#endif int32_t isBkdDataMeta(char* name, int32_t len) { const char* pCurrent = "CURRENT"; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b35f401cb9..a09b940a19 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -70,7 +70,7 @@ static void streamMetaEnvInit() { streamTimerInit(); } -void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);} +void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { taosCloseRef(streamBackendId); @@ -1104,14 +1104,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; - entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); + entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate; entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } if ((*pTask)->chkInfo.checkpointingId != 0) { - entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId)? 1:0; + entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0; entry.checkpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; @@ -1172,7 +1172,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } - _end: +_end: streamMetaClearHbMsg(&hbMsg); return TSDB_CODE_SUCCESS; } @@ -1304,28 +1304,28 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { } void streamMetaRLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-rlock", pMeta->vgId); + // stTrace("vgId:%d meta-rlock", pMeta->vgId); taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-runlock", pMeta->vgId); + // stTrace("vgId:%d meta-runlock", pMeta->vgId); int32_t code = taosThreadRwlockUnlock(&pMeta->lock); if (code != TSDB_CODE_SUCCESS) { stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); } else { -// stTrace("vgId:%d meta-runlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-runlock completed", pMeta->vgId); } } void streamMetaWLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wlock", pMeta->vgId); + // stTrace("vgId:%d meta-wlock", pMeta->vgId); taosThreadRwlockWrlock(&pMeta->lock); -// stTrace("vgId:%d meta-wlock completed", pMeta->vgId); + // stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { -// stTrace("vgId:%d meta-wunlock", pMeta->vgId); + // stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosThreadRwlockUnlock(&pMeta->lock); } @@ -1395,7 +1395,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) pMeta->sendMsgBeforeClosing = true; } - pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; streamMetaWUnLock(pMeta); if (isLeader) { @@ -1531,8 +1531,8 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { int32_t num = taosArrayGetSize(pMeta->pTaskList); - for(int32_t i = 0; i < num; ++i) { - STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + for (int32_t i = 0; i < num; ++i) { + STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId)); if (ppTask == NULL) { continue; @@ -1633,7 +1633,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 - ", readyTs:%" PRId64 " total elapsed time:%.2fs", + ", readyTs:%" PRId64 " total elapsed time:%.2fs", pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 454ed4297c..764bf6e026 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -22,7 +22,7 @@ #define DEFAULT_MAP_CAPACITY 131072 #define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 100000 +#define MAX_NUM_SCALABLE_BF 64 #define MIN_NUM_SCALABLE_BF 10 #define DEFAULT_PREADD_BUCKET 1 #define MAX_INTERVAL MILLISECOND_PER_MINUTE @@ -81,7 +81,9 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) { static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t watermark) { if (watermark <= adjInterval) { watermark = TMAX(originInt / adjInterval, 1) * adjInterval; - } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { + } + + if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; } return watermark; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index c90e05bcf6..c472207b27 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -1,40 +1,104 @@ -MESSAGE(STATUS "build stream unit test") - -# GoogleTest requires at least C++11 -SET(CMAKE_CXX_STANDARD 11) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # bloomFilterTest -ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") -TARGET_LINK_LIBRARIES(streamUpdateTest - PUBLIC os util common gtest gtest_main stream executor index +#TARGET_LINK_LIBRARIES(streamUpdateTest + #PUBLIC os util common gtest gtest_main stream executor index + #) + +#TARGET_INCLUDE_DIRECTORIES( + #streamUpdateTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) +#TARGET_LINK_LIBRARIES( + #checkpointTest + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #checkpointTest + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_executable(backendTest "") + +#target_sources(backendTest + #PRIVATE + #"backendTest.cpp" +#) + +#TARGET_LINK_LIBRARIES( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + +#TARGET_INCLUDE_DIRECTORIES( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#add_test( + #NAME streamUpdateTest + #COMMAND streamUpdateTest +#) + +#add_test( + #NAME checkpointTest + #COMMAND checkpointTest +#) +#add_test( + #NAME backendTest + #COMMAND backendTest +#) + + +#add_executable(backendTest "") + +#target_sources(backendTest + #PUBLIC + #"backendTest.cpp" +#) + +#target_include_directories( + #backendTest + #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +#) + +#target_link_libraries( + #backendTest + #PUBLIC rocksdb + #PUBLIC os common gtest stream executor qcom index transport util +#) + + +MESSAGE(STATUS "build parser unit test") + +IF(NOT TD_DARWIN) + # GoogleTest requires at least C++11 + SET(CMAKE_CXX_STANDARD 11) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(backendTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES( + backendTest + PUBLIC rocksdb + PUBLIC os common gtest stream executor qcom index transport util vnode ) -TARGET_INCLUDE_DIRECTORIES( - streamUpdateTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) + TARGET_INCLUDE_DIRECTORIES( + backendTest + PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/" + PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" + ) -ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) -TARGET_LINK_LIBRARIES( - checkpointTest - PUBLIC os common gtest stream executor qcom index transport util -) - -TARGET_INCLUDE_DIRECTORIES( - checkpointTest - PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" -) - -add_test( - NAME streamUpdateTest - COMMAND streamUpdateTest -) - -add_test( - NAME checkpointTest - COMMAND checkpointTest -) \ No newline at end of file + ADD_TEST( + NAME backendTest + COMMAND backendTest + ) +ENDIF () \ No newline at end of file diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp new file mode 100644 index 0000000000..a949748eb5 --- /dev/null +++ b/source/libs/stream/test/backendTest.cpp @@ -0,0 +1,437 @@ +#include + +#include +#include +#include +#include +#include "streamBackendRocksdb.h" +#include "streamSnapshot.h" +#include "streamState.h" +#include "tstream.h" +#include "tstreamFileState.h" +#include "tstreamUpdate.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +class BackendEnv : public ::testing::Test { + protected: + virtual void SetUp() {} + virtual void TearDown() {} +}; + +void *backendCreate() { + const char *streamPath = "/tmp"; + void *p = NULL; + + // char *absPath = NULL; + // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2); + // STaskDbWrapper *p = taskDbOpen((char *)streamPath, (char *)"stream-backend", -1); + // ASSERT(p != NULL); + return p; +} + +SStreamState *stateCreate(const char *path) { + SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask)); + pTask->ver = 1024; + pTask->id.streamId = 1023; + pTask->id.taskId = 1111111; + SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL); + pTask->pMeta = pMeta; + + SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024); + ASSERT(p != NULL); + return p; +} +void *backendOpen() { + streamMetaInit(); + const char *path = "/tmp/backend"; + SStreamState *p = stateCreate(path); + ASSERT(p != NULL); + + // write bacth + // default/state/fill/sess/func/parname/partag + int32_t size = 100; + std::vector tsArray; + for (int32_t i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + SWinKey key; // = {.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + + tsArray.push_back(ts); + } + for (int32_t i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char *newVal = NULL; + streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(len == strlen(val)); + } + int64_t ts = tsArray[0]; + SWinKey key = {0}; // {.groupId = (uint64_t)(0), .ts = ts}; + key.groupId = (uint64_t)(0); + key.ts = ts; + + streamStateDel_rocksdb(p, &key); + + streamStateClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + int64_t ts = tsArray[i]; + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t len = 0; + char *newVal = NULL; + int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); + ASSERT(code != 0); + } + tsArray.clear(); + + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + + SWinKey key = {0}; //{.groupId = (uint64_t)(i), .ts = ts}; + key.groupId = (uint64_t)(i); + key.ts = ts; + + const char *val = "value data"; + int32_t vlen = strlen(val); + streamStatePut_rocksdb(p, &key, (char *)val, vlen); + } + + SWinKey winkey; + int32_t code = streamStateGetFirst_rocksdb(p, &key); + ASSERT(code == 0); + ASSERT(key.ts == tsArray[0]); + + SStreamStateCur *pCurr = streamStateSeekToLast_rocksdb(p); + ASSERT(pCurr != NULL); + streamStateFreeCur(pCurr); + + winkey.groupId = 0; + winkey.ts = tsArray[0]; + char *val = NULL; + int32_t len = 0; + + pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey); + ASSERT(pCurr != NULL); + + streamStateFreeCur(pCurr); + + tsArray.clear(); + for (int i = 0; i < size; i++) { + int64_t ts = taosGetTimestampMs(); + tsArray.push_back(ts); + STupleKey key = {0}; + key.groupId = (uint64_t)(0); //= {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i}; + key.ts = ts; + key.exprIdx = i; + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateFuncPut_rocksdb(p, &key, val, len); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char *val = NULL; + int32_t len = 0; + streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len); + ASSERT(len == strlen("Value")); + } + for (int i = 0; i < size; i++) { + STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[i]; + key.exprIdx = i; + + char *val = NULL; + int32_t len = 0; + streamStateFuncDel_rocksdb(p, &key); + } + + // session put + tsArray.clear(); + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)}; + key.win.skey = i; + key.win.ekey = i; + key.groupId = (uint64_t)(0); + tsArray.push_back(i); + + const char *val = "Value"; + int32_t len = strlen(val); + streamStateSessionPut_rocksdb(p, &key, val, len); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + } + + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len)); + ASSERT(strncmp(pval, val, len) == 0); + taosMemoryFreeClear(pval); + } + + pCurr = streamStateSessionSeekToLast_rocksdb(p, 0); + ASSERT(pCurr != NULL); + + { + SSessionKey key; + memset(&key, 0, sizeof(key)); + char *val = NULL; + int32_t vlen = 0; + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key); + + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + + ASSERT(key.groupId == 0 && key.win.ekey == tsArray[tsArray.size() - 2]); + + pCurr = streamStateSessionSeekKeyNext_rocksdb(p, &key); + code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); + ASSERT(code == 0); + ASSERT(vlen == strlen("Value")); + ASSERT(key.groupId == 0 && key.win.skey == tsArray[tsArray.size() - 1]); + + ASSERT(0 == streamStateSessionAddIfNotExist_rocksdb(p, &key, 10, (void **)&val, &len)); + + ASSERT(0 == + streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len)); + } + for (int i = 0; i < size; i++) { + SSessionKey key = {0}; //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)}; + key.win.skey = tsArray[i]; + key.win.ekey = tsArray[i]; + key.groupId = (uint64_t)(0); + + const char *val = "Value"; + int32_t len = strlen(val); + + char *pval = NULL; + ASSERT(0 == streamStateSessionDel_rocksdb(p, &key)); + } + + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + const char *val = "Value"; + int32_t vlen = strlen(val); + ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0); + } + for (int i = 0; i < size; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char *val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0); + taosMemoryFreeClear(val); + } + { + SWinKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[0]}; + key.groupId = (uint64_t)(0); + key.ts = tsArray[0]; + SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key); + ASSERT(pCurr != NULL); + + char *val = NULL; + int32_t vlen = 0; + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value")); + streamStateFreeCur(pCurr); + + pCurr = streamStateFillSeekKeyNext_rocksdb(p, &key); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + ASSERT(vlen == strlen("Value") && key.groupId == 1 && key.ts == tsArray[1]); + + key.groupId = 1; + key.ts = tsArray[1]; + + pCurr = streamStateFillSeekKeyPrev_rocksdb(p, &key); + ASSERT(pCurr != NULL); + ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); + + ASSERT(vlen == strlen("Value") && key.groupId == 0 && key.ts == tsArray[0]); + } + + for (int i = 0; i < size - 1; i++) { + SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; + key.groupId = (uint64_t)(i); + key.ts = tsArray[i]; + char *val = NULL; + int32_t vlen = 0; + ASSERT(streamStateFillDel_rocksdb(p, &key) == 0); + taosMemoryFreeClear(val); + } + streamStateSessionClear_rocksdb(p); + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + + for (int i = 0; i < size; i++) { + char tbname[TSDB_TABLE_NAME_LEN] = {0}; + sprintf(tbname, "%s_%d", "tbname", i); + ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname)); + } + for (int i = 0; i < size; i++) { + char *val = NULL; + ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val)); + ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0); + taosMemoryFree(val); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + code = streamDefaultPut_rocksdb(p, key, val, strlen(val)); + ASSERT(code == 0); + } + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "tbname_%d", i); + + char *val = NULL; + int32_t len = 0; + code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len); + ASSERT(code == 0); + } + SArray *result = taosArrayInit(8, sizeof(void *)); + streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result); + ASSERT(taosArrayGetSize(result) >= 0); + + return p; + // streamStateClose((SStreamState *)p, true); +} +TEST_F(BackendEnv, checkOpen) { + SStreamState *p = (SStreamState *)backendOpen(); + int64_t tsStart = taosGetTimestampMs(); + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + // do checkpoint 2 + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2); + { + void *pBatch = streamStateCreateBatch(); + int32_t size = 0; + char valBuf[256] = {0}; + for (int i = 0; i < size; i++) { + char key[128] = {0}; + sprintf(key, "key_%d", i); + char val[128] = {0}; + sprintf(val, "val_%d", i); + streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val, + (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf); + } + streamStatePutBatch_rocksdb(p, pBatch); + streamStateDestroyBatch(pBatch); + } + + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3); + + const char *path = "/tmp/backend/stream"; + const char *dump = "/tmp/backend/stream/dump"; + // taosMkDir(dump); + taosMulMkDir(dump); + SBkdMgt *mgt = bkdMgtCreate((char *)path); + SArray *result = taosArrayInit(4, sizeof(void *)); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump); + + bkdMgtDestroy(mgt); + streamStateClose((SStreamState *)p, true); + taosRemoveDir(path); +} + +TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; } + +typedef struct BdKV { + uint32_t k; + uint32_t v; +} BdKV; + +BdKV kvDict[] = {{0, 2}, {1, 2}, {15, 16}, {31, 32}, {56, 64}, {100, 128}, + {200, 256}, {500, 512}, {1000, 1024}, {2000, 2048}, {3000, 4096}}; + +TEST_F(BackendEnv, backendUtil) { + for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) { + ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v); + } +} + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 0dc2cc13f5..0caad479e5 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -25,46 +25,49 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +// tsSnodeAddress = ""; +// tsS3StreamEnabled = 0; + +#include "cos.h" #include "rsync.h" #include "streamInt.h" -#include "cos.h" -int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); +// int main(int argc, char **argv) { +// testing::InitGoogleTest(&argc, argv); - if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { - printf("error"); - } - if (s3Init() < 0) { - return -1; - } - strcpy(tsSnodeAddress, "127.0.0.1"); - int ret = RUN_ALL_TESTS(); - s3CleanUp(); - return ret; -} +// if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { +// printf("error"); +// } +// if (s3Init() < 0) { +// return -1; +// } +// strcpy(tsSnodeAddress, "127.0.0.1"); +// int ret = RUN_ALL_TESTS(); +// s3CleanUp(); +// return ret; +// } TEST(testCase, checkpointUpload_Test) { - stopRsync(); - startRsync(); + // stopRsync(); + // startRsync(); taosSsleep(5); char* id = "2013892036"; - uploadCheckpoint(id, "/root/offset/"); + // uploadCheckpoint(id, "/root/offset/"); } TEST(testCase, checkpointDownload_Test) { char* id = "2013892036"; - downloadCheckpoint(id, "/root/offset/download/"); + // downloadCheckpoint(id, "/root/offset/download/"); } TEST(testCase, checkpointDelete_Test) { char* id = "2013892036"; - deleteCheckpoint(id); + // deleteCheckpoint(id); } TEST(testCase, checkpointDeleteFile_Test) { char* id = "2013892036"; - deleteCheckpointFile(id, "offset-ver0"); + // deleteCheckpointFile(id, "offset-ver0"); } diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index 1b999e5fb0..59171876ff 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -14,10 +14,7 @@ class StreamStateEnv : public ::testing::Test { streamMetaInit(); backend = streamBackendInit(path, 0, 0); } - virtual void TearDown() { - streamMetaCleanup(); - // indexClose(index); - } + virtual void TearDown() { streamMetaCleanup(); } const char *path = TD_TMP_DIR_PATH "stream"; void *backend; @@ -50,6 +47,14 @@ bool equalSBF(SScalableBf *left, SScalableBf *right) { } TEST(TD_STREAM_UPDATE_TEST, update) { + const char *streamPath = "/tmp"; + + char *absPath = NULL; + void *p = NULL; + // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2); + // p = taskDbOpen((char *)streamPath, (char *)"test", -1); + p = bkdMgtCreate((char *)streamPath); + // const int64_t interval = 20 * 1000; // const int64_t watermark = 10 * 60 * 1000; // SUpdateInfo *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e06ea70f70..7ff6116137 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1343,7 +1343,7 @@ ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager, + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, (void*)pSyncNode->rid, syncEnv()->pTimerManager, &pSyncNode->pPingTimer); atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } else { @@ -1415,8 +1415,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) { static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; if (syncIsInit()) { - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager, - &pSyncNode->pHeartbeatTimer); + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, (void*)pSyncNode->rid, + syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer); atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } else { sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); @@ -2153,7 +2153,11 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex static void syncNodeEqPingTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (atomic_load_64(&pNode->pingTimerLogicClockUser) <= atomic_load_64(&pNode->pingTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_PING, atomic_load_64(&pNode->pingTimerLogicClock), @@ -2173,7 +2177,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, pNode, syncEnv()->pTimerManager, &pNode->pPingTimer); + taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, + &pNode->pPingTimer); } } @@ -2224,7 +2229,11 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; - SSyncNode* pNode = param; + int64_t rid = (int64_t)param; + SSyncNode* pNode = syncNodeAcquire(rid); + + if (pNode == NULL) return; + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; @@ -2245,7 +2254,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } _out: - taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, pNode, syncEnv()->pTimerManager, + taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager, &pNode->pHeartbeatTimer); } else { @@ -3385,4 +3394,4 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } -#endif \ No newline at end of file +#endif diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 3ee65f11dd..455128e6ec 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -316,7 +316,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 3. Try to Recycle a page - if (!pPage && !pCache->lru.pLruPrev->isAnchor) { + if (!pPageH && !pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; tdbPCacheRemovePageFromHash(pCache, pPage); tdbPCachePinPage(pCache, pPage); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 7483588593..c010e31320 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -256,21 +256,21 @@ void transAsyncPoolDestroy(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); bool transAsyncPoolIsEmpty(SAsyncPool* pool); -#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ - do { \ - for (int i = 0; i < pool->nAsync; i++) { \ - uv_async_t* async = &(pool->asyncs[i]); \ - SAsyncItem* item = async->data; \ - while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ - tTrace("destroy msg in async pool "); \ - queue* h = QUEUE_HEAD(&item->qmsg); \ - QUEUE_REMOVE(h); \ - msgType* msg = QUEUE_DATA(h, msgType, q); \ - if (msg != NULL) { \ - freeFunc(msg); \ - } \ - } \ - } \ +#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \ + do { \ + for (int i = 0; i < pool->nAsync; i++) { \ + uv_async_t* async = &(pool->asyncs[i]); \ + SAsyncItem* item = async->data; \ + while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ + tTrace("destroy msg in async pool "); \ + queue* h = QUEUE_HEAD(&item->qmsg); \ + QUEUE_REMOVE(h); \ + msgType* msg = QUEUE_DATA(h, msgType, q); \ + if (msg != NULL) { \ + freeFunc(msg, param); \ + } \ + } \ + } \ } while (0) #define ASYNC_CHECK_HANDLE(exh1, id) \ @@ -427,6 +427,8 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, void transDQCancel(SDelayQueue* queue, SDelayTask* task); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); + +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b); /* * init global func */ diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6de10cbb9e..c4ca39c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -191,6 +191,15 @@ static void httpDestroyMsg(SHttpMsg* msg) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } +static void httpDestroyMsgWrapper(void* cont, void* param) { + httpDestroyMsg((SHttpMsg*)cont); + // if (msg == NULL) return; + + // taosMemoryFree(msg->server); + // taosMemoryFree(msg->uri); + // taosMemoryFree(msg->cont); + // taosMemoryFree(msg); +} static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; @@ -554,7 +563,7 @@ void transHttpEnvDestroy() { httpSendQuit(); taosThreadJoin(load->thread, NULL); - TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL); transAsyncPoolDestroy(load->asyncPool); uv_loop_close(load->loop); taosMemoryFree(load->loop); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2370efa460..6ae72eac14 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, /// NULL,cliHandleUpdate}; static FORCE_INLINE void destroyCmsg(void* cmsg); + +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -1963,7 +1965,17 @@ static FORCE_INLINE void destroyCmsg(void* arg) { transFreeMsg(pMsg->msg.pCont); taosMemoryFree(pMsg); } - +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) { + SCliMsg* pMsg = arg; + if (pMsg == NULL) { + return; + } + if (param != NULL) { + SCliThrd* pThrd = param; + if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle); + } + destroyCmsg(pMsg); +} static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { if (param == NULL) return; @@ -2057,7 +2069,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); @@ -2224,7 +2236,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset1"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); @@ -2251,7 +2263,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { EPSET_FORWARD_INUSE(&pCtx->epSet); } } else { - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + if (!transEpSetIsEqual2(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset2"); transPrintEpSet(&pCtx->epSet); transPrintEpSet(&epSet); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 4619722743..27099c76fc 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -70,7 +70,7 @@ int32_t transDecompressMsg(char** msg, int32_t len) { char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); STransMsgHead* pNewHead = (STransMsgHead*)buf; int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, - len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); @@ -603,6 +603,17 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { } return true; } +bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b324ca5f91..5a1ef31b7d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -671,7 +671,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrd* pThrd) { +static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); } +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -1394,7 +1395,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } taosThreadJoin(pThrd->thread, NULL); SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index a54c301de9..c891ee4579 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,6 +16,12 @@ #define ALLOW_FORBID_FUNC #include "os.h" +typedef union { + volatile int64_t i; + volatile double d; + //double d; +} double_number; + #ifdef WINDOWS // add @@ -339,6 +345,21 @@ void atomic_store_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_store_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void atomic_store_ptr(void* ptr, void* val) { #ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); @@ -393,6 +414,23 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_exchange_double(double volatile *ptr, int64_t val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + int64_t iNew = val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) { + return ret_num.d; + } + } +} + void* atomic_exchange_ptr(void* ptr, void* val) { #ifdef WINDOWS #ifdef _WIN64 @@ -551,6 +589,21 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_add_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d + val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_add_ptr(void* ptr, void* val) { #ifdef WINDOWS return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); @@ -657,6 +710,21 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #endif } +double atomic_fetch_sub_double(double volatile *ptr, double val){ + for (;;) { + double_number old_num = {0}; + old_num.d = *ptr; // current old value + + double_number new_num = {0}; + new_num.d = old_num.d - val; + + double_number ret_num = {0}; + ret_num.i = atomic_val_compare_exchange_64((volatile int64_t *)ptr, old_num.i, new_num.i); + + if (ret_num.i == old_num.i) return ret_num.d; + } +} + void* atomic_fetch_sub_ptr(void* ptr, void* val) { #ifdef WINDOWS return interlocked_fetch_sub_ptr(ptr, val); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index fea7a4f63d..187461826a 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -563,9 +563,9 @@ int32_t taosGetCpuCores(float *numOfCores, bool physical) { } void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { - static int64_t lastSysUsed = 0; - static int64_t lastSysTotal = 0; - static int64_t lastProcTotal = 0; + static int64_t lastSysUsed = -1; + static int64_t lastSysTotal = -1; + static int64_t lastProcTotal = -1; static int64_t curSysUsed = 0; static int64_t curSysTotal = 0; static int64_t curProcTotal = 0; @@ -580,12 +580,14 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { curSysTotal = curSysUsed + sysCpu.idle; curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { - if (cpu_system != NULL) { - *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; - } - if (cpu_engine != NULL) { - *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + if(lastSysUsed >= 0 && lastSysTotal >=0 && lastProcTotal >=0){ + if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { + if (cpu_system != NULL) { + *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; + } + if (cpu_engine != NULL) { + *cpu_engine = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + } } } @@ -821,19 +823,27 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int } void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - static int64_t last_rchars = 0; - static int64_t last_wchars = 0; - static int64_t last_read_bytes = 0; - static int64_t last_write_bytes = 0; + static int64_t last_rchars = -1; + static int64_t last_wchars = -1; + static int64_t last_read_bytes = -1; + static int64_t last_write_bytes = -1; static int64_t cur_rchars = 0; static int64_t cur_wchars = 0; static int64_t cur_read_bytes = 0; static int64_t cur_write_bytes = 0; if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) { - *rchars = cur_rchars - last_rchars; - *wchars = cur_wchars - last_wchars; - *read_bytes = cur_read_bytes - last_read_bytes; - *write_bytes = cur_write_bytes - last_write_bytes; + if(last_rchars >=0 && last_wchars >=0 && last_read_bytes >=0 && last_write_bytes >= 0){ + *rchars = cur_rchars - last_rchars; + *wchars = cur_wchars - last_wchars; + *read_bytes = cur_read_bytes - last_read_bytes; + *write_bytes = cur_write_bytes - last_write_bytes; + } + else{ + *rchars = 0; + *wchars = 0; + *read_bytes = 0; + *write_bytes = 0; + } last_rchars = cur_rchars; last_wchars = cur_wchars; last_read_bytes = cur_read_bytes; @@ -900,13 +910,20 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { - static int64_t last_receive_bytes = 0; - static int64_t last_transmit_bytes = 0; + static int64_t last_receive_bytes = -1; + static int64_t last_transmit_bytes = -1; int64_t cur_receive_bytes = 0; int64_t cur_transmit_bytes = 0; if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { - *receive_bytes = cur_receive_bytes - last_receive_bytes; - *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + if(last_receive_bytes >= 0 && last_transmit_bytes >= 0){ + *receive_bytes = cur_receive_bytes - last_receive_bytes; + *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + } + else{ + *receive_bytes = 0; + *transmit_bytes = 0; + } + last_receive_bytes = cur_receive_bytes; last_transmit_bytes = cur_transmit_bytes; } else { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8c73604727..1f3aaa3835 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST, "view not exists in db //mnode-compact TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") @@ -619,6 +620,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid stream quer TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_QUERY_MISMATCH, "Columns number mismatch with query result") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT, "Operator not supported multi result") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0b7ad330a7..82993e8449 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -375,4 +375,8 @@ bool tjsonValidateJson(const char* jIn) { return true; } -const char* tjsonGetError() { return cJSON_GetErrorPtr(); } \ No newline at end of file +const char* tjsonGetError() { return cJSON_GetErrorPtr(); } + +void tjsonDeleteItemFromObject(const SJson* pJson, const char* pName) { + cJSON_DeleteItemFromObject((cJSON*)pJson, pName); +} \ No newline at end of file diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 3b4975b701..7af794546b 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -20,6 +20,9 @@ #define DEFAULT_GROWTH 2 #define DEFAULT_TIGHTENING_RATIO 0.5 +#define DEFAULT_MAX_BLOOMFILTERS 4 +#define SBF_INVALID -1 +#define SBF_VALID 0 static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate); @@ -32,6 +35,8 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { if (pSBf == NULL) { return NULL; } + pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS; + pSBf->status = SBF_VALID; pSBf->numBits = 0; pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *)); if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) { @@ -45,6 +50,9 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) { } int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } int32_t size = taosArrayGetSize(pSBf->bfArray); SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1); ASSERT(pNormalBf); @@ -52,6 +60,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -59,6 +68,9 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le } int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -74,6 +86,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth, pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO); if (pNormalBf == NULL) { + pSBf->status = SBF_INVALID; return TSDB_CODE_OUT_OF_MEMORY; } } @@ -81,6 +94,9 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) { } int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) { + if (pSBf->status == SBF_INVALID) { + return TSDB_CODE_FAILED; + } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); @@ -93,6 +109,10 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32 } static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) { + if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) { + return NULL; + } + SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate); if (pNormalBf == NULL) { return NULL; @@ -128,6 +148,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) { } if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1; if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1; + if (tEncodeU32(pEncoder, pSBf->maxBloomFilters) < 0) return -1; + if (tEncodeI8(pEncoder, pSBf->status) < 0) return -1; return 0; } @@ -150,6 +172,8 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) { } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error; if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error; + if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error; + if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error; return pSBf; _error: diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 6bcf547136..dfd8d95c9c 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -46,7 +46,9 @@ class TDTestCase(TBase): # clusterDnodes.starttaosd(1) # time.sleep(5) autoGen.insert_data(5000, True) - tdSql.execute(f"flush database {self.db}") + self.flushDb(True) + # wait flush operation over + time.sleep(5) # sql = 'show vnodes;' # while True: @@ -67,7 +69,6 @@ class TDTestCase(TBase): dirs = glob.glob(dnodesRootDir) for dir in dirs: if os.path.isdir(dir): - tdLog.debug("delete dir: %s " % (dnodesRootDir)) self.remove_directory(os.path.join(dir, "wal")) sc.dnodeStart(1) @@ -88,7 +89,7 @@ class TDTestCase(TBase): if bFinish: break - self.timestamp_step = 1 + self.timestamp_step = 1000 self.insert_rows = 6000 self.checkInsertCorrect() self.checkAggCorrect() diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py new file mode 100644 index 0000000000..8b6844820e --- /dev/null +++ b/tests/army/community/query/show.py @@ -0,0 +1,152 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + + +class TDTestCase(TBase): + updatecfgDict = { + } + + def insertData(self): + tdLog.info(f"create table and insert data.") + self.stb = "stb" + self.db = "db" + self.childtable_count = 10 + self.insert_rows = 10000 + + self.autoGen = AutoGen(startTs = 1600000000000*1000*1000, batch=500, fillOne=True) + self.autoGen.create_db(self.db, 2, 3, "precision 'ns'") + self.autoGen.create_stable(stbname = self.stb, tag_cnt = 5, column_cnt = 20, binary_len = 10, nchar_len = 5) + self.autoGen.create_child(self.stb, "child", self.childtable_count) + self.autoGen.insert_data(self.insert_rows, True) + + tdLog.info("create view.") + tdSql.execute(f"use {self.db}") + sqls = [ + "create view viewc0c1 as select c0,c1 from stb ", + "create view viewc0c1c2 as select c0,c1,c2 from stb ", + "create view viewc0c3 as select c0,c3 from stb where c3=1", + "create view viewc0c4c5 as select c4,c5 from stb ", + "create view viewc0c6 as select c0,c1,c6 from stb ", + "create view viewc0c7 as select c0,c1 from stb ", + "create view viewc0c7c8 as select c0,c7,c8 from stb where c8>0", + "create view viewc0c3c1 as select c0,c3,c1 from stb ", + "create view viewc2c4 as select c2,c4 from stb ", + "create view viewc2c5 as select c2,c5 from stb ", + ] + tdSql.executes(sqls) + + def checkView(self): + tdLog.info(f"check view like.") + + # like + sql = f"show views like 'view%'" + tdSql.query(sql) + tdSql.checkRows(10) + + sql = f"show views like 'vie_c0c1c2'" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,0,"viewc0c1c2") + + sql = f"show views like '%c2c_'" + tdSql.query(sql) + tdSql.checkRows(2) + tdSql.checkData(0,0, "viewc2c4") + tdSql.checkData(1,0, "viewc2c5") + + sql = f"show views like '%' " + tdSql.query(sql) + tdSql.checkRows(10) + + # zero + sql = "show views like '_' " + tdSql.query(sql) + tdSql.checkRows(0) + sql = "show views like 'a%' " + tdSql.query(sql) + tdSql.checkRows(0) + + + def doQuery(self): + tdLog.info(f"do query.") + + # __group_key + sql = f"select count(*) from {self.stb} " + tdSql.query(sql) + # column index 1 value same with 2 + allRows = self.insert_rows * self.childtable_count + tdSql.checkFirstValue(sql, allRows) + + def checkShow(self): + # not support + sql = "show accounts;" + tdSql.error(sql) + + # check result + sql = "SHOW CLUSTER;" + tdSql.query(sql) + tdSql.checkRows(1) + sql = "SHOW COMPACTS;" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "SHOW COMPACT 1;" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "SHOW CLUSTER MACHINES;" + tdSql.query(sql) + tdSql.checkRows(1) + + # run to check crash + sqls = [ + "show scores;", + "SHOW CLUSTER VARIABLES", + "SHOW BNODES;", + ] + tdSql.executes(sqls) + + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check view + self.checkView() + + # do action + self.doQuery() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index d7544a897c..747ac7c8ec 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "connection_pool_size": 8, - "num_of_records_per_req": 2000, + "num_of_records_per_req": 4000, "prepared_rand": 1000, "thread_count": 2, "create_table_thread_count": 1, @@ -18,26 +18,27 @@ "drop": "yes", "vgroups": 2, "replica": 1, - "duration":"1d", - "keep": "3d,6d,30d" + "duration":"15d", + "flush_each_batch":"yes", + "keep": "60d,100d,200d" }, "super_tables": [ { "name": "stb", "child_table_exists": "no", - "childtable_count": 4, - "insert_rows": 1000000, + "childtable_count": 2, + "insert_rows": 2000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-13d", + "start_timestamp":"now-90d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, { "type": "double", "name": "dc"}, - { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, - { "type": "int", "name": "ic" }, + { "type": "int", "name": "ic" ,"max": 1,"min": 1}, { "type": "bigint", "name": "bi" }, { "type": "utinyint", "name": "uti"}, { "type": "usmallint", "name": "usi"}, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 45519d925f..e9173dda00 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -58,8 +58,8 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # come from s3_basic.json - self.childtable_count = 4 - self.insert_rows = 1000000 + self.childtable_count = 2 + self.insert_rows = 2000000 self.timestamp_step = 1000 def createStream(self, sname): diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index d1f02e7865..cf21977c75 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -14,15 +14,18 @@ import time # Auto Gen class # class AutoGen: - def __init__(self, fillOne=False): - self.ts = 1600000000000 - self.batch_size = 100 + def __init__(self, startTs = 1600000000000, step = 1000, batch = 100, fillOne=False): + self.startTs = startTs + self.ts = startTs + self.step = step + self.batch_size = batch + self.fillOne = fillOne seed = time.time() % 10000 random.seed(seed) - self.fillOne = fillOne # set start ts def set_start_ts(self, ts): + self.startTs = ts self.ts = ts # set batch size @@ -111,9 +114,9 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): + def create_db(self, dbname, vgroups = 2, replica = 1, others=""): self.dbname = dbname - tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') + tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica} {others}') # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): @@ -167,12 +170,12 @@ class AutoGen: def insert_data(self, cnt, bContinue=False): if not bContinue: - self.ts = 1600000000000 + self.ts = self.startTs - currTs = 1600000000000 + currTs = self.startTs for i in range(self.child_cnt): name = f"{self.child_name}{i}" - currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, self.step) self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 12f1f62f63..c32790022a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -15,10 +15,11 @@ ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py +,,n,army,python3 ./test.py -f community/query/show.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 # @@ -237,7 +238,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 @@ -1084,6 +1086,7 @@ ,,y,script,./test.sh -f tsim/parser/join_multivnode.sim ,,y,script,./test.sh -f tsim/parser/join.sim ,,y,script,./test.sh -f tsim/parser/last_cache.sim +,,y,script,./test.sh -f tsim/parser/last_both.sim ,,y,script,./test.sh -f tsim/parser/last_groupby.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim ,,y,script,./test.sh -f tsim/parser/lastrow2.sim @@ -1210,6 +1213,7 @@ ,,y,script,./test.sh -f tsim/stream/deleteState.sim ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,y,script,./test.sh -f tsim/stream/distributeMultiLevelInterval0.sim ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim ,,y,script,./test.sh -f tsim/stream/drop_stream.sim ,,y,script,./test.sh -f tsim/stream/event0.sim diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 01192763a1..9f526819c8 100644 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -214,7 +214,7 @@ function lcovFunc { '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.c' '*/build_version.cc'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ - '*/sql.c' '*/sql.y'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\ --branch-coverage --function-coverage -o coverage.info # generate result diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim new file mode 100644 index 0000000000..d7daf4d333 --- /dev/null +++ b/tests/script/tsim/parser/last_both.sim @@ -0,0 +1,150 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======================== dnode1 start +$db = testdb +sql drop database if exists $db +sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf; +sql alter table st2 add column c1 int; +sql alter table st2 drop column c1; + +run tsim/parser/last_cache_query.sim + +sql flush database $db +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +run tsim/parser/last_cache_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sql drop database if exists $db +sql create database $db minrows 10 stt_trigger 1 +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) +sql create table tb1 using st2 tags (1); +sql create table tb2 using st2 tags (2); +sql create table tb3 using st2 tags (3); +sql create table tb4 using st2 tags (4); +sql create table tb5 using st2 tags (1); +sql create table tb6 using st2 tags (2); +sql create table tb7 using st2 tags (3); +sql create table tb8 using st2 tags (4); +sql create table tb9 using st2 tags (5); +sql create table tba using st2 tags (5); +sql create table tbb using st2 tags (5); +sql create table tbc using st2 tags (5); +sql create table tbd using st2 tags (5); +sql create table tbe using st2 tags (5); +sql create table tbf using st2 tags (5); + +sql insert into tb9 values ("2021-05-09 10:12:26.000",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.001",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.002",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.003",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.004",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.005",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.006",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.007",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.008",28, 29, '30', -1005) +sql insert into tb9 values ("2021-05-09 10:12:26.009",28, 29, '30', -1005) +sql delete from tb9 where ts = "2021-05-09 10:12:26.000" +sql flush database $db + +sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) + +sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3', -1001) +sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001) +sql insert into tb2 values ("2021-05-11 10:11:15",-6, -7, '-8', -3001) + +sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002) +sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002) +sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002) + +sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003) +sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003) +sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003) + +sql insert into tb5 values ("2021-05-09 10:12:22",19, 20, '21', -1004) +sql insert into tb6 values ("2021-05-11 10:12:23",22, 23, NULL, -2004) +sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004) +sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004) + +sql insert into tba values ("2021-05-10 10:12:27",31, 32, NULL, -2005) +sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) +sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) +sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) + +sql drop table tbf +sql alter database $db cachemodel 'both' +sql alter database $db cachesize 2 +sleep 11000 + +run tsim/parser/last_cache_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index 6cd5309590..c5961f2183 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -357,6 +357,112 @@ if $data45 != 5 then return -1 endi +sql select last_row(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != NULL then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != NULL then + print expect NULL actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != NULL then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != NULL then + return -1 +endi +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +if $data45 != 5 then + return -1 +endi + print "test tbn" sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) @@ -386,3 +492,5 @@ if $data04 != @70-01-01 07:59:57.000@ then return -1 endi +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim index c3a75d635b..5b95d4fad7 100644 --- a/tests/script/tsim/query/query_count0.sim +++ b/tests/script/tsim/query/query_count0.sim @@ -175,5 +175,51 @@ if $data33 != 3 then goto loop3 endi +print step3 +print =============== create database +sql create database test3 vgroups 1; +sql use test3; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql insert into t1 values(1648791213000,0,1,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.1); +sql insert into t1 values(1648791223002,0,3,3,2.1); +sql insert into t1 values(1648791223003,1,4,3,3.1); +sql insert into t1 values(1648791223004,1,5,3,4.1); +sql insert into t1 values(1648791223005,2,6,3,4.1); + +$loop_count = 0 +loop4: + +sleep 300 +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop4 +endi + +print 1 sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; +sql select _wstart, count(*),max(b) from t1 count_window(3) having max(b) > 6; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +# row 0 +if $rows != 0 then + print ======rows=$rows + return -1 +endi + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim new file mode 100644 index 0000000000..784ab7f4a5 --- /dev/null +++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim @@ -0,0 +1,267 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system sh/cfg.sh -n dnode1 -c streamAggCnt -v 2 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + + + +print ===== step1 +sql drop stream if exists streams1; +sql drop database if exists test; +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s); + +sleep 1000 + +sql insert into ts1 values(1648791213000,1,1,3,4.1); +sql insert into ts1 values(1648791223000,2,2,3,1.1); +sql insert into ts1 values(1648791233000,3,3,3,2.1); +sql insert into ts1 values(1648791243000,4,4,3,3.1); + +sql insert into ts2 values(1648791213000,1,5,3,4.1); +sql insert into ts2 values(1648791223000,2,6,3,1.1); +sql insert into ts2 values(1648791233000,3,7,3,2.1); +sql insert into ts2 values(1648791243000,4,8,3,3.1); + + +$loop_count = 0 +loop0: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop0 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop0 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop0 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop0 +endi + + +sql insert into ts1 values(1648791213000,1,9,3,4.1); + +$loop_count = 0 +loop1: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop1 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop1 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop1 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop1 +endi + +sql delete from ts2 where ts = 1648791243000 ; + +$loop_count = 0 +loop2: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop2 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop2 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +sql delete from ts2 where ts = 1648791223000 ; + +$loop_count = 0 +loop3: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop3 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop3 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop3 +endi + + +sql insert into ts1 values(1648791233001,3,9,3,2.1); + +$loop_count = 0 +loop4: + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sleep 1000 +print 2 select * from streamt1; +sql select * from streamt1; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 + +if $rows != 4 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); + + +print ===== over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 567d963c86..f193e375d9 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -99,8 +99,14 @@ sql select * from d2.stb2 sql select * from d2.ctb2 sql select * from d2.ntb2 -print =============== check show +$loop_cnt = 5 +$loop_idx = 0 +$loop_flag = 0 + +loop_check_sysinfo_0: +print =============== check show of sysinfo 0 sql_error show users +sql_error show user privileges sql_error show cluster sql_error select * from information_schema.ins_dnodes sql_error select * from information_schema.ins_mnodes @@ -129,6 +135,9 @@ sql show topics sql show subscriptions sql show functions sql_error show grants +sql_error show grants full; +sql_error show grants logs; +sql_error show cluster machines; sql show queries sql show connections sql show apps @@ -142,7 +151,7 @@ sql_error show dnode 1 variables; sql show variables; -print =============== check information_schema +print =============== check information_schema of sysinfo 0 sql show databases if $rows != 3 then return -1 @@ -155,7 +164,7 @@ sql_error select * from information_schema.ins_modules sql_error select * from information_schema.ins_qnodes sql_error select * from information_schema.ins_cluster sql_error select * from information_schema.ins_users -sql_error select * from information_schema.ins_user_privilege +sql_error select * from information_schema.ins_user_privileges sql select * from information_schema.ins_databases sql select * from information_schema.ins_functions sql select * from information_schema.ins_indexes @@ -166,11 +175,14 @@ sql select * from information_schema.ins_topics sql select * from information_schema.ins_subscriptions sql select * from information_schema.ins_streams sql_error select * from information_schema.ins_grants +sql_error select * from information_schema.ins_grants_full +sql_error select * from information_schema.ins_grants_logs +sql_error select * from information_schema.ins_machines sql_error select * from information_schema.ins_vgroups sql select * from information_schema.ins_configs sql_error select * from information_schema.ins_dnode_variables -print =============== check performance_schema +print =============== check performance_schema of sysinfo 0 sql use performance_schema; sql select * from performance_schema.perf_connections sql select * from performance_schema.perf_queries @@ -178,4 +190,114 @@ sql select * from performance_schema.perf_consumers sql select * from performance_schema.perf_trans sql select * from performance_schema.perf_apps +goto loop_check_switch + +loop_check_sysinfo_1: +print =============== check show of sysinfo 1 +sql show users +sql show user privileges +sql show cluster +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql show snodes +sql select * from information_schema.ins_qnodes +sql show dnodes +sql show snodes +sql show qnodes +sql show mnodes +sql show db.vgroups +sql_error show db.stables +sql_error show db.tables +sql show indexes from stb from db +sql show databases +sql show d2.vgroups +sql show d2.stables +sql show d2.tables +sql show indexes from stb2 from d2 +#sql_error show create database db +sql_error show create table db.stb; +sql_error show create table db.ctb; +sql_error show create table db.ntb; +sql show streams +sql show consumers +sql show topics +sql show subscriptions +sql show functions +sql show grants +sql show grants full; +sql show grants logs; +sql show cluster machines; +sql show queries +sql show connections +sql show apps +sql show transactions +sql show create database d2 +sql show create table d2.stb2; +sql show create table d2.ctb2; +sql show create table d2.ntb2; +sql show local variables; +sql show dnode 1 variables; +sql show variables; + + +print =============== check information_schema of sysinfo 1 +sql show databases +if $rows != 3 then + return -1 +endi + +sql use information_schema; +sql select * from information_schema.ins_dnodes +sql select * from information_schema.ins_mnodes +sql_error select * from information_schema.ins_modules +sql select * from information_schema.ins_qnodes +sql select * from information_schema.ins_cluster +sql select * from information_schema.ins_users +sql select * from information_schema.ins_user_privileges +sql select * from information_schema.ins_databases +sql select * from information_schema.ins_functions +sql select * from information_schema.ins_indexes +sql select * from information_schema.ins_stables +sql select * from information_schema.ins_tables +sql select * from information_schema.ins_tags +sql select * from information_schema.ins_topics +sql select * from information_schema.ins_subscriptions +sql select * from information_schema.ins_streams +sql select * from information_schema.ins_grants +sql select * from information_schema.ins_grants_full +sql select * from information_schema.ins_grants_logs +sql select * from information_schema.ins_machines +sql select * from information_schema.ins_vgroups +sql select * from information_schema.ins_configs +sql select * from information_schema.ins_dnode_variables + +print =============== check performance_schema of sysinfo 1 +sql use performance_schema; +sql select * from performance_schema.perf_connections +sql select * from performance_schema.perf_queries +sql select * from performance_schema.perf_consumers +sql select * from performance_schema.perf_trans +sql select * from performance_schema.perf_apps + +loop_check_switch: +if $loop_idx > $loop_cnt then + goto loop_check_end +endi +$loop_idx = $loop_idx + 1 + +if $loop_flag == 0 then + system taos -P7100 -s 'alter user sysinfo0 sysinfo 1' + sleep 3000 + $loop_flag = 1 + goto loop_check_sysinfo_1 +else + system taos -P7100 -s 'alter user sysinfo0 sysinfo 0' + sleep 3000 + $loop_flag = 0 + goto loop_check_sysinfo_0 +endi + +loop_check_end: +print =============== check end + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 169c70e675..690de65dbb 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -267,6 +267,7 @@ class TDTestCase: updatecfgDict["monitorInterval"] = "5" updatecfgDict["monitorMaxLogs"] = "10" updatecfgDict["monitorComp"] = "1" + updatecfgDict["monitorForceV2"] = "0" updatecfgDict["audit"] = '0' diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py similarity index 78% rename from tests/system-test/7-tmq/tmqVnodeTransform.py rename to tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py index c2b002ead6..a853489c3f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py @@ -122,135 +122,7 @@ class TDTestCase: tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - # expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - # tdSql.query(queryString) - # expectRowsList.append(tdSql.getRows()) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - tdLog.info("create ctb1") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tmqCom.getStartCommitNotifyFromTmqsim() - - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) - pInsertThread.join() - pInsertThread1.join() - - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - if expectrowcnt / 2 > resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) - tdLog.exit("%d tmq consume rows error!"%consumerId) - - # tmqCom.checkFileContent(consumerId, queryString) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName':'dbt'} - - ntbName = "ntb" - - topicNameList = ['topic2'] - tmqCom.initConsumerTable() - - sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) - tdLog.info("create nomal table sql: %s"%sqlString) - tdSql.execute(sqlString) - - tdLog.info("create topics from nomal table") - queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query("flush database %s"%(paraDict['dbName'])) - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) - tdLog.info("alter table sql: %s"%sqlString) - tdSql.error(sqlString) - expectRows = 0 - resultList = tmqCom.selectConsumeResult(expectRows) - time.sleep(1) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 2 end ...... ") - + def tmqCase3(self): tdLog.printNoPrefix("======== test case 3: ") paraDict = {'dbName': 'dbt', @@ -330,12 +202,90 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + # restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + def run(self): - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() self.prepareTestEnv() self.tmqCase3() + self.prepareTestEnv() + self.tmqCaseDbname() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py new file mode 100644 index 0000000000..40879d5c66 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -0,0 +1,266 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctbn" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(1) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index e9b9b9e944..847bbcf4be 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -105,7 +105,8 @@ SWords shellCommands[] = { {"create or replace aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, -#ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE + {"create view as select", 0, 0, NULL}, {"compact database ", 0, 0, NULL}, #endif {"describe ", 0, 0, NULL}, @@ -162,13 +163,20 @@ SWords shellCommands[] = { {"show create database \\G;", 0, 0, NULL}, {"show create stable \\G;", 0, 0, NULL}, {"show create table \\G;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"show create view \\G;", 0, 0, NULL}, +#endif {"show connections;", 0, 0, NULL}, + {"show compact", 0, 0, NULL}, + {"show compacts;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, {"show cluster alive;", 0, 0, NULL}, + {"show cluster machines;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, {"show functions;", 0, 0, NULL}, + {"show licences;", 0, 0, NULL}, {"show mnodes;", 0, 0, NULL}, {"show queries;", 0, 0, NULL}, // 80 @@ -185,6 +193,7 @@ SWords shellCommands[] = { {"show table distributed ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, {"show tags from ", 0, 0, NULL}, + {"show table tags from ", 0, 0, NULL}, {"show topics;", 0, 0, NULL}, {"show transactions;", 0, 0, NULL}, {"show users;", 0, 0, NULL}, @@ -194,7 +203,10 @@ SWords shellCommands[] = { {"show vgroups;", 0, 0, NULL}, {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, + {"show grants full;", 0, 0, NULL}, + {"show grants logs;", 0, 0, NULL}, #ifdef TD_ENTERPRISE + {"show views;", 0, 0, NULL}, {"split vgroup ", 0, 0, NULL}, #endif {"insert into values(", 0, 0, NULL}, @@ -302,6 +314,20 @@ char* key_systable[] = { char* udf_language[] = {"\'Python\'", "\'C\'"}; +// global keys can tips on anywhere +char* global_keys[] = { + "tbname", + "now", + "_wstart", + "_wend", + "_wduration", + "_qstart", + "_qend", + "_qduration", + "_qtag", + "_isfilled" + }; + // // ------- global variant define --------- // @@ -341,8 +367,9 @@ bool waitAutoFill = false; #define WT_VAR_KEYSELECT 20 #define WT_VAR_SYSTABLE 21 #define WT_VAR_LANGUAGE 22 +#define WT_VAR_GLOBALKEYS 23 -#define WT_VAR_CNT 23 +#define WT_VAR_CNT 24 #define WT_TEXT 0xFF @@ -494,10 +521,12 @@ void showHelp() { show connections;\n\ show cluster;\n\ show cluster alive;\n\ + show cluster machines;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ show functions;\n\ + show licences;\n\ show mnodes;\n\ show queries;\n\ show query ;\n\ @@ -513,6 +542,7 @@ void showHelp() { show table distributed ;\n\ show tags from \n\ show tags from \n\ + show table tags from \n\ show topics;\n\ show transactions;\n\ show users;\n\ @@ -522,6 +552,8 @@ void showHelp() { show vgroups;\n\ show consumers;\n\ show grants;\n\ + show grants full;\n\ + show grants logs;\n\ ----- T ----- \n\ trim database ;\n\ ----- U ----- \n\ @@ -534,8 +566,13 @@ void showHelp() { balance vgroup ;\n\ balance vgroup leader on \n\ compact database ; \n\ + crate view as select ...\n\ redistribute vgroup dnode ;\n\ - split vgroup ;"); + split vgroup ;\n\ + show compacts;\n\ + show compact \n\ + show views;\n\ + show create view ;"); #endif printf("\n\n"); @@ -699,6 +736,7 @@ bool shellAutoInit() { GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*)); GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*)); GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*)); + GenerateVarType(WT_VAR_GLOBALKEYS, global_keys, sizeof(global_keys) / sizeof(char*)); return true; } @@ -1800,6 +1838,13 @@ bool matchEnd(TAOS* con, SShellCmd* cmd) { goto _return; } + // global keys + if (fillWithType(con, cmd, last, WT_VAR_GLOBALKEYS)) { + ret = true; + goto _return; + } + + _return: taosMemoryFree(ps); return ret; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 23424cea98..995d3d04ec 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -75,7 +75,7 @@ bool shellIsEmptyCommand(const char *cmd) { int32_t shellRunSingleCommand(char *command) { shellCmdkilled = false; - + if (shellIsEmptyCommand(command)) { return 0; } @@ -1019,7 +1019,7 @@ void shellReadHistory() { char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); int32_t read_size = 0; - while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { + while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) > 0) { line[read_size - 1] = '\0'; taosMemoryFree(pHistory->hist[pHistory->hend]); pHistory->hist[pHistory->hend] = taosStrdup(line); @@ -1315,7 +1315,7 @@ int32_t shellExecute() { shellSetConn(shell.conn, runOnce); shellReadHistory(); - if(shell.args.is_bi_mode) { + if(shell.args.is_bi_mode) { // need set bi mode printf("Set BI mode is true.\n"); #ifndef WEBSOCKET diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 2c334eb67b..01619decc5 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1018,7 +1018,7 @@ int sml_escape_Test() { ASSERT(numFields == 5); ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0); ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0); - ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0); + ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\ree", sizeof("inode\"i,= s_f\\ree") - 1) == 0); ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0); TAOS_ROW row = NULL; @@ -1044,6 +1044,88 @@ int sml_escape_Test() { return code; } +// test field with end of escape +int sml_escape1_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "stab,t1\\=1 c1=3,c2=\"32fw\" 1661943970000000000", + "stab,t1=1\\ c1=3,c2=\"32fw\" 1661943980000000000", + "stab,t1=1 c1\\=3,c2=\"32fw\" 1661943990000000000", + }; + for(int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++){ + pRes = taos_schemaless_insert(taos, (char**)&sql[i], 1, TSDB_SML_LINE_PROTOCOL, 0); + int code = taos_errno(pRes); + ASSERT(code); + } + + const char *sql1[] = { + "stab\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000", + "stab\\\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000", + "stab,t1\\\\=1 c1=3,c2=\"32fw\" 1661943970000000000", + "stab,t1=1\\\\ c1=3,c2=\"32fw\" 1661943980000000000", + "stab,t1=1 c1\\\\=3,c2=\"32fw\" 1661943990000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 5); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stab"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 6); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 6); + ASSERT(strncmp(fields[1].name, "c1", sizeof("c1") - 1) == 0); + ASSERT(strncmp(fields[2].name, "c2", sizeof("c2") - 1) == 0); + ASSERT(strncmp(fields[3].name, "c1\\", sizeof("c1\\") - 1) == 0); + ASSERT(strncmp(fields[4].name, "t1\\", sizeof("t1\\") - 1) == 0); + ASSERT(strncmp(fields[5].name, "t1", sizeof("t1") - 1) == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + + if (ts == 1661943970000) { + ASSERT(*(double *)row[1] == 3); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(row[3] == NULL); + ASSERT(strncmp(row[4], "1", sizeof("1") - 1) == 0); + ASSERT(row[5] == NULL); + }else if (ts == 1661943980000) { + ASSERT(*(double *)row[1] == 3); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(row[3] == NULL); + ASSERT(row[4] == NULL); + ASSERT(strncmp(row[5], "1\\", sizeof("1\\") - 1) == 0); + }else if (ts == 1661943990000) { + ASSERT(row[1] == NULL); + ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0); + ASSERT(*(double *)row[3] == 3); + ASSERT(row[4] == NULL); + ASSERT(strncmp(row[5], "1", sizeof("1") - 1) == 0); + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1775,17 +1857,14 @@ int main(int argc, char *argv[]) { ASSERT(ret); ret = sml_escape_Test(); ASSERT(!ret); + ret = sml_escape1_Test(); + ASSERT(!ret); ret = sml_ts3116_Test(); ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); ret = sml_ts3303_Test(); ASSERT(!ret); - - // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ - // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); - // } - // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test();