From 58ade44c7ff8a30fd488280dcdb96617290f8f90 Mon Sep 17 00:00:00 2001 From: xleili Date: Wed, 3 Aug 2022 19:10:13 +0800 Subject: [PATCH 01/51] doc(driver):update nodejs reference 3.0 in CN --- .../node/nativeexample/async_query_example.js | 3 +- docs/examples/node/nativeexample/connect.js | 27 ++++--- .../nativeexample/influxdb_line_example.js | 2 +- .../node/nativeexample/insert_example.js | 12 +-- .../node/nativeexample/multi_bind_example.js | 10 +-- .../nativeexample/opentsdb_json_example.js | 2 +- .../nativeexample/opentsdb_telnet_example.js | 2 +- .../node/nativeexample/param_bind_example.js | 33 ++++---- .../node/nativeexample/query_example.js | 4 +- .../node/nativeexample/subscribe_demo.js | 51 ++++++++++++- docs/examples/node/package.json | 4 +- docs/examples/node/restexample/connect.js | 4 +- docs/zh/14-reference/03-connector/node.mdx | 75 ++++++++++--------- 13 files changed, 139 insertions(+), 90 deletions(-) diff --git a/docs/examples/node/nativeexample/async_query_example.js b/docs/examples/node/nativeexample/async_query_example.js index 25b78bc48a..432d8b8f6c 100644 --- a/docs/examples/node/nativeexample/async_query_example.js +++ b/docs/examples/node/nativeexample/async_query_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); const cursor = conn.cursor(); @@ -18,4 +18,3 @@ try { conn.close(); }, 2000); } -// bug here: jira 14506 diff --git a/docs/examples/node/nativeexample/connect.js b/docs/examples/node/nativeexample/connect.js index da791c566e..93d22762ad 100644 --- a/docs/examples/node/nativeexample/connect.js +++ b/docs/examples/node/nativeexample/connect.js @@ -1,13 +1,20 @@ -const taos = require("td2.0-connector"); +const { options, connect } = require("@tdengine/rest"); -var conn = taos.connect({ - host: "localhost", - port: 6030, - user: "root", - password: "taosdata", -}); -conn.close(); +async function test() { + options.path = "/rest/sql"; + options.host = "localhost"; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query("SELECT server_version()"); + res.toString(); + } catch (err) { + console.log(err); + } +} +test(); -// run with: node connect.js // output: -// Successfully connected to TDengine +// server_version() | +// =================== +// 3.0.0 | diff --git a/docs/examples/node/nativeexample/influxdb_line_example.js b/docs/examples/node/nativeexample/influxdb_line_example.js index 2050bee545..57170770d8 100644 --- a/docs/examples/node/nativeexample/influxdb_line_example.js +++ b/docs/examples/node/nativeexample/influxdb_line_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/insert_example.js b/docs/examples/node/nativeexample/insert_example.js index ade9d83158..42050fa251 100644 --- a/docs/examples/node/nativeexample/insert_example.js +++ b/docs/examples/node/nativeexample/insert_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -11,11 +11,11 @@ try { cursor.execute( "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)" ); - var sql = `INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)`; - cursor.execute(sql); + var sql = `INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)`; + cursor.execute(sql,{'quiet':false}); } finally { cursor.close(); conn.close(); diff --git a/docs/examples/node/nativeexample/multi_bind_example.js b/docs/examples/node/nativeexample/multi_bind_example.js index 6ef8b30c09..6cc3993397 100644 --- a/docs/examples/node/nativeexample/multi_bind_example.js +++ b/docs/examples/node/nativeexample/multi_bind_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -24,10 +24,10 @@ function insertData() { ); // bind table name and tags - let tagBind = new taos.TaosBind(2); - tagBind.bindBinary("California.SanFrancisco"); - tagBind.bindInt(2); - cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + let tagBind = new taos.TaosMultiBindArr(2); + tagBind.multiBindBinary(["California.SanFrancisco"]); + tagBind.multiBindInt([2]); + cursor.stmtSetTbnameTags("d1001", tagBind.getMultiBindArr()); // bind values let valueBind = new taos.TaosMultiBindArr(4); diff --git a/docs/examples/node/nativeexample/opentsdb_json_example.js b/docs/examples/node/nativeexample/opentsdb_json_example.js index 2d78444a3f..4ea36ca68b 100644 --- a/docs/examples/node/nativeexample/opentsdb_json_example.js +++ b/docs/examples/node/nativeexample/opentsdb_json_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/opentsdb_telnet_example.js b/docs/examples/node/nativeexample/opentsdb_telnet_example.js index 7f80f55883..8f5d17822b 100644 --- a/docs/examples/node/nativeexample/opentsdb_telnet_example.js +++ b/docs/examples/node/nativeexample/opentsdb_telnet_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/param_bind_example.js b/docs/examples/node/nativeexample/param_bind_example.js index c7e04c71a0..efe422586f 100644 --- a/docs/examples/node/nativeexample/param_bind_example.js +++ b/docs/examples/node/nativeexample/param_bind_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -23,25 +23,22 @@ function insertData() { ); // bind table name and tags - let tagBind = new taos.TaosBind(2); - tagBind.bindBinary("California.SanFrancisco"); - tagBind.bindInt(2); - cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + let tagBind = new taos.TaosMultiBindArr(2); + tagBind.multiBindBinary(["California.SanFrancisco"]); + tagBind.multiBindInt([2]); + cursor.stmtSetTbnameTags("d1001", tagBind.getMultiBindArr()); // bind values - let rows = [ - [1648432611249, 10.3, 219, 0.31], - [1648432611749, 12.6, 218, 0.33], - ]; - for (let row of rows) { - let valueBind = new taos.TaosBind(4); - valueBind.bindTimestamp(row[0]); - valueBind.bindFloat(row[1]); - valueBind.bindInt(row[2]); - valueBind.bindFloat(row[3]); - cursor.stmtBindParam(valueBind.getBind()); - cursor.stmtAddBatch(); - } + let rows = [[1648432611249, 1648432611749], [10.3, 12.6], [219, 218], [0.31, 0.33]]; + + let valueBind = new taos.TaosMultiBindArr(4); + valueBind.multiBindTimestamp(rows[0]); + valueBind.multiBindFloat(rows[1]); + valueBind.multiBindInt(rows[2]); + valueBind.multiBindFloat(rows[3]); + cursor.stmtBindParamBatch(valueBind.getMultiBindArr()); + cursor.stmtAddBatch(); + // execute cursor.stmtExecute(); diff --git a/docs/examples/node/nativeexample/query_example.js b/docs/examples/node/nativeexample/query_example.js index a51bc939ae..11cfd9a605 100644 --- a/docs/examples/node/nativeexample/query_example.js +++ b/docs/examples/node/nativeexample/query_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); const cursor = conn.cursor(); @@ -9,8 +9,6 @@ query.execute().then(function (result) { // output: // Successfully connected to TDengine -// Query OK, 2 row(s) in set (0.00317767s) - // ts | current | // ======================================================= // 2018-10-03 14:38:05.000 | 10.3 | diff --git a/docs/examples/node/nativeexample/subscribe_demo.js b/docs/examples/node/nativeexample/subscribe_demo.js index bdf0c98687..c4f7e6df84 100644 --- a/docs/examples/node/nativeexample/subscribe_demo.js +++ b/docs/examples/node/nativeexample/subscribe_demo.js @@ -1,4 +1,51 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); -// 未完成 \ No newline at end of file +var cursor = conn.cursor(); + +function runConsumer() { + + // create topic + cursor.execute("create topic topic_name_example as select * from meters"); + + let consumer = taos.consumer({ + 'group.id': 'tg2', + 'td.connect.user': 'root', + 'td.connect.pass': 'taosdata', + 'msg.with.table.name': 'true', + 'enable.auto.commit': 'true' + }); + + // subscribe the topic just created. + consumer.subscribe("topic_name_example"); + + // get subscribe topic list + let topicList = consumer.subscription(); + console.log(topicList); + + for (let i = 0; i < 5; i++) { + let msg = consumer.consume(100); + console.log(msg.topicPartition); + console.log(msg.block); + console.log(msg.fields) + consumer.commit(msg); + console.log(`=======consumer ${i} done`) + } + + consumer.unsubscribe(); + consumer.close(); + + // drop topic + cursor.execute("drop topic topic_name_example"); +} + + +try { + runConsumer(); +} finally { + + setTimeout(() => { + cursor.close(); + conn.close(); + }, 2000); +} \ No newline at end of file diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index f56196d2e5..36d3f016b5 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,7 +4,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "td2.0-connector": "^2.0.12", - "td2.0-rest-connector": "^1.0.0" + "@tdengine/client": "^3.0.0", + "@tdengine/rest": "^3.0.0" } } diff --git a/docs/examples/node/restexample/connect.js b/docs/examples/node/restexample/connect.js index b84ce2fadf..9da370043d 100644 --- a/docs/examples/node/restexample/connect.js +++ b/docs/examples/node/restexample/connect.js @@ -1,4 +1,4 @@ -const { options, connect } = require("td2.0-rest-connector"); +const { options, connect } = require("@tdengine/rest"); async function test() { options.path = "/rest/sqlt"; @@ -17,4 +17,4 @@ test(); // output: // server_version() | // =================== -// 2.4.0.12 | +// 3.0.0 | diff --git a/docs/zh/14-reference/03-connector/node.mdx b/docs/zh/14-reference/03-connector/node.mdx index 9f2bed9e97..b089da99d2 100644 --- a/docs/zh/14-reference/03-connector/node.mdx +++ b/docs/zh/14-reference/03-connector/node.mdx @@ -15,11 +15,11 @@ import NodeOpenTSDBTelnet from "../../07-develop/03-insert-data/_js_opts_telnet. import NodeOpenTSDBJson from "../../07-develop/03-insert-data/_js_opts_json.mdx"; import NodeQuery from "../../07-develop/04-query-data/_js.mdx"; -`td2.0-connector` 和 `td2.0-rest-connector` 是 TDengine 的官方 Node.js 语言连接器。Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。 +`@tdengine/client` 和 `@tdengine/rest` 是 TDengine 的官方 Node.js 语言连接器。 Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。注意：从 TDengine 3.0 开始 Node.js 原生连接器的包名由 `td2.0-connector` 改名为 `@tdengine/client` 而 rest 连接器的包名由 `td2.0-rest-connector` 改为 `@tdengine/rest`。并且不与 TDengine 2.x 兼容。 -`td2.0-connector` 是**原生连接器**，它通过 TDengine 客户端驱动程序（taosc）连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。`td2.0-rest-connector` 是 **REST 连接器**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 的运行实例。REST 连接器可以在任何平台运行，但性能略为下降，接口实现的功能特性集合和原生接口有少量不同。 +`@tdengine/client` 是**原生连接器**，它通过 TDengine 客户端驱动程序（taosc）连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。`@tdengine/rest` 是 **REST 连接器**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 的运行实例。REST 连接器可以在任何平台运行，但性能略为下降，接口实现的功能特性集合和原生接口有少量不同。 -Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-connector-node)。 +Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-connector-node/tree/3.0)。 ## 支持的平台 @@ -58,7 +58,7 @@ REST 连接器支持所有能运行 Node.js 的平台。 - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) -- `td2.0-connector` 2.0.6 支持 Node.js LTS v10.9.0 或更高版本, Node.js LTS v12.8.0 或更高版本；2.0.5 及更早版本支持 Node.js LTS v10.x 版本。其他版本可能存在包兼容性的问题 +- `@tdengine/client` 3.0.0 支持 Node.js LTS v10.9.0 或更高版本, Node.js LTS v12.8.0 或更高版本；其他版本可能存在包兼容性的问题 - `make` - C 语言编译器，[GCC](https://gcc.gnu.org) v4.8.5 或更高版本 @@ -90,14 +90,14 @@ REST 连接器支持所有能运行 Node.js 的平台。 ```bash -npm install td2.0-connector +npm install @tdengine/client ``` ```bash -npm i td2.0-rest-connector +npm install @tdengine/rest ``` @@ -109,13 +109,13 @@ npm i td2.0-rest-connector 验证方法： -- 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/develop/examples/nodejs/nodejsChecker.js)到本地。 +- 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/nodejsChecker.js)到本地。 - 在命令行中执行以下命令。 ```bash npm init -y -npm install td2.0-connector +npm install @tdengine/client node nodejsChecker.js host=localhost ``` @@ -128,11 +128,11 @@ node nodejsChecker.js host=localhost -安装并引用 `td2.0-connector` 包。 +安装并引用 `@tdengine/client` 包。 ```javascript //A cursor also needs to be initialized in order to interact with TDengine from Node.js. -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); var conn = taos.connect({ host: "127.0.0.1", user: "root", @@ -149,12 +149,12 @@ conn.close(); -安装并引用 `td2.0-rest-connector` 包。 +安装并引用 `@tdengine/rest` 包。 ```javascript //A cursor also needs to be initialized in order to interact with TDengine from Node.js. -import { options, connect } from "td2.0-rest-connector"; -options.path = "/rest/sqlt"; +import { options, connect } from "@tdengine/rest"; +options.path = "/rest/sql"; // set host options.host = "localhost"; // set other options like user/passwd @@ -190,26 +190,23 @@ let cursor = conn.cursor(); + ## 更多示例程序 | 示例程序 | 示例程序描述 | | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------- | -| [connection](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/cursorClose.js) | 建立连接的示例。 | -| [stmtBindBatch](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindParamBatchSample.js) | 绑定多行参数插入的示例。 | -| [stmtBind](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindParamSample.js) | 一行一行绑定参数插入的示例。 | -| [stmtBindSingleParamBatch](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindSingleParamBatchSample.js) | 按列绑定参数插入的示例。 | -| [stmtUseResult](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtUseResultSample.js) | 绑定参数查询的示例。 | -| [json tag](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testJsonTag.js) | Json tag 的使用示例。 | -| [Nanosecond](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testNanoseconds.js) | 时间戳为纳秒精度的使用的示例。 | -| [Microsecond](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testMicroseconds.js) | 时间戳为微秒精度的使用的示例。 | -| [schemless insert](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testSchemalessInsert.js) | schemless 插入的示例。 | -| [subscribe](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testSubscribe.js) | 订阅的使用示例。 | -| [asyncQuery](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/tset.js) | 异步查询的使用示例。 | -| [REST](https://github.com/taosdata/taos-connector-node/blob/develop/typescript-rest/example/example.ts) | 使用 REST 连接的 TypeScript 使用示例。 | +| [basicUse](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/queryExample.js) | 基本的使用如如建立连接，执行 SQL 等操作。 | +| [stmtBindBatch](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/bindParamBatch.js) | 绑定多行参数插入的示例。 | | +| [stmtBindSingleParamBatch](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/bindSingleParamBatch.js) | 按列绑定参数插入的示例。 | +| [stmtQuery](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/stmtQuery.js) | 绑定参数查询的示例。 | +| [schemless insert](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/schemaless.js) | schemless 插入的示例。 | +| [TMQ](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/tmq.js) | 订阅的使用示例。 | +| [asyncQuery](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/asyncQueryExample.js) | 异步查询的使用示例。 | +| [REST](https://github.com/taosdata/taos-connector-node/blob/3.0/typescript-rest/example/example.ts) | 使用 REST 连接的 TypeScript 使用示例。 | ## 使用限制 -Node.js 连接器 >= v2.0.6 目前支持 node 的版本为：支持 >=v12.8.0 <= v12.9.1 || >=v10.20.0 <= v10.9.0 ；2.0.5 及更早版本支持 v10.x 版本，其他版本可能存在包兼容性的问题。 +native 连接器（`@tdengine/client`） >= v3.0.0 目前支持 node 的版本为：支持 >=v12.8.0 <= v12.9.1 || >=v10.20.0 <= v10.9.0 ；2.0.5 及更早版本支持 v10.x 版本，其他版本可能存在包兼容性的问题。 ## 其他说明 @@ -225,7 +222,7 @@ Node.js 连接器的使用参见[视频教程](https://www.taosdata.com/blog/202 2. Node.js 版本 - 连接器 >v2.0.6 目前兼容的 Node.js 版本为：>=v10.20.0 <= v10.9.0 || >=v12.8.0 <= v12.9.1 + 原生连接器 `@tdengine/client` 目前兼容的 Node.js 版本为：>=v10.20.0 <= v10.9.0 || >=v12.8.0 <= v12.9.1 3. "Unable to establish connection"，"Unable to resolve FQDN" @@ -235,18 +232,22 @@ Node.js 连接器的使用参见[视频教程](https://www.taosdata.com/blog/202 ### 原生连接器 -| td2.0-connector 版本 | 说明 | -| -------------------- | ---------------------------------------------------------------- | -| 2.0.12 | 修复 cursor.close() 报错的 bug。 | -| 2.0.11 | 支持绑定参数、json tag、schemaless 接口等功能。 | -| 2.0.10 | 支持连接管理，普通查询、连续查询、获取系统信息、订阅功能等功能。 | - +| package name | version | TDengine version | 说明 | +|------------------|---------|---------------------|------------------------------------------------------------------| +| @tdengine/client | 3.0.0 | 3.0.0 | 支持TDengine 3.0 且不与2.x 兼容。 | +| td2.0-connector | 2.0.12 | 2.4.x；2.5.x；2.6.x | 修复 cursor.close() 报错的 bug。 | +| td2.0-connector | 2.0.11 | 2.4.x；2.5.x；2.6.x | 支持绑定参数、json tag、schemaless 接口等功能。 | +| td2.0-connector | 2.0.10 | 2.4.x；2.5.x；2.6.x | 支持连接管理，普通查询、连续查询、获取系统信息、订阅功能等功能。 | ### REST 连接器 -| td2.0-rest-connector 版本 | 说明 | -| ------------------------- | ---------------------------------------------------------------- | -| 1.0.3 | 支持连接管理、普通查询、获取系统信息、错误信息、连续查询等功能。 | +| package name | version | TDengine version | 说明 | +|----------------------|---------|---------------------|---------------------------------------------------------------------------| +| @tdengine/rest | 3.0.0 | 3.0.0 | 支持 TDegnine 3.0，且不与2.x 兼容。 | +| td2.0-rest-connector | 1.0.7 | 2.4.x；2.5.x；2.6.x | 移除默认端口 6041。 | +| td2.0-rest-connector | 1.0.6 | 2.4.x；2.5.x；2.6.x | 修复create，insert，update，alter 等SQL 执行返回的 affectRows 错误的bug。 | +| td2.0-rest-connector | 1.0.5 | 2.4.x；2.5.x；2.6.x | 支持云服务 cloud Token； | +| td2.0-rest-connector | 1.0.3 | 2.4.x；2.5.x；2.6.x | 支持连接管理、普通查询、获取系统信息、错误信息、连续查询等功能。 | ## API 参考 -[API 参考](https://docs.taosdata.com/api/td2.0-connector/) +[API 参考](https://docs.taosdata.com/api/td2.0-connector/) \ No newline at end of file From fd18a2b8c63fff243ed2033050310d6f9c04ea23 Mon Sep 17 00:00:00 2001 From: xleili Date: Wed, 3 Aug 2022 20:28:18 +0800 Subject: [PATCH 02/51] docs(driver):nodejs 3.0 reference fixed with comment --- docs/examples/node/nativeexample/connect.js | 2 +- docs/examples/node/restexample/connect.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/node/nativeexample/connect.js b/docs/examples/node/nativeexample/connect.js index 93d22762ad..bb027d4fe8 100644 --- a/docs/examples/node/nativeexample/connect.js +++ b/docs/examples/node/nativeexample/connect.js @@ -17,4 +17,4 @@ test(); // output: // server_version() | // =================== -// 3.0.0 | +// 3.0.0.0 | diff --git a/docs/examples/node/restexample/connect.js b/docs/examples/node/restexample/connect.js index 9da370043d..132e284ce9 100644 --- a/docs/examples/node/restexample/connect.js +++ b/docs/examples/node/restexample/connect.js @@ -17,4 +17,4 @@ test(); // output: // server_version() | // =================== -// 3.0.0 | +// 3.0.0.0 | From bd81d9b5fc8ea08dc86e5cfe1e033f794f490704 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 3 Aug 2022 21:08:52 +0800 Subject: [PATCH 03/51] enh: improve catalog performance --- include/common/tmsg.h | 2 + source/dnode/mnode/impl/src/mndQuery.c | 6 + source/dnode/vnode/src/vnd/vnodeQuery.c | 6 + source/libs/catalog/inc/catalogInt.h | 24 +++ source/libs/catalog/src/ctgAsync.c | 237 +++++++++++++++++------- source/libs/catalog/src/ctgCache.c | 54 ++++++ source/libs/catalog/src/ctgRemote.c | 81 +++++--- source/libs/catalog/src/ctgUtil.c | 57 ++++++ source/libs/qworker/src/qwMsg.c | 7 +- 9 files changed, 382 insertions(+), 92 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9bc2ae4268..7ebe7e465c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3053,6 +3053,7 @@ int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); typedef struct { + int32_t msgIdx; int32_t msgType; int32_t msgLen; void* msg; @@ -3066,6 +3067,7 @@ typedef struct { typedef struct { int32_t reqType; + int32_t msgIdx; int32_t msgLen; int32_t rspCode; void* msg; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 2beeb10335..654f46ec85 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -84,6 +84,9 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -111,6 +114,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } else { rsp.rspCode = 0; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.info.rspLen; rsp.msg = reqMsg.info.rsp; @@ -136,6 +140,8 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { *(int32_t*)((char*)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t*)((char*)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t*)((char*)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t*)((char*)pRsp + offset) = htonl(p->rspCode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d18ba88268..707c4bc471 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -273,6 +273,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -301,6 +304,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { break; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.contLen; rsp.rspCode = reqMsg.code; @@ -327,6 +331,8 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { *(int32_t *)((char *)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t *)((char *)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t *)((char *)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t *)((char *)pRsp + offset) = htonl(p->rspCode); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 816309beb1..a4e47e8053 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -80,6 +80,7 @@ typedef enum { CTG_TASK_GET_UDF, CTG_TASK_GET_USER, CTG_TASK_GET_SVR_VER, + CTG_TASK_GET_TB_META_BATCH, } CTG_TASK_TYPE; typedef enum { @@ -110,6 +111,22 @@ typedef struct SCtgTbMetaCtx { int32_t flag; } SCtgTbMetaCtx; +typedef struct SCtgFetch { + int32_t reqIdx; + int32_t fetchIdx; + int32_t flag; + SCtgTbCacheInfo tbInfo; + int32_t vgId; +} SCtgFetch; + +typedef struct SCtgTbMetaBCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pTbMetas; + SArray* pFetchs; +} SCtgTbMetaBCtx; + + typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; @@ -218,6 +235,7 @@ typedef struct SCtgJob { int32_t batchId; SHashObj* pBatchs; SArray* pTasks; + int32_t subTaskNum; int32_t taskDone; SMetaData jobRes; int32_t jobResCode; @@ -276,6 +294,7 @@ typedef struct SCtgTask { int32_t taskId; SCtgJob* pJob; void* taskCtx; + SArray* msgCtxs; SCtgMsgCtx msgCtx; int32_t code; void* res; @@ -284,6 +303,7 @@ typedef struct SCtgTask { SArray* pParents; SCtgSubRes subRes; SHashObj* pBatchs; + int32_t msgIdx; } SCtgTask; typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); @@ -487,6 +507,8 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_GET_TASK_MSGCTX(_task) ((CTG_TASK_GET_TB_META_BATCH == (_task)->type) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) + #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) @@ -586,6 +608,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -672,6 +695,7 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 0184ac3a60..8aa33b759a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -50,6 +50,31 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_META_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaBCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbMetaBCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pTbMetas = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { char *dbFName = (char*)param; SCtgTask task = {0}; @@ -328,7 +353,6 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { } - int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pDb) { @@ -452,7 +476,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const } SCtgJob *pJob = *job; - + + pJob->subTaskNum = taskNum; pJob->queryId = pConn->requestId; pJob->userFp = fp; pJob->pCtg = pCtg; @@ -506,10 +531,16 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } +#if 0 for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META, name, NULL)); } +#else + if (tbMetaNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META_BATCH, pReq->pTableMeta, NULL)); + } +#endif for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); @@ -586,6 +617,15 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbMetaBRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableMeta = pTask->res; + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpDbVgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbVgroup) { @@ -847,43 +887,55 @@ _return: int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); +#if CTG_BATCH_FETCH + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; +#else + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SName* pName = ctx->pName; + int32_t flag = ctx->flag; + int32_t* vgId = &ctx->vgId; +#endif - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); return TSDB_CODE_SUCCESS; } case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -898,52 +950,58 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * return TSDB_CODE_SUCCESS; } - ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (pTask->msgCtx.lastOut) { - TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut); - STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out; + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; TSWAP(pLastOut->tbMeta, pOut->tbMeta); } break; } case TDMT_VND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { break; } if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; - if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + exist = 1; + } } if (0 == exist) { - TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out); + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); - } else { - taosMemoryFreeClear(pOut->tbMeta); - - SET_META_TYPE_CTABLE(pOut->metaType); } } break; @@ -954,17 +1012,20 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * break; } - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; ctgUpdateTbMetaToCache(pCtg, pOut, false); if (CTG_IS_META_BOTH(pOut->metaType)) { memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); - } else if (CTG_IS_META_CTABLE(pOut->metaType)) { - SName stbName = *ctx->pName; + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; strcpy(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; - stbCtx.flag = ctx->flag; + stbCtx.flag = flag; stbCtx.pName = &stbName; CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); @@ -977,8 +1038,19 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } +*/ +#if CTG_BATCH_FETCH + SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + pRes->code = 0; + pRes->pRes = pOut->tbMeta; + pOut->tbMeta = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pTbMetas); + } +#else TSWAP(pTask->res, pOut->tbMeta); +#endif _return: @@ -986,8 +1058,10 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } - ctgHandleTaskEnd(pTask, code); - + if (pTask->res || code) { + ctgHandleTaskEnd(pTask, code); + } + CTG_RET(code); } @@ -1224,38 +1298,37 @@ _return: CTG_RET(code); } -int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { +int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32_t* vgId) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; int32_t code = 0; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_SYS_DB(flag)) { + ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(pName)); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, pTask)); } - if (CTG_FLAG_IS_STB(ctx->flag)) { - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_STB(flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pName)); // if get from mnode failed, will not try vnode - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); } SCtgDBCache *dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); } else { SBuildUseDBInput input = {0}; @@ -1284,11 +1357,40 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask)); + SCtgTbMetaCtx* pCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pCtx->flag, pCtx->pName, &pCtx->vgId)); return TSDB_CODE_SUCCESS; } +int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, (SCtgTbMetaBCtx*)pTask->taskCtx, (SArray**)&pTask->res)); + if (pTask->res) { + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + + pTask->msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgAddMsgCtx(pTask->msgCtxs, 0, NULL, NULL)); + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); + } + + pTask->msgIdx = 0; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1597,19 +1699,20 @@ int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) { SCtgAsyncFps gCtgAsyncFps[] = { - {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, - {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, - {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, - {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, - {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, - {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, - {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, - {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, - {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, - {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, - {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, - {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, - {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, + {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, + {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, + {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, + {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, + {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, + {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, + {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, + {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, + {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, + {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, + {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, + {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2e8e259151..1b5968d426 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2152,6 +2152,60 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { + int32_t tbNum = taosArrayGetSize(ctx->pNames); + SName* fName = taosArrayGet(ctx->pNames, 0); + int32_t fIdx = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(ctx->pNames, i); + SCtgTbMetaCtx nctx = {0}; + nctx.flag = CTG_FLAG_UNKNOWN_STB; + nctx.pName = pName; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(nctx.flag); + } + + STableMeta *pTableMeta = NULL; + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, &nctx, &pTableMeta)); + SMetaRes res = {0}; + + if (pTableMeta) { + if (CTG_FLAG_MATCH_STB(nctx.flag, pTableMeta->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(nctx.flag)) || (CTG_FLAG_IS_SYS_DB(nctx.flag)))) { + res.pRes = pTableMeta; + } else { + taosMemoryFreeClear(pTableMeta); + } + } + + if (NULL == res.pRes) { + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(nctx.flag)) { + CTG_FLAG_SET_STB(nctx.flag, nctx.tbInfo.tbType); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = nctx.flag; + + taosArrayPush(ctx->pFetchs, &fetch); + } + + taosArrayPush(ctx->pTbMetas, &res); + } + + if (NULL == ctx->pFetchs) { + TSWAP(*pTbMetas, ctx->pTbMetas); + } + + return TSDB_CODE_SUCCESS; +} int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { int32_t code = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 0f97b5c5b1..d489496e37 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -46,6 +46,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.reqType); + rsp.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); + offset += sizeof(rsp.msgIdx); rsp.msgLen = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.msgLen); rsp.rspCode = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); @@ -57,6 +59,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.pData = rsp.msg; taskMsg.len = rsp.msgLen; } else { + rsp.msgIdx = pTask->msgIdx++; rsp.reqType = -1; taskMsg.msgType = -1; taskMsg.pData = NULL; @@ -64,6 +67,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } pTask->pBatchs = pBatchs; + pTask->msgIdx = rsp.msgIdx; ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); @@ -431,19 +435,19 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT SHashObj* pBatchs = pTask->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); - int32_t taskNum = taosArrayGetSize(pTask->pJob->pTasks); SCtgBatch newBatch = {0}; SBatchMsg req = {0}; if (NULL == pBatch) { - newBatch.pMsgs = taosArrayInit(taskNum, sizeof(SBatchMsg)); - newBatch.pTaskIds = taosArrayInit(taskNum, sizeof(int32_t)); + newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); + newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } newBatch.conn = *pConn; + req.msgIdx = pTask->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -456,16 +460,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, newBatch.dbFName); } newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; @@ -480,6 +493,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT return TSDB_CODE_SUCCESS; } + req.msgIdx = pTask->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -492,16 +506,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, newBatch.dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, vgId); @@ -517,7 +540,7 @@ _return: } int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { - *msg = taosMemoryMalloc(pBatch->msgSize); + *msg = taosMemoryCalloc(1, pBatch->msgSize); if (NULL == (*msg)) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -532,6 +555,8 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { for (int32_t i = 0; i < num; ++i) { SBatchMsg* pReq = taosArrayGet(pBatch->pMsgs, i); + *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgIdx); + offset += sizeof(pReq->msgIdx); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgType); offset += sizeof(pReq->msgType); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgLen); @@ -599,7 +624,8 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -645,7 +671,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -696,7 +722,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, input->db)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -746,7 +773,8 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)dbFName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -796,7 +824,8 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)indexName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -849,7 +878,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)tbFName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -899,7 +928,8 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)funcName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -949,7 +979,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)user)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -1004,7 +1035,9 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); #else @@ -1068,13 +1101,14 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); #else @@ -1129,7 +1163,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, @@ -1188,7 +1222,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); #else @@ -1233,7 +1268,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index bdf60b110b..9f0d0f71d5 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -88,6 +88,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get user]"; case CTG_TASK_GET_SVR_VER: return "[get svr ver]"; + case CTG_TASK_GET_TB_META_BATCH: + return "[bget table meta]"; default: return "unknown"; } @@ -460,6 +462,15 @@ void ctgResetTbMetaTask(SCtgTask* pTask) { taosMemoryFreeClear(pTask->res); } +void ctgFreeBatchMeta(void* meta) { + if (NULL == meta) { + return; + } + + SMetaRes* pRes = (SMetaRes*)meta; + taosMemoryFreeClear(pRes->pRes); +} + void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -500,6 +511,15 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchMeta(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } default: qError("invalid task type %d", type); break; @@ -554,6 +574,11 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchMeta); + *pRes = NULL; + break; + } default: qError("invalid task type %d", type); break; @@ -583,6 +608,21 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pTbMetas, ctgFreeBatchMeta); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_HASH: { SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -679,6 +719,23 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ return TSDB_CODE_SUCCESS; } +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { + SCtgMsgCtx ctx = {0}; + + ctx.reqType = reqType; + ctx.out = out; + if (target) { + ctx.target = strdup(target); + if (NULL == ctx.target) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pCtxs, &ctx); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { switch (hashMethod) { diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e8ffd98153..6bf2529551 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -387,10 +387,13 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); - QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); + QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(sql); + return code; } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { From bdee43e3cc8ba8e11d6fab21bdbf6f7536c48098 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 09:01:35 +0800 Subject: [PATCH 04/51] fix: fix crash issue --- source/libs/catalog/src/ctgAsync.c | 13 ++++++++++++- source/libs/qworker/src/qwMsg.c | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8aa33b759a..8dc32d306a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1058,7 +1058,18 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } - if (pTask->res || code) { +#if CTG_BATCH_FETCH + if (code) { + SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + pRes->code = code; + pRes->pRes = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pTbMetas); + } + } +#endif + + if (pTask->res) { ctgHandleTaskEnd(pTask, code); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 6bf2529551..2348f1ef21 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -388,11 +388,11 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); - QW_SCH_TASK_DLOG("processQuery end, node:%p", node); _return: - taosMemoryFree(sql); + QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%d", node, code); + return code; } From bc7698366a5b712b93a71208dae8e3f4d8ced3c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 11:11:26 +0800 Subject: [PATCH 05/51] enh: launch tables in same db --- source/libs/catalog/src/ctgCache.c | 273 +++++++++++++++++++++++++++- source/libs/catalog/src/ctgRemote.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 + 3 files changed, 274 insertions(+), 3 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1b5968d426..6cec690bad 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -242,7 +242,6 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S goto _return; } - int32_t sz = 0; pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); @@ -282,7 +281,6 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, goto _return; } - int32_t sz = 0; char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); @@ -2152,6 +2150,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet return TSDB_CODE_SUCCESS; } +#if 0 int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); @@ -2206,6 +2205,276 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe return TSDB_CODE_SUCCESS; } +#endif + +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { + int32_t tbNum = taosArrayGetSize(ctx->pNames); + int32_t fIdx = 0; + SName* pName = taosArrayGet(ctx->pNames, 0); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t flag = CTG_FLAG_UNKNOWN_STB; + uint64_t lastSuid = 0; + STableMeta* lastTableMeta = NULL; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(flag); + strcpy(dbFName, pName->dbname); + } else { + tNameGetFullDbName(pName, dbFName); + } + + SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + for (int32_t i = 0; i < tbNum; ++i) { + SMetaRes res = {0}; + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + } + + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(ctx->pNames, i); + SMetaRes res = {0}; + + pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + STableMeta* tbMeta = pCache->pMeta; + + SCtgTbMetaCtx nctx = {0}; + nctx.flag = flag; + nctx.tbInfo.inCache = true; + nctx.tbInfo.dbId = dbCache->dbId; + nctx.tbInfo.suid = tbMeta->suid; + nctx.tbInfo.tbType = tbMeta->tableType; + + STableMeta* pTableMeta = NULL; + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + //ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + // PROCESS FOR CHILD TABLE + + if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { + cloneTableMeta(lastTableMeta, &pTableMeta); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + int32_t metaSize = sizeof(SCTableMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + pName->tname, nctx.tbInfo.tbType, dbFName); + + char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + STableMeta* stbMeta = pCache->pMeta; + if (stbMeta->suid != nctx.tbInfo.suid) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + metaSize = CTG_META_SIZE(stbMeta); + pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); + if (NULL == pTableMeta) { + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + lastSuid = pTableMeta->suid; + lastTableMeta = pTableMeta; + } + + if (NULL == ctx->pFetchs) { + TSWAP(*pTbMetas, ctx->pTbMetas); + } + + return TSDB_CODE_SUCCESS; +} + int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { int32_t code = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index d489496e37..a8d5a7ae95 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -69,7 +69,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pTask->pBatchs = pBatchs; pTask->msgIdx = rsp.msgIdx; - ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, pTask->msgIdx, TMSG_INFO(taskMsg.msgType + 1)); (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index d8fda57791..4cad6a078b 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -462,3 +462,5 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + + From d8d6cacd0f71297461fb4ac42d49fc079cdbcb38 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 16:23:45 +0800 Subject: [PATCH 06/51] enh: improve getting table hash performance --- source/libs/catalog/inc/catalogInt.h | 18 ++- source/libs/catalog/src/ctgAsync.c | 205 +++++++++++++++++++++++++-- source/libs/catalog/src/ctgCache.c | 39 +++-- source/libs/catalog/src/ctgUtil.c | 136 +++++++++++++++++- 4 files changed, 360 insertions(+), 38 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a4e47e8053..e12bcfd18f 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -81,6 +81,7 @@ typedef enum { CTG_TASK_GET_USER, CTG_TASK_GET_SVR_VER, CTG_TASK_GET_TB_META_BATCH, + CTG_TASK_GET_TB_HASH_BATCH, } CTG_TASK_TYPE; typedef enum { @@ -114,6 +115,7 @@ typedef struct SCtgTbMetaCtx { typedef struct SCtgFetch { int32_t reqIdx; int32_t fetchIdx; + int32_t resIdx; int32_t flag; SCtgTbCacheInfo tbInfo; int32_t vgId; @@ -122,11 +124,10 @@ typedef struct SCtgFetch { typedef struct SCtgTbMetaBCtx { int32_t fetchNum; SArray* pNames; - SArray* pTbMetas; + SArray* pResList; SArray* pFetchs; } SCtgTbMetaBCtx; - typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; @@ -154,6 +155,14 @@ typedef struct SCtgTbHashCtx { SName* pName; } SCtgTbHashCtx; +typedef struct SCtgTbHashBCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pResList; + SArray* pFetchs; +} SCtgTbHashBCtx; + + typedef struct SCtgIndexCtx { char indexFName[TSDB_INDEX_FNAME_LEN]; } SCtgIndexCtx; @@ -507,7 +516,7 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_GET_TASK_MSGCTX(_task) ((CTG_TASK_GET_TB_META_BATCH == (_task)->type) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) +#define CTG_GET_TASK_MSGCTX(_task) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) @@ -608,7 +617,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -687,6 +696,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); +int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8dc32d306a..eed4a956b5 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -65,7 +65,7 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTbMetaBCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pTbMetas = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); @@ -74,7 +74,6 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } - int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { char *dbFName = (char*)param; SCtgTask task = {0}; @@ -178,6 +177,31 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_HASH_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashBCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbHashBCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTask task = {0}; @@ -542,10 +566,16 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const } #endif +#if 0 for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH, name, NULL)); } +#else + if (tbHashNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH_BATCH, pReq->pTableHash, NULL)); + } +#endif for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); @@ -656,6 +686,14 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbHashBRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableHash = pTask->res; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { @@ -1041,12 +1079,12 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * */ #if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pTbMetas); + TSWAP(pTask->res, ctx->pResList); } #else TSWAP(pTask->res, pOut->tbMeta); @@ -1060,11 +1098,11 @@ _return: #if CTG_BATCH_FETCH if (code) { - SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); pRes->code = code; pRes->pRes = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pTbMetas); + TSWAP(pTask->res, ctx->pResList); } } #endif @@ -1146,6 +1184,62 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTbHashBCtx* ctx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, true)); + + CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (code) { + SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); // TODO + + SMetaRes res = {0}; + int32_t num = taosArrayGetSize(ctx->pNames); + for (int32_t i = 0; i < num; ++i) { + SMetaRes *pRes = taosArrayGet(ctx->pResList, i); + pRes->code = code; + pRes->pRes = NULL; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + } + + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1377,23 +1471,25 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, (SCtgTbMetaBCtx*)pTask->taskCtx, (SArray**)&pTask->res)); - if (pTask->res) { + CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, pCtx, pCtx->pNames)); + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } - - SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); pTask->msgIdx = pFetch->fetchIdx; - CTG_ERR_RET(ctgAddMsgCtx(pTask->msgCtxs, 0, NULL, NULL)); CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); } @@ -1472,6 +1568,90 @@ _return: CTG_RET(code); } +int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbHashBCtx* pCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t dbNum = 1; + int32_t fIdx = 0; + int32_t tbNum = 0; + int32_t code = 0; + + for (int32_t i = 0; i < dbNum; ++i) { // TODO + SName* pName = taosArrayGet(pCtx->pNames, 0); + if (IS_SYS_DBNAME(pName->dbname)) { + strcpy(dbFName, pName->dbname); + } else { + tNameGetFullDbName(pName, dbFName); + } + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + + if (NULL != dbCache) { + CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, dbFName, false)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; + } else { + if (NULL == pCtx->pFetchs) { + pCtx->pFetchs = taosArrayInit(dbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.resIdx = tbNum; + + tbNum += taosArrayGetSize(pCtx->pNames); // TODO + + taosArrayPush(pCtx->pFetchs, &fetch); + taosArraySetSize(pCtx->pResList, tbNum); + } + } + + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + + pTask->msgIdx = pFetch->fetchIdx; + + SBuildUseDBInput input = {0}; + if (IS_SYS_DBNAME(pName->dbname)) { + strcpy(input.db, pName->dbname); + } else { + tNameGetFullDbName(pName, input.db); + } + + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + } + + pTask->msgIdx = 0; + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + + return code; +} + + int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1724,6 +1904,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, + {ctgInitGetTbHashBTask, ctgLaunchGetTbHashBTask, ctgHandleGetTbHashBRsp, ctgDumpTbHashBRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6cec690bad..1a28ff3e25 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2151,7 +2151,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet } #if 0 -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pResList) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); int32_t fIdx = 0; @@ -2196,21 +2196,21 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe taosArrayPush(ctx->pFetchs, &fetch); } - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); } if (NULL == ctx->pFetchs) { - TSWAP(*pTbMetas, ctx->pTbMetas); + TSWAP(*pResList, ctx->pResList); } return TSDB_CODE_SUCCESS; } #endif -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { - int32_t tbNum = taosArrayGetSize(ctx->pNames); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList) { + int32_t tbNum = taosArrayGetSize(pList); int32_t fIdx = 0; - SName* pName = taosArrayGet(ctx->pNames, 0); + SName* pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; @@ -2241,14 +2241,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); } return TSDB_CODE_SUCCESS; } for (int32_t i = 0; i < tbNum; ++i) { - SName* pName = taosArrayGet(ctx->pNames, i); + SName* pName = taosArrayGet(pList, i); SMetaRes res = {0}; pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); @@ -2264,7 +2264,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2282,7 +2282,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2315,7 +2315,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2324,6 +2324,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { cloneTableMeta(lastTableMeta, &pTableMeta); + memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); if (pCache) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); @@ -2333,7 +2334,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2367,7 +2368,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); continue; @@ -2388,7 +2389,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); continue; @@ -2414,7 +2415,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); @@ -2440,7 +2441,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); @@ -2462,16 +2463,12 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); lastSuid = pTableMeta->suid; lastTableMeta = pTableMeta; } - if (NULL == ctx->pFetchs) { - TSWAP(*pTbMetas, ctx->pTbMetas); - } - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 9f0d0f71d5..c841255b42 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -90,6 +90,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get svr ver]"; case CTG_TASK_GET_TB_META_BATCH: return "[bget table meta]"; + case CTG_TASK_GET_TB_HASH_BATCH: + return "[bget table hash]"; default: return "unknown"; } @@ -471,6 +473,16 @@ void ctgFreeBatchMeta(void* meta) { taosMemoryFreeClear(pRes->pRes); } +void ctgFreeBatchHash(void* hash) { + if (NULL == hash) { + return; + } + + SMetaRes* pRes = (SMetaRes*)hash; + taosMemoryFreeClear(pRes->pRes); +} + + void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -520,6 +532,15 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { *pRes = NULL; // no need to free it break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchHash(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } default: qError("invalid task type %d", type); break; @@ -579,6 +600,11 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { *pRes = NULL; break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchHash); + *pRes = NULL; + break; + } default: qError("invalid task type %d", type); break; @@ -610,7 +636,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { } case CTG_TASK_GET_TB_META_BATCH: { SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - taosArrayDestroyEx(taskCtx->pTbMetas, ctgFreeBatchMeta); + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchMeta); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -629,6 +655,17 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + SCtgTbHashBCtx* taskCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchHash); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_INDEX: { SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -837,6 +874,103 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } +int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update) { + int32_t code = 0; + SMetaRes res = {0}; + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; + + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + + int32_t tbNum = taosArrayGetSize(pCtx->pNames); + + if (1 == vgNum) { + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + for (int32_t i = 0; i < tbNum; ++i) { + vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == vgInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *vgInfo = *(SVgroupInfo*)pIter; + + ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = vgInfo; + } else { + res.pRes = vgInfo; + taosArrayPush(pCtx->pResList, &res); + } + } + + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFullName, "%s.", dbFName); + int32_t offset = strlen(tbFullName); + SName* pName = NULL; + int32_t tbNameLen = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + pName = taosArrayGet(pCtx->pNames, i); + + tbNameLen = offset + strlen(pName->tname); + strcpy(tbFullName + offset, pName->tname); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); + + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pNewVg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *pNewVg = *vgInfo; + + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = pNewVg; + } else { + res.pRes = pNewVg; + taosArrayPush(pCtx->pResList, &res); + } + } + + CTG_RET(code); +} + + int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; From 8fc86a31d0011761ddc7e365bda36ab62e2d63a3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 4 Aug 2022 20:53:43 +0800 Subject: [PATCH 07/51] enh: asynchronous catalog interface optimization --- include/libs/catalog/catalog.h | 9 +- source/libs/parser/inc/parUtil.h | 7 +- source/libs/parser/src/parUtil.c | 102 +++++++++++++++--- source/libs/parser/src/parser.c | 8 +- .../libs/parser/test/mockCatalogService.cpp | 43 +++++--- source/libs/parser/test/mockCatalogService.h | 1 + source/libs/parser/test/parInsertTest.cpp | 13 ++- source/libs/parser/test/parTestUtil.cpp | 11 +- 8 files changed, 151 insertions(+), 43 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 1d1849f24a..ee566d759a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -58,12 +58,17 @@ typedef struct SDbInfo { int64_t dbId; } SDbInfo; +typedef struct STablesReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SArray* pTables; +} STablesReq; + typedef struct SCatalogReq { SArray* pDbVgroup; // element is db full name SArray* pDbCfg; // element is db full name SArray* pDbInfo; // element is db full name - SArray* pTableMeta; // element is SNAME - SArray* pTableHash; // element is SNAME + SArray* pTableMeta; // element is STablesReq + SArray* pTableHash; // element is STablesReq SArray* pUdf; // element is udf name SArray* pIndex; // element is index name SArray* pUser; // element is SUserAuthInfo diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 896e2bc239..24f34637bf 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -38,6 +38,11 @@ typedef struct SMsgBuf { char* buf; } SMsgBuf; +typedef struct SParseTablesMetaReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SHashObj* pTables; +} SParseTablesMetaReq; + typedef struct SParseMetaCache { SHashObj* pTableMeta; // key is tbFName, element is STableMeta* SHashObj* pDbVgroup; // key is dbFName, element is SArray* @@ -94,7 +99,7 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); -void destoryParseMetaCache(SParseMetaCache* pMetaCache); +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e51800aece..ae5a281aab 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -474,6 +474,24 @@ static int32_t buildDbReq(SHashObj* pDbsHash, SArray** pDbs) { return TSDB_CODE_SUCCESS; } +static int32_t buildTableReqFromDb(SHashObj* pDbsHash, SArray** pDbs) { + if (NULL != pDbsHash) { + *pDbs = taosArrayInit(taosHashGetSize(pDbsHash), sizeof(STablesReq)); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SParseTablesMetaReq* p = taosHashIterate(pDbsHash, NULL); + while (NULL != p) { + STablesReq req = {0}; + strcpy(req.dbFName, p->dbFName); + buildTableReq(p->pTables, &req.pTables); + taosArrayPush(*pDbs, &req); + p = taosHashIterate(pDbsHash, p); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) { if (NULL != pUserAuthHash) { *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo)); @@ -513,12 +531,12 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) { } int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - int32_t code = buildTableReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); + int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = buildTableReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); + code = buildTableReqFromDb(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); } if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg); @@ -587,6 +605,24 @@ static int32_t putDbDataToCache(const SArray* pDbReq, const SArray* pDbData, SHa return TSDB_CODE_SUCCESS; } +static int32_t putDbTableDataToCache(const SArray* pDbReq, const SArray* pTableData, SHashObj** pTable) { + int32_t ndbs = taosArrayGetSize(pDbReq); + int32_t tableNo = 0; + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = taosArrayGet(pDbReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(taosArrayGet(pReq->pTables, j), fullName); + if (TSDB_CODE_SUCCESS != putMetaDataToHash(fullName, strlen(fullName), pTableData, tableNo, pTable)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ++tableNo; + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUserAuthData, SHashObj** pUserAuth) { int32_t nvgs = taosArrayGetSize(pUserAuthReq); for (int32_t i = 0; i < nvgs; ++i) { @@ -612,12 +648,12 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas } int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { - int32_t code = putTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); + int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = putTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); + code = putDbTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); } if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, &pMetaCache->pDbCfg); @@ -657,14 +693,38 @@ static int32_t reserveTableReqInCache(int32_t acctId, const char* pDb, const cha return reserveTableReqInCacheImpl(fullName, len, pTables); } +static int32_t reserveTableReqInDbCacheImpl(int32_t acctId, const char* pDb, const char* pTable, SHashObj* pDbs) { + SParseTablesMetaReq req = {0}; + int32_t len = snprintf(req.dbFName, sizeof(req.dbFName), "%d.%s", acctId, pDb); + int32_t code = reserveTableReqInCache(acctId, pDb, pTable, &req.pTables); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pDbs, req.dbFName, len, &req, sizeof(SParseTablesMetaReq)); + } + return code; +} + +static int32_t reserveTableReqInDbCache(int32_t acctId, const char* pDb, const char* pTable, SHashObj** pDbs) { + if (NULL == *pDbs) { + *pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + char fullName[TSDB_DB_FNAME_LEN]; + int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); + SParseTablesMetaReq* pReq = taosHashGet(*pDbs, fullName, len); + if (NULL == pReq) { + return reserveTableReqInDbCacheImpl(acctId, pDb, pTable, *pDbs); + } + return reserveTableReqInCache(acctId, pDb, pTable, &pReq->pTables); +} + int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); } int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableMeta); } int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { @@ -711,13 +771,11 @@ int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, } int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); } int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableVgroup); } int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) { @@ -919,10 +977,24 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { return TSDB_CODE_SUCCESS; } -void destoryParseMetaCache(SParseMetaCache* pMetaCache) { - taosHashCleanup(pMetaCache->pTableMeta); +void destoryParseTablesMetaReqHash(SHashObj* pHash) { + SParseTablesMetaReq* p = taosHashIterate(pHash, NULL); + while (NULL != p) { + taosHashCleanup(p->pTables); + p = taosHashIterate(pHash, p); + } + taosHashCleanup(pHash); +} + +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + if (request) { + destoryParseTablesMetaReqHash(pMetaCache->pTableMeta); + destoryParseTablesMetaReqHash(pMetaCache->pTableVgroup); + } else { + taosHashCleanup(pMetaCache->pTableMeta); + taosHashCleanup(pMetaCache->pTableVgroup); + } taosHashCleanup(pMetaCache->pDbVgroup); - taosHashCleanup(pMetaCache->pTableVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); taosHashCleanup(pMetaCache->pUserAuth); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 53ee717af2..34cd783ace 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -85,13 +85,13 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { taosMemoryFreeClear(pVal->datum.p); } - + if (pParam->is_null && 1 == *(pParam->is_null)) { pVal->node.resType.type = TSDB_DATA_TYPE_NULL; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } - + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; @@ -187,7 +187,7 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(&metaCache, pCatalogReq); } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, true); terrno = code; return code; } @@ -203,7 +203,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, false); terrno = code; return code; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4158453110..6717a1fdf1 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -472,12 +472,16 @@ class MockCatalogServiceImpl { int32_t getAllTableMeta(SArray* pTableMetaReq, SArray** pTableMetaData) const { if (NULL != pTableMetaReq) { - int32_t ntables = taosArrayGetSize(pTableMetaReq); - *pTableMetaData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), (STableMeta**)&res.pRes); - taosArrayPush(*pTableMetaData, &res); + int32_t ndbs = taosArrayGetSize(pTableMetaReq); + *pTableMetaData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableMetaReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.code = catalogGetTableMeta((const SName*)taosArrayGet(pReq->pTables, j), (STableMeta**)&res.pRes); + taosArrayPush(*pTableMetaData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -485,13 +489,17 @@ class MockCatalogServiceImpl { int32_t getAllTableVgroup(SArray* pTableVgroupReq, SArray** pTableVgroupData) const { if (NULL != pTableVgroupReq) { - int32_t ntables = taosArrayGetSize(pTableVgroupReq); - *pTableVgroupData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), (SVgroupInfo*)res.pRes); - taosArrayPush(*pTableVgroupData, &res); + int32_t ndbs = taosArrayGetSize(pTableVgroupReq); + *pTableVgroupData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableVgroupReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pReq->pTables, j), (SVgroupInfo*)res.pRes); + taosArrayPush(*pTableVgroupData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -677,12 +685,17 @@ int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SM return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } +void MockCatalogService::destoryTablesReq(void* p) { + STablesReq* pRes = (STablesReq*)p; + taosArrayDestroy(pRes->pTables); +} + void MockCatalogService::destoryCatalogReq(SCatalogReq* pReq) { taosArrayDestroy(pReq->pDbVgroup); taosArrayDestroy(pReq->pDbCfg); taosArrayDestroy(pReq->pDbInfo); - taosArrayDestroy(pReq->pTableMeta); - taosArrayDestroy(pReq->pTableHash); + taosArrayDestroyEx(pReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pReq->pTableHash, destoryTablesReq); taosArrayDestroy(pReq->pUdf); taosArrayDestroy(pReq->pIndex); taosArrayDestroy(pReq->pUser); diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index d76a6abca8..c0330692ee 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -50,6 +50,7 @@ struct MockTableMeta { class MockCatalogServiceImpl; class MockCatalogService { public: + static void destoryTablesReq(void* p); static void destoryCatalogReq(SCatalogReq* pReq); static void destoryMetaRes(void* p); static void destoryMetaArrayRes(void* p); diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 22a1be2579..6014a97efa 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include + #include #include "mockCatalogService.h" @@ -20,6 +22,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace { @@ -63,7 +66,9 @@ class InsertTest : public Test { int32_t runAsync() { cxt_.async = true; - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), std::bind(_destoryParseMetaCache, _1, cref(request))); code_ = parseInsertSyntax(&cxt_, &res_, metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "parseInsertSyntax code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -81,6 +86,8 @@ class InsertTest : public Test { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); g_mockCatalogService->catalogGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; code_ = putMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "putMetaDataToCache code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -144,8 +151,8 @@ class InsertTest : public Test { static const int max_err_len = 1024; static const int max_sql_len = 1024 * 1024; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 235cc487fb..3fe4b533e4 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -24,6 +24,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace ParserTest { @@ -118,8 +119,8 @@ class ParserTestBaseImpl { TEST_INTERFACE_ASYNC_API }; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } @@ -340,7 +341,9 @@ class ParserTestBaseImpl { doParse(&cxt, query.get()); SQuery* pQuery = *(query.get()); - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); doCollectMetaKey(&cxt, pQuery, metaCache.get()); unique_ptr catalogReq(new SCatalogReq(), @@ -353,6 +356,8 @@ class ParserTestBaseImpl { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); doGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); doAuthenticate(&cxt, pQuery, metaCache.get()); From 16ba0c658e9ca25847357ee4279fc219931fbeb9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 15:36:05 +0800 Subject: [PATCH 08/51] enh: refactor getting table meta/hash --- source/client/src/clientImpl.c | 46 +++++++-- source/libs/catalog/inc/catalogInt.h | 23 +++-- source/libs/catalog/src/ctgAsync.c | 119 +++++++++++----------- source/libs/catalog/src/ctgCache.c | 144 +++++++-------------------- source/libs/catalog/src/ctgRemote.c | 8 +- source/libs/catalog/src/ctgUtil.c | 43 +++++++- 6 files changed, 189 insertions(+), 194 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..20637b7e6e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1922,7 +1922,7 @@ _OVER: return code; } -int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, +int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) { SName name; @@ -1957,20 +1957,33 @@ int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, i return -1; } - taosArrayPush(pList, &name); + char dbFName[TSDB_DB_FNAME_LEN]; + sprintf(dbFName, "%d.%.*s", acctId, dbLen, dbName); + + STablesReq* pDb = taosHashGet(pHash, dbFName, strlen(dbFName)); + if (pDb) { + taosArrayPush(pDb->pTables, &name); + } else { + STablesReq db; + db.pTables = taosArrayInit(20, sizeof(SName)); + strcpy(db.dbFName, dbFName); + taosArrayPush(db.pTables, &name); + taosHashPut(pHash, dbFName, strlen(dbFName), &db, sizeof(db)); + } return TSDB_CODE_SUCCESS; } int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq) { - *pReq = taosArrayInit(10, sizeof(SName)); - if (NULL == *pReq) { + SHashObj* pHash = taosHashInit(3, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } bool inEscape = false; int32_t code = 0; + void *pIter = NULL; int32_t vIdx = 0; int32_t vPos[2]; @@ -1985,7 +1998,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2035,7 +2048,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2067,14 +2080,31 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, goto _return; } + int32_t dbNum = taosHashGetSize(pHash); + *pReq = taosArrayInit(dbNum, sizeof(STablesReq)); + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayPush(*pReq, pDb); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); + return TSDB_CODE_SUCCESS; _return: terrno = TSDB_CODE_TSC_INVALID_OPERATION; - taosArrayDestroy(*pReq); - *pReq = NULL; + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayDestroy(pDb->pTables); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); return terrno; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index e12bcfd18f..d4b8a39197 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -32,6 +32,7 @@ extern "C" { #define CTG_DEFAULT_RENT_SLOT_SIZE 10 #define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_DEFAULT_BATCH_NUM 64 +#define CTG_DEFAULT_FETCH_NUM 8 #define CTG_RENT_SLOT_SECOND 1.5 @@ -113,7 +114,8 @@ typedef struct SCtgTbMetaCtx { } SCtgTbMetaCtx; typedef struct SCtgFetch { - int32_t reqIdx; + int32_t dbIdx; + int32_t tbIdx; int32_t fetchIdx; int32_t resIdx; int32_t flag; @@ -121,12 +123,12 @@ typedef struct SCtgFetch { int32_t vgId; } SCtgFetch; -typedef struct SCtgTbMetaBCtx { +typedef struct SCtgTbMetasCtx { int32_t fetchNum; SArray* pNames; SArray* pResList; SArray* pFetchs; -} SCtgTbMetaBCtx; +} SCtgTbMetasCtx; typedef struct SCtgTbIndexCtx { SName* pName; @@ -155,12 +157,12 @@ typedef struct SCtgTbHashCtx { SName* pName; } SCtgTbHashCtx; -typedef struct SCtgTbHashBCtx { +typedef struct SCtgTbHashsCtx { int32_t fetchNum; SArray* pNames; SArray* pResList; SArray* pFetchs; -} SCtgTbHashBCtx; +} SCtgTbHashsCtx; typedef struct SCtgIndexCtx { @@ -617,7 +619,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList); +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -696,7 +698,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); -int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update); +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); @@ -708,6 +710,8 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); +int32_t ctgGetTablesReqNum(SArray *pList); +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); void ctgFreeSTableIndex(void *info); void ctgClearSubTaskRes(SCtgSubRes *pRes); @@ -717,6 +721,11 @@ void ctgFreeTbCacheImpl(SCtgTbCache *pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup); +FORCE_INLINE SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); + return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); +} + extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index eed4a956b5..616a9cafe7 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -50,7 +50,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { +int32_t ctgInitGetTbMetasTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SName *name = (SName*)param; SCtgTask task = {0}; @@ -58,18 +58,19 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaBCtx)); + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetasCtx)); if (NULL == task.taskCtx) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCtgTbMetaBCtx* ctx = task.taskCtx; + SCtgTbMetasCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; } @@ -177,7 +178,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { +int32_t ctgInitGetTbHashsTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SName *name = (SName*)param; SCtgTask task = {0}; @@ -185,18 +186,19 @@ int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashBCtx)); + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashsCtx)); if (NULL == task.taskCtx) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCtgTbHashBCtx* ctx = task.taskCtx; + SCtgTbHashsCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; } @@ -477,9 +479,9 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param) { int32_t code = 0; - int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); + int32_t tbMetaNum = (int32_t)ctgGetTablesReqNum(pReq->pTableMeta); int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); - int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); + int32_t tbHashNum = (int32_t)ctgGetTablesReqNum(pReq->pTableHash); int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; int32_t dnodeNum = pReq->dNodeRequired ? 1 : 0; @@ -647,7 +649,7 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgDumpTbMetaBRes(SCtgTask* pTask) { +int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableMeta = pTask->res; @@ -686,7 +688,7 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgDumpTbHashBRes(SCtgTask* pTask) { +int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableHash = pTask->res; @@ -929,9 +931,9 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); #if CTG_BATCH_FETCH - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(ctx->pNames, pFetch); int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; #else @@ -1079,7 +1081,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * */ #if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1098,7 +1100,7 @@ _return: #if CTG_BATCH_FETCH if (code) { - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = code; pRes->pRes = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1184,9 +1186,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - SCtgTbHashBCtx* ctx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); @@ -1197,7 +1199,8 @@ int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf case TDMT_MND_USE_DB: { SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; - CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, true)); + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1217,12 +1220,10 @@ int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf _return: if (code) { - SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); // TODO - - SMetaRes res = {0}; - int32_t num = taosArrayGetSize(ctx->pNames); + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + int32_t num = taosArrayGetSize(pReq->pTables); for (int32_t i = 0; i < num; ++i) { - SMetaRes *pRes = taosArrayGet(ctx->pResList, i); + SMetaRes *pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); pRes->code = code; pRes->pRes = NULL; } @@ -1468,12 +1469,21 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { +int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, pCtx, pCtx->pNames)); + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); + baseResIdx += taosArrayGetSize(pReq->pTables); + } + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -1487,7 +1497,7 @@ int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); pTask->msgIdx = pFetch->fetchIdx; CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); @@ -1568,46 +1578,33 @@ _return: CTG_RET(code); } -int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { +int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgTbHashBCtx* pCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* pCtx = (SCtgTbHashsCtx*)pTask->taskCtx; SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - int32_t dbNum = 1; - int32_t fIdx = 0; - int32_t tbNum = 0; + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; int32_t code = 0; - for (int32_t i = 0; i < dbNum; ++i) { // TODO - SName* pName = taosArrayGet(pCtx->pNames, 0); - if (IS_SYS_DBNAME(pName->dbname)) { - strcpy(dbFName, pName->dbname); - } else { - tNameGetFullDbName(pName, dbFName); - } + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, dbFName, false)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; - } else { - if (NULL == pCtx->pFetchs) { - pCtx->pFetchs = taosArrayInit(dbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.resIdx = tbNum; - tbNum += taosArrayGetSize(pCtx->pNames); // TODO - - taosArrayPush(pCtx->pFetchs, &fetch); - taosArraySetSize(pCtx->pResList, tbNum); + baseResIdx += taosArrayGetSize(pReq->pTables); + } else { + ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + + baseResIdx += taosArrayGetSize(pReq->pTables); + taosArraySetSize(pCtx->pResList, baseResIdx); } } @@ -1624,7 +1621,7 @@ int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); pTask->msgIdx = pFetch->fetchIdx; @@ -1903,8 +1900,8 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, - {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, - {ctgInitGetTbHashBTask, ctgLaunchGetTbHashBTask, ctgHandleGetTbHashBRsp, ctgDumpTbHashBRes, NULL, NULL}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetasRes, NULL, NULL}, + {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1a28ff3e25..930361419e 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2151,7 +2151,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet } #if 0 -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pResList) { +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, SArray** pResList) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); int32_t fIdx = 0; @@ -2189,7 +2189,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } SCtgFetch fetch = {0}; - fetch.reqIdx = i; + fetch.tbIdx = i; fetch.fetchIdx = fIdx++; fetch.flag = nctx.flag; @@ -2207,9 +2207,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } #endif -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList) { +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList) { int32_t tbNum = taosArrayGetSize(pList); - int32_t fIdx = 0; SName* pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; @@ -2230,18 +2229,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - SMetaRes res = {0}; - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); } return TSDB_CODE_SUCCESS; @@ -2249,22 +2238,12 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe for (int32_t i = 0; i < tbNum; ++i) { SName* pName = taosArrayGet(pList, i); - SMetaRes res = {0}; pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); continue; } @@ -2272,17 +2251,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); continue; } @@ -2296,21 +2266,20 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe nctx.tbInfo.suid = tbMeta->suid; nctx.tbInfo.tbType = tbMeta->tableType; + SMetaRes res = {0}; STableMeta* pTableMeta = NULL; if (tbMeta->tableType != TSDB_CHILD_TABLE) { int32_t metaSize = CTG_META_SIZE(tbMeta); pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pTableMeta, tbMeta, metaSize); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); @@ -2326,10 +2295,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe cloneTableMeta(lastTableMeta, &pTableMeta); memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); @@ -2342,15 +2309,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe int32_t metaSize = sizeof(SCTableMeta); pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pTableMeta, tbMeta, metaSize); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", pName->tname, nctx.tbInfo.tbType, dbFName); @@ -2358,17 +2324,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); continue; @@ -2379,17 +2336,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); continue; @@ -2400,22 +2348,11 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); @@ -2424,24 +2361,13 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe STableMeta* stbMeta = pCache->pMeta; if (stbMeta->suid != nctx.tbInfo.suid) { - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); @@ -2451,16 +2377,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe metaSize = CTG_META_SIZE(stbMeta); pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); if (NULL == pTableMeta) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); res.pRes = pTableMeta; taosArrayPush(ctx->pResList, &res); @@ -2469,6 +2393,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe lastTableMeta = pTableMeta; } + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index a8d5a7ae95..e464f0f36c 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -466,9 +466,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -512,9 +512,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index c841255b42..d21eacf756 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -635,7 +635,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { break; } case CTG_TASK_GET_TB_META_BATCH: { - SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* taskCtx = (SCtgTbMetasCtx*)pTask->taskCtx; taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchMeta); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -656,7 +656,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { break; } case CTG_TASK_GET_TB_HASH_BATCH: { - SCtgTbHashBCtx* taskCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* taskCtx = (SCtgTbHashsCtx*)pTask->taskCtx; taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchHash); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -874,7 +874,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update) { +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SMetaRes res = {0}; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); @@ -888,7 +888,7 @@ int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - int32_t tbNum = taosArrayGetSize(pCtx->pNames); + int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { void *pIter = taosHashIterate(dbInfo->vgHash, NULL); @@ -923,7 +923,7 @@ int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d int32_t tbNameLen = 0; for (int32_t i = 0; i < tbNum; ++i) { - pName = taosArrayGet(pCtx->pNames, i); + pName = taosArrayGet(pNames, i); tbNameLen = offset + strlen(pName->tname); strcpy(tbFullName + offset, pName->tname); @@ -1112,4 +1112,37 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, cha return TSDB_CODE_SUCCESS; } +int32_t ctgGetTablesReqNum(SArray *pList) { + if (NULL == pList) { + return 0; + } + + int32_t total = 0; + int32_t n = taosArrayGetSize(pList); + for (int32_t i = 0; i < n; ++i) { + STablesReq *pReq = taosArrayGet(pList, i); + total += taosArrayGetSize(pReq->pTables); + } + + return total; +} + +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag) { + if (NULL == (*pFetchs)) { + *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.dbIdx = dbIdx; + fetch.tbIdx = tbIdx; + fetch.fetchIdx = (*fetchIdx)++; + fetch.resIdx = resIdx; + fetch.flag = flag; + + taosArrayPush(*pFetchs, &fetch); + + return TSDB_CODE_SUCCESS; +} + + From 992912246f734edcab444ff7815599da5e6f479d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 15:58:41 +0800 Subject: [PATCH 09/51] fix: fix get db name issue --- source/libs/catalog/src/ctgAsync.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 616a9cafe7..37994aa97f 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1621,16 +1621,12 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pTask->msgIdx = pFetch->fetchIdx; SBuildUseDBInput input = {0}; - if (IS_SYS_DBNAME(pName->dbname)) { - strcpy(input.db, pName->dbname); - } else { - tNameGetFullDbName(pName, input.db); - } + strcpy(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; From ac63ee40c95c95e1f2a458e0bd71ff8196eb219f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 17:44:11 +0800 Subject: [PATCH 10/51] fix: fix get tb meta for tb cfg case --- source/libs/catalog/inc/catalogInt.h | 6 +- source/libs/catalog/src/ctgAsync.c | 194 ++++++++++++++++++++++++--- source/libs/catalog/src/ctgUtil.c | 4 + 3 files changed, 178 insertions(+), 26 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d4b8a39197..c452f61b75 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -720,11 +720,7 @@ void ctgClearHandle(SCatalog* pCtg); void ctgFreeTbCacheImpl(SCtgTbCache *pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup); - -FORCE_INLINE SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { - STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); - return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); -} +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 37994aa97f..bbca85ed14 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -930,18 +930,10 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); -#if CTG_BATCH_FETCH - SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); - int32_t flag = pFetch->flag; - int32_t* vgId = &pFetch->vgId; -#else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SName* pName = ctx->pName; int32_t flag = ctx->flag; int32_t* vgId = &ctx->vgId; -#endif CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1080,17 +1072,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * } */ -#if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = 0; - pRes->pRes = pOut->tbMeta; - pOut->tbMeta = NULL; - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - } -#else TSWAP(pTask->res, pOut->tbMeta); -#endif _return: @@ -1098,7 +1080,177 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } -#if CTG_BATCH_FETCH + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + } + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + if (CTG_FLAG_IS_STB(flag)) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL != dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + } + + return TSDB_CODE_SUCCESS; + } + + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; + TSWAP(pLastOut->tbMeta, pOut->tbMeta); + } + + break; + } + case TDMT_VND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (CTG_FLAG_IS_STB(flag)) { + break; + } + + if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); + + taosMemoryFreeClear(pOut->tbMeta); + + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + } else if (CTG_IS_META_BOTH(pOut->metaType)) { + int32_t exist = 0; + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + exist = 1; + } + } + + if (0 == exist) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + } + } + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + ctgUpdateTbMetaToCache(pCtg, pOut, false); + + if (CTG_IS_META_BOTH(pOut->metaType)) { + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (NULL == pOut->tbMeta) { + ctgDebug("stb no longer exist, stbName:%s", stbName.tname); + CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; + } + + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } +*/ + + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + pRes->code = 0; + pRes->pRes = pOut->tbMeta; + pOut->tbMeta = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = code; @@ -1107,7 +1259,6 @@ _return: TSWAP(pTask->res, ctx->pResList); } } -#endif if (pTask->res) { ctgHandleTaskEnd(pTask, code); @@ -1116,6 +1267,7 @@ _return: CTG_RET(code); } + int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; @@ -1896,7 +2048,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, - {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetasRes, NULL, NULL}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, }; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index d21eacf756..de9ea7fb5a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1144,5 +1144,9 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fet return TSDB_CODE_SUCCESS; } +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); + return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); +} From 3ec64ad253deff2875027e155a37616b16b42fda Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 09:26:19 +0800 Subject: [PATCH 11/51] fix: fix task timeout error --- source/common/src/systable.c | 6 +++--- source/libs/scheduler/src/schStatus.c | 2 +- source/libs/scheduler/src/schTask.c | 1 + source/libs/scheduler/src/scheduler.c | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 29d7b557c8..675d3bbfdd 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -308,9 +308,9 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, + {.name = "kill_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "query_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index a4fa4f2839..64cda573f1 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -64,7 +64,7 @@ _return: int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { SSchJob *pJob = schAcquireJob(jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ecd0253e1c..520ddee25e 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -536,6 +536,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { pTask->maxExecTimes++; + pTask->maxRetryTimes++; if (pTask->timeoutUsec < SCH_MAX_TASK_TIMEOUT_USEC) { pTask->timeoutUsec *= 2; if (pTask->timeoutUsec > SCH_MAX_TASK_TIMEOUT_USEC) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3a15523040..a37cd4fd9e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -150,7 +150,7 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { SSchJob *pJob = schAcquireJob(*jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } From b2c8d2161fc4fea47eeb390da968defeaa468358 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 6 Aug 2022 10:47:15 +0800 Subject: [PATCH 12/51] fix: all null value group with all null out --- source/libs/function/src/builtinsimpl.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e78a78e7d8..7fd48519a3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3625,6 +3625,11 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // todo assign the tag value and the corresponding row data int32_t currentRow = pBlock->info.rows; + if (pEntryInfo->numOfRes <= 0) { + colDataAppendNULL(pCol, currentRow); + setNullSelectivityValue(pCtx, pBlock, currentRow); + return pEntryInfo->numOfRes; + } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; if (type == TSDB_DATA_TYPE_FLOAT) { @@ -4957,6 +4962,11 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; + if (pInfo->numSampled == 0) { + colDataAppendNULL(pCol, currentRow); + setNullSelectivityValue(pCtx, pBlock, currentRow); + return pInfo->numSampled; + } for (int32_t i = 0; i < pInfo->numSampled; ++i) { colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); From d5d873c7f89c7744eef055b1bb290fdfe7c92959 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 11:32:29 +0800 Subject: [PATCH 13/51] fix(query): fix interp fill(prev) issue --- source/libs/executor/src/timewindowoperator.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8a0564c129..4cbe9e467f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2087,7 +2087,7 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock) { + int32_t rowIndex, SSDataBlock* pResBlock, bool isPrevRowSet) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2163,6 +2163,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } case TSDB_FILL_PREV: { + if (!isPrevRowSet) { + break; + } + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; @@ -2170,6 +2174,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } case TSDB_FILL_NEXT: { + if (!isPrevRowSet) { + break; + } + char* p = colDataGetData(pSrc, rowIndex); colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); pResBlock->info.rows += 1; @@ -2235,6 +2243,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); //int32_t numOfRows = 0; + bool isPrevRowSet = false; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2269,6 +2278,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2285,7 +2295,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2302,10 +2312,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } else { // it is the last row of current block doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; } } else { // ts > pSliceInfo->current while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2329,6 +2340,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2347,7 +2359,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { //check if need to interpolate after ts range while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From 580b2e587b65833b4bb5e37e4a09fa281b5a6086 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 6 Aug 2022 14:29:12 +0800 Subject: [PATCH 14/51] fix: set selectivity value for the null group --- source/libs/function/src/builtinsimpl.c | 43 ++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7fd48519a3..e3e98a6895 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -64,6 +64,9 @@ typedef struct SMinmaxResInfo { bool assign; // assign the first value or not int64_t v; STuplePos tuplePos; + + STuplePos nullTuplePos; + bool nullTupleSaved; } SMinmaxResInfo; typedef struct STopBotResItem { @@ -75,6 +78,10 @@ typedef struct STopBotResItem { typedef struct STopBotRes { int32_t maxSize; int16_t type; + + STuplePos nullTuplePos; + bool nullTupleSaved; + STopBotResItem* pItems; } STopBotRes; @@ -221,6 +228,10 @@ typedef struct SSampleInfo { int32_t numSampled; uint8_t colType; int16_t colBytes; + + STuplePos nullTuplePos; + bool nullTupleSaved; + char* data; STuplePos* tuplePos; } SSampleInfo; @@ -1134,6 +1145,9 @@ bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) SMinmaxResInfo* buf = GET_ROWCELL_INTERBUF(pResultInfo); buf->assign = false; buf->tuplePos.pageId = -1; + + buf->nullTupleSaved = false; + buf->nullTuplePos.pageId = -1; return true; } @@ -1575,6 +1589,10 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } _min_max_over: + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved ) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pBuf->nullTuplePos); + pBuf->nullTupleSaved = true; + } return numOfElems; } @@ -1615,7 +1633,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pEntryInfo->numOfRes > 0) { setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); } else { - setNullSelectivityValue(pCtx, pBlock, currentRow); + setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); } return pEntryInfo->numOfRes; @@ -3366,6 +3384,8 @@ bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pRes->maxSize = pCtx->param[1].param.i; + pRes->nullTupleSaved = false; + pRes->nullTuplePos.pageId = -1; return true; } @@ -3403,6 +3423,10 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + pRes->nullTupleSaved = true; + } return TSDB_CODE_SUCCESS; } @@ -3427,6 +3451,11 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + pRes->nullTupleSaved = true; + } + return TSDB_CODE_SUCCESS; } @@ -3627,7 +3656,7 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { colDataAppendNULL(pCol, currentRow); - setNullSelectivityValue(pCtx, pBlock, currentRow); + setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); return pEntryInfo->numOfRes; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { @@ -4902,7 +4931,8 @@ bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) pInfo->numSampled = 0; pInfo->colType = pCtx->resDataInfo.type; pInfo->colBytes = pCtx->resDataInfo.bytes; - + pInfo->nullTuplePos.pageId = -1; + pInfo->nullTupleSaved = false; pInfo->data = (char*)pInfo + sizeof(SSampleInfo); pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); @@ -4948,6 +4978,11 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { doReservoirSample(pCtx, pInfo, data, i); } + if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + pInfo->nullTupleSaved = true; + } + SET_VAL(pResInfo, pInfo->numSampled, pInfo->numSampled); return TSDB_CODE_SUCCESS; } @@ -4964,7 +4999,7 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { colDataAppendNULL(pCol, currentRow); - setNullSelectivityValue(pCtx, pBlock, currentRow); + setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); return pInfo->numSampled; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { From 70f396799397ee754316b12c605924444fb4ec70 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 14:42:48 +0800 Subject: [PATCH 15/51] fix(query): fix interp + fill(prev) result error --- source/libs/executor/src/timewindowoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4cbe9e467f..1eee2c197e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2291,6 +2291,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { + //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; + if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { From d639c87898bed30826d7c853a5a76e1340be8b3e Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 6 Aug 2022 14:51:40 +0800 Subject: [PATCH 16/51] test: temp --- tests/system-test/1-insert/performanceInsert.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/1-insert/performanceInsert.json b/tests/system-test/1-insert/performanceInsert.json index de410c30f2..8c995346e5 100644 --- a/tests/system-test/1-insert/performanceInsert.json +++ b/tests/system-test/1-insert/performanceInsert.json @@ -1,7 +1,7 @@ { "filetype": "insert", "cfgdir": "/etc/taos/", - "host": "test216", + "host": "localhost", "port": 6030, "user": "root", "password": "taosdata", @@ -10,14 +10,14 @@ "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, - "interlace_rows": 1000, + "interlace_rows": 0, "num_of_records_per_req": 100000, "databases": [ { "dbinfo": { "name": "db", "drop": "yes", - "vgroups": 24 + "vgroups": 8 }, "super_tables": [ { @@ -29,8 +29,8 @@ "batch_create_tbl_num": 50000, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 5, - "interlace_rows": 100000, + "insert_rows": 1000, + "interlace_rows": 0, "insert_interval": 0, "max_sql_len": 10000000, "disorder_ratio": 0, From f09e48ca0b9928ba3bd9a0b90ee3fdf22ce3434c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 6 Aug 2022 15:28:43 +0800 Subject: [PATCH 17/51] ci: map taosrest into container for arm --- tests/parallel_test/run_case.sh | 2 ++ tests/parallel_test/run_container.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 52a5627e34..eda66a884a 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -49,11 +49,13 @@ if [ $ent -eq 0 ]; then export PATH=$PATH:/home/TDengine/debug/build/bin export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDengine else export PATH=$PATH:/home/TDinternal/debug/build/bin export LD_LIBRARY_PATH=/home/TDinternal/debug/build/lib ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDinternal/community fi mkdir -p /var/lib/taos/subscribe diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index affd9128a4..f0ee9be46f 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -100,6 +100,7 @@ docker run \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ + -v $WORKDIR/taos-connector-python/taosrest:/usr/local/lib/python3.8/site-packages/taosrest:ro \ --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $extra_param ret=$? exit $ret From ecd560ccb0ae7ecd8d93654a90ad6427c0994c0e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 15:30:11 +0800 Subject: [PATCH 18/51] start timer for particular msg --- include/libs/transport/trpc.h | 21 +++++--- include/util/taoserror.h | 1 + source/client/src/clientEnv.c | 17 +++++-- source/libs/transport/inc/transComm.h | 5 +- source/libs/transport/inc/transportInt.h | 1 + source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 65 ++++++++++++++---------- source/os/src/osSocket.c | 6 +-- source/util/src/terror.c | 1 + 9 files changed, 73 insertions(+), 45 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 50f9959177..467f0a9ff0 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -41,12 +41,13 @@ typedef struct { typedef struct SRpcHandleInfo { // rpc info - void *handle; // rpc handle returned to app - int64_t refId; // refid, used by server - int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp); - int32_t persistHandle; // persist handle or not + void *handle; // rpc handle returned to app + int64_t refId; // refid, used by server + int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) + int8_t persistHandle; // persist handle or not + int8_t hasEpSet; + STraceId traceId; - int8_t hasEpSet; // app info void *ahandle; // app handle set by client @@ -69,8 +70,9 @@ typedef struct SRpcMsg { SRpcHandleInfo info; } SRpcMsg; -typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -84,12 +86,15 @@ typedef struct SRpcInit { // the following is for client app ecurity only char *user; // user name - // call back to process incoming msg, code shall be ignored by server app + // call back to process incoming msg RpcCfp cfp; - // user defined retry func + // retry not not for particular msg RpcRfp rfp; + // set up timeout for particular msg + RpcTfp tfp; + void *parent; } SRpcInit; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 97a664d776..27fb057b44 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -46,6 +46,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) +#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) //common & util #define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d88a587f6a..ae80868167 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -60,7 +60,7 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { } static void deregisterRequest(SRequestObj *pRequest) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable + const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable assert(pRequest != NULL); STscObj *pTscObj = pRequest->pTscObj; @@ -77,13 +77,13 @@ static void deregisterRequest(SRequestObj *pRequest) { if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { - atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } - + if (duration >= SLOW_QUERY_INTERVAL) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); } - + releaseTscObj(pTscObj->id); } @@ -109,6 +109,12 @@ static bool clientRpcRfp(int32_t code, tmsg_t msgType) { } } +// start timer for particular msgType +static bool clientRpcTfp(int32_t code, tmsg_t msgType) { + // + return false; +} + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -118,6 +124,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; + rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -375,7 +382,7 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); taosConvInit(); - + rpcInit(); SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a81d6db80f..44075f316c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -95,8 +95,9 @@ typedef void* queue[2]; #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) #define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit -#define TRANS_RETRY_INTERVAL 15 // ms retry interval -#define TRANS_CONN_TIMEOUT 3 // connect timeout +#define TRANS_RETRY_INTERVAL 15 // retry interval (ms) +#define TRANS_CONN_TIMEOUT 3 // connect timeout (s) +#define TRANS_READ_TIMEOUT 200 // read timeout (ms) typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 5fb2980ceb..6aeeffa192 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); + bool (*startTimer)(int32_t code, tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 7633820292..0a0dcef378 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -48,6 +48,7 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; + pRpc->startTimer = pInit->tfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 431e479123..6458fc99b7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -24,6 +24,7 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; queue wreqQueue; + uv_timer_t* timer; void* hostThrd; @@ -108,6 +109,8 @@ static int sockDebugInfo(struct sockaddr* sockname, char* dst) { sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); return r; } +// register timer for read +static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn // static void cliTimeoutCb(uv_timer_t* handle); // alloc buf for recv @@ -330,6 +333,11 @@ void cliHandleResp(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; + if (uv_is_active((uv_handle_t*)conn->timer)) { + tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); + uv_timer_stop(conn->timer); + } + STransMsgHead* pHead = NULL; transDumpFromBuffer(&conn->readBuf, (char**)&pHead); pHead->code = htonl(pHead->code); @@ -409,7 +417,7 @@ void cliHandleResp(SCliConn* conn) { uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); } -void cliHandleExcept(SCliConn* pConn) { +void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); @@ -428,7 +436,7 @@ void cliHandleExcept(SCliConn* pConn) { STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg transMsg = {0}; - transMsg.code = pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL; + transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; @@ -459,31 +467,17 @@ void cliHandleExcept(SCliConn* pConn) { } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } +void cliHandleExcept(SCliConn* conn) { + tTrace("%s conn except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + cliHandleExceptImpl(conn, -1); +} -// void cliTimeoutCb(uv_timer_t* handle) { -// SCliThrd* pThrd = handle->data; -// STrans* pTransInst = pThrd->pTransInst; -// int64_t currentTime = pThrd->nextTimeout; -// tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); -// -// SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); -// while (p != NULL) { -// while (!QUEUE_IS_EMPTY(&p->conn)) { -// queue* h = QUEUE_HEAD(&p->conn); -// SCliConn* c = QUEUE_DATA(h, SCliConn, q); -// if (c->expireTime < currentTime) { -// QUEUE_REMOVE(h); -// transUnrefCliHandle(c); -// } else { -// break; -// } -// } -// p = taosHashIterate((SHashObj*)pThrd->pool, p); -// } -// -// pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); -// uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0); -// } +void cliReadTimeoutCb(uv_timer_t* handle) { + // set up timeout cb + SCliConn* conn = handle->data; + tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); +} void* createConnPool(int size) { // thread local, no lock @@ -638,6 +632,11 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { conn->connReq.data = conn; + // set read timeout + conn->timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, conn->timer); + conn->timer->data = conn; + transReqQueueInit(&conn->wreqQueue); transQueueInit(&conn->cliMsgs, NULL); @@ -660,7 +659,6 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; - if (conn->task != NULL) transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { @@ -675,6 +673,15 @@ static void cliDestroy(uv_handle_t* handle) { } SCliConn* conn = handle->data; + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosMemoryFree(conn->timer); + conn->timer = NULL; + } + if (conn->task != NULL) { + transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + conn->task = NULL; + } transRemoveExHandle(transGetRefMgt(), conn->refId); taosMemoryFree(conn->ip); conn->stream->data = NULL; @@ -764,6 +771,10 @@ void cliSend(SCliConn* pConn) { CONN_SET_PERSIST_BY_APP(pConn); } + if (pTransInst->startTimer != NULL && pTransInst->startTimer(0, pMsg->msgType)) { + tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); + uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); + } uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 98dfaa4972..f34032056c 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -333,7 +333,7 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nwritten; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; @@ -362,7 +362,7 @@ int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nread; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; @@ -912,7 +912,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr * sa = result->ai_addr; + struct sockaddr *sa = result->ai_addr; struct sockaddr_in *si = (struct sockaddr_in *)sa; struct in_addr ia = si->sin_addr; uint32_t ip = ia.s_addr; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e4b6983dcb..4780d85a30 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -52,6 +52,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish c TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") From a17f46a59c7aa331ae099a90a49cf4b0df202482 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 15:43:19 +0800 Subject: [PATCH 19/51] fix: fix msgIdx mismatch issue --- source/libs/catalog/inc/catalogInt.h | 22 ++-- source/libs/catalog/src/ctgAsync.c | 174 +++++++++++++++----------- source/libs/catalog/src/ctgRemote.c | 179 +++++++++++++++++---------- source/libs/catalog/src/ctgUtil.c | 8 +- 4 files changed, 235 insertions(+), 148 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c452f61b75..4ef23860cd 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -239,6 +239,7 @@ typedef struct SCtgBatch { SRequestConnInfo conn; char dbFName[TSDB_DB_FNAME_LEN]; SArray* pTaskIds; + SArray* pMsgIdxs; } SCtgBatch; typedef struct SCtgJob { @@ -287,6 +288,7 @@ typedef struct SCtgTaskCallbackParam { SArray* taskId; int32_t reqType; int32_t batchId; + SArray* msgIdx; } SCtgTaskCallbackParam; @@ -314,12 +316,16 @@ typedef struct SCtgTask { SArray* pParents; SCtgSubRes subRes; SHashObj* pBatchs; - int32_t msgIdx; } SCtgTask; +typedef struct SCtgTaskReq { + SCtgTask* pTask; + int32_t msgIdx; +} SCtgTaskReq; + typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); -typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); +typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTaskReq*, int32_t, const SDataBuf *, int32_t); typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); typedef int32_t (*ctgCloneTaskResFp)(SCtgTask*, void**); typedef int32_t (*ctgCompTaskFp)(SCtgTask*, void*, bool*); @@ -518,7 +524,7 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_GET_TASK_MSGCTX(_task) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) +#define CTG_GET_TASK_MSGCTX(_task, _id) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) @@ -665,7 +671,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTaskReq* tReq); int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); @@ -673,9 +679,9 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTaskReq* tReq); int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask); @@ -698,7 +704,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); -int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index bbca85ed14..63d99cc58b 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -381,7 +381,10 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == pDb) { + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb || NULL == pTb) { + taosHashCleanup(pDb); + taosHashCleanup(pTb); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -400,18 +403,26 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); } - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableMeta, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + + dbNum = taosArrayGetSize(pReq->pTableHash); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableHash, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { @@ -430,16 +441,6 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashCleanup(pDb); // REFRESH TABLE META - SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); @@ -924,12 +925,13 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SName* pName = ctx->pName; int32_t flag = ctx->flag; @@ -947,7 +949,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); return TSDB_CODE_SUCCESS; } @@ -967,7 +969,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -976,7 +978,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } return TSDB_CODE_SUCCESS; @@ -1014,7 +1016,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { @@ -1033,7 +1035,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * if (0 == exist) { TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); } } break; @@ -1088,14 +1090,15 @@ _return: } -int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SName* pName = ctgGetFetchName(ctx->pNames, pFetch); int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; @@ -1112,7 +1115,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); return TSDB_CODE_SUCCESS; } @@ -1132,7 +1135,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -1141,7 +1144,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } return TSDB_CODE_SUCCESS; @@ -1179,7 +1182,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { @@ -1192,13 +1195,14 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf taosMemoryFreeClear(pOut->tbMeta); CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); if (pOut->tbMeta) { + ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; } } if (0 == exist) { TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); } } break; @@ -1268,8 +1272,9 @@ _return: } -int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1301,8 +1306,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1338,12 +1344,13 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1352,7 +1359,7 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1393,8 +1400,9 @@ _return: } -int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1415,8 +1423,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1428,8 +1437,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1441,13 +1451,14 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbInfoRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { CTG_RET(TSDB_CODE_APP_ERROR); } -int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1459,8 +1470,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1472,8 +1484,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1485,8 +1498,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1498,8 +1512,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; bool pass = false; @@ -1542,8 +1557,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetSvrVerRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1556,7 +1572,8 @@ _return: CTG_RET(code); } -int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32_t* vgId) { +int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq *tReq, int32_t flag, SName* pName, int32_t* vgId) { + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; int32_t code = 0; @@ -1564,14 +1581,14 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32 if (CTG_FLAG_IS_SYS_DB(flag)) { ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(pName)); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, tReq)); } if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pName)); // if get from mnode failed, will not try vnode - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } SCtgDBCache *dbCache = NULL; @@ -1586,14 +1603,14 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32 ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); } else { SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } _return: @@ -1616,7 +1633,10 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { } SCtgTbMetaCtx* pCtx = (SCtgTbMetaCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pCtx->flag, pCtx->pName, &pCtx->vgId)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pCtx->flag, pCtx->pName, &pCtx->vgId)); return TSDB_CODE_SUCCESS; } @@ -1651,11 +1671,11 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); - pTask->msgIdx = pFetch->fetchIdx; - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); } - - pTask->msgIdx = 0; return TSDB_CODE_SUCCESS; } @@ -1680,8 +1700,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1717,8 +1740,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1746,7 +1772,10 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -1775,18 +1804,17 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - pTask->msgIdx = pFetch->fetchIdx; - SBuildUseDBInput input = {0}; strcpy(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } - pTask->msgIdx = 0; - _return: if (dbCache) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 660d91ea8a..a9f2d426bc 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -22,9 +22,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; - SArray* pTaskId = cbParam->taskId; SCatalog* pCtg = pJob->pCtg; - int32_t taskNum = taosArrayGetSize(pTaskId); + int32_t taskNum = taosArrayGetSize(cbParam->taskId); SDataBuf taskMsg = *pMsg; int32_t offset = 0; int32_t msgNum = (TSDB_CODE_SUCCESS == rspCode && pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; @@ -42,7 +41,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } for (int32_t i = 0; i < taskNum; ++i) { - int32_t* taskId = taosArrayGet(pTaskId, i); + int32_t* taskId = taosArrayGet(cbParam->taskId, i); + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); if (msgNum > 0) { rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); @@ -59,8 +59,10 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.msgType = rsp.reqType; taskMsg.pData = rsp.msg; taskMsg.len = rsp.msgLen; + + ASSERT(rsp.msgIdx == *msgIdx); } else { - rsp.msgIdx = pTask->msgIdx++; + rsp.msgIdx = *msgIdx; rsp.reqType = -1; taskMsg.msgType = -1; taskMsg.pData = NULL; @@ -68,11 +70,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } pTask->pBatchs = pBatchs; - pTask->msgIdx = rsp.msgIdx; - ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, pTask->msgIdx, TMSG_INFO(taskMsg.msgType + 1)); - - (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = rsp.msgIdx; + + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1)); + + (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -341,7 +346,10 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { pTask->pBatchs = pBatchs; #endif - CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, cbParam->reqType, pMsg, rspCode)); #if CTG_BATCH_FETCH CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -359,7 +367,7 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int32_t msgType, +int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArray* pMsgIdx, int32_t msgType, SMsgSendInfo** pMsgSendInfo) { int32_t code = 0; SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -379,6 +387,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 param->refId = pJob->refId; param->taskId = pTaskId; param->batchId = batchId; + param->msgIdx = pMsgIdx; msgSendInfo->param = param; msgSendInfo->paramFreeFp = ctgFreeMsgSendParam; @@ -397,10 +406,10 @@ _return: } int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, SArray* pTaskId, int32_t batchId, - char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { + SArray* pMsgIdx, char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; SMsgSendInfo* pMsgSendInfo = NULL; - CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, msgType, &pMsgSendInfo)); + CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); @@ -431,9 +440,10 @@ _return: CTG_RET(code); } -int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTask* pTask, int32_t msgType, void* msg, +int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTaskReq* tReq, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SHashObj* pBatchs = pTask->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); @@ -443,13 +453,14 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == pBatch) { newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); - if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { + newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); + if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } newBatch.conn = *pConn; - req.msgIdx = pTask->msgIdx; + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -459,6 +470,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(newBatch.pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { @@ -469,7 +483,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; @@ -496,7 +510,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT return TSDB_CODE_SUCCESS; } - req.msgIdx = pTask->msgIdx; + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -506,6 +520,10 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(pBatch->pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(pBatch->pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { @@ -516,7 +534,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; @@ -588,8 +606,8 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg)); - code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->dbFName, *vgId, - pBatch->msgType, msg, pBatch->msgSize); + code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, + pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize); pBatch->pTaskIds = NULL; CTG_ERR_JRET(code); @@ -628,10 +646,13 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -639,7 +660,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -674,10 +695,13 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -685,7 +709,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -706,10 +730,11 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -726,10 +751,10 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, input->db)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, input->db)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -737,7 +762,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -778,10 +803,13 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)dbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)dbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -789,7 +817,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -830,10 +858,13 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)indexName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)indexName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -841,7 +872,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -884,10 +915,13 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -895,7 +929,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -936,10 +970,13 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)funcName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)funcName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -947,7 +984,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -988,10 +1025,13 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)user)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)user)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -999,7 +1039,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1020,7 +1060,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char* msg = NULL; SEpSet* pVnodeEpSet = NULL; @@ -1044,10 +1085,10 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1055,7 +1096,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1076,15 +1117,16 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, pTask); + return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); } int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgroupInfo, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); int32_t reqType = TDMT_VND_TABLE_META; @@ -1118,10 +1160,10 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1132,7 +1174,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); #endif } @@ -1175,14 +1217,17 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, &tReq, reqType, msg, msgLen)); #else SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1193,7 +1238,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); #endif } @@ -1234,10 +1279,13 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1245,7 +1293,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1280,10 +1328,13 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1291,7 +1342,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index de9ea7fb5a..8e5fb90f1a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -26,6 +26,7 @@ void ctgFreeMsgSendParam(void* param) { SCtgTaskCallbackParam* pParam = (SCtgTaskCallbackParam*)param; taosArrayDestroy(pParam->taskId); + taosArrayDestroy(pParam->msgIdx); taosMemoryFree(param); } @@ -874,8 +875,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); if (vgNum <= 0) { @@ -904,7 +906,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); pRes->pRes = vgInfo; } else { @@ -958,7 +960,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); pRes->pRes = pNewVg; } else { From 66adfa9e42719cd9c2ef6b12b6161d4536ace70f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 15:50:12 +0800 Subject: [PATCH 20/51] start timer for particular msg --- source/client/src/clientEnv.c | 4 +++- source/libs/transport/inc/transComm.h | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index ae80868167..812351e208 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -111,7 +111,9 @@ static bool clientRpcRfp(int32_t code, tmsg_t msgType) { // start timer for particular msgType static bool clientRpcTfp(int32_t code, tmsg_t msgType) { - // + if (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_CREATE_TABLE) { + return true; + } return false; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 44075f316c..ad0ce4f5e1 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -94,10 +94,10 @@ typedef void* queue[2]; /* Return the structure holding the given element. */ #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) -#define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit -#define TRANS_RETRY_INTERVAL 15 // retry interval (ms) -#define TRANS_CONN_TIMEOUT 3 // connect timeout (s) -#define TRANS_READ_TIMEOUT 200 // read timeout (ms) +#define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit +#define TRANS_RETRY_INTERVAL 15 // retry interval (ms) +#define TRANS_CONN_TIMEOUT 3 // connect timeout (s) +#define TRANS_READ_TIMEOUT 3000 // read timeout (ms) typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; From 16a24f5a8452ae240a6390ff8b5219a7e3eb6331 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 16:12:49 +0800 Subject: [PATCH 21/51] fix: add rsp msg size --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..47f6fef6a8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1308,8 +1308,8 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, code: %s, gtid: %s", pMsg->info.handle, - TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), tbuf); + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle, + TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); From 784752010afc5fac6aa8a7898530c929d6abe267 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 22/51] fix(query): fix interp fill(next) result errors --- source/libs/executor/inc/executorimpl.h | 3 + source/libs/executor/src/timewindowoperator.c | 85 +++++++++++++++---- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b62ff2bef1..f8a7ab330f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -737,6 +737,9 @@ typedef struct STimeSliceOperatorInfo { SInterval interval; int64_t current; SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + bool isPrevRowSet; + bool isNextRowSet; int32_t fillType; // fill type SColumn tsCol; // primary timestamp column SExprSupp scalarSup; // scalar calculation diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1eee2c197e..9fcb255e9e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2084,10 +2084,30 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock memcpy(pkey->pData, val, pkey->bytes); } } + + pSliceInfo->isPrevRowSet = true; +} + +static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + // null data should not be kept since it can not be used to perform interpolation + if (!colDataIsNull_s(pColInfoData, i)) { + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + memcpy(pkey->pData, val, pkey->bytes); + } + } + + pSliceInfo->isNextRowSet = true; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock, bool isPrevRowSet) { + int32_t rowIndex, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2163,7 +2183,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } case TSDB_FILL_PREV: { - if (!isPrevRowSet) { + if (!pSliceInfo->isPrevRowSet) { break; } @@ -2174,12 +2194,14 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } case TSDB_FILL_NEXT: { - if (!isPrevRowSet) { + if (!pSliceInfo->isNextRowSet) { break; } - char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + //char* p = colDataGetData(pSrc, rowIndex); + //colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); + colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; break; } @@ -2213,6 +2235,35 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB taosArrayPush(pInfo->pPrevRow, &key); } + pInfo->isPrevRowSet = false; + + return TSDB_CODE_SUCCESS; +} + +static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + if (pInfo->pNextRow != NULL) { + return TSDB_CODE_SUCCESS; + } + + pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pNextRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SGroupKeys key = {0}; + key.bytes = pColInfo->info.bytes; + key.type = pColInfo->info.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pColInfo->info.bytes); + taosArrayPush(pInfo->pNextRow, &key); + } + + pInfo->isNextRowSet = false; + return TSDB_CODE_SUCCESS; } @@ -2242,15 +2293,19 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); - //int32_t numOfRows = 0; - bool isPrevRowSet = false; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { break; } - int32_t code = initPrevRowsKeeper(pSliceInfo, pBlock); + int32_t code; + code = initPrevRowsKeeper(pSliceInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + code = initNextRowsKeeper(pSliceInfo, pBlock); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2272,13 +2327,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); char* v = colDataGetData(pSrc, i); - //colDataAppend(pDst, numOfRows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false); } pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2293,13 +2346,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } else if (ts < pSliceInfo->current) { //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2316,11 +2368,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } else { // it is the last row of current block doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; } } else { // ts > pSliceInfo->current + //in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2344,7 +2398,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2363,7 +2416,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { //check if need to interpolate after ts range while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From 8c3cc0161adaaeff9267808dbe79186383e5441f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 23/51] fix(query): fix interp fill(next) result errors --- source/libs/executor/src/timewindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9fcb255e9e..f078bf4b56 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2344,8 +2344,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { - //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + //in case interpolation window starts and ends between two datapoints, fill(prev), fill(next) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepNextRows(pSliceInfo, pBlock, i + 1); if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); From 97794bd9d81b7bfe5a16e0a01fb034e1aa0b2c0d Mon Sep 17 00:00:00 2001 From: dingbo Date: Sat, 6 Aug 2022 16:28:51 +0800 Subject: [PATCH 24/51] docs: add releases.md --- .gitignore | 1 + docs/zh/28-releases.md | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 docs/zh/28-releases.md diff --git a/.gitignore b/.gitignore index c25f60075c..1c609d3ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,4 @@ contrib/* !contrib/test sql debug*/ +.env \ No newline at end of file diff --git a/docs/zh/28-releases.md b/docs/zh/28-releases.md new file mode 100644 index 0000000000..5f30325829 --- /dev/null +++ b/docs/zh/28-releases.md @@ -0,0 +1,9 @@ +--- +sidebar_label: 发布历史 +title: 发布历史 +--- + +import Release from "/components/Release"; + + + From d62f075a49722d735130ef01ae5e8971278f886e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 6 Aug 2022 16:42:17 +0800 Subject: [PATCH 25/51] chore: update libtaos ws 29424d5 for3.0 (#15804) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index a2b45a5bfa..de6409a8c6 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git - GIT_TAG 648cc62 + GIT_TAG 29424d5 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 78fb4f51b0824a4a3e1397fa3158d6f9ce52aa64 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 26/51] fix(query): fix interp fill(next) result errors --- source/libs/executor/src/timewindowoperator.c | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f078bf4b56..d12410869a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2107,7 +2107,7 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock) { + SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2198,8 +2198,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } - //char* p = colDataGetData(pSrc, rowIndex); - //colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; @@ -2344,15 +2342,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { - //in case interpolation window starts and ends between two datapoints, fill(prev), fill(next) need to interpolate + // in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); - doKeepNextRows(pSliceInfo, pBlock, i + 1); if (i < pBlock->info.rows - 1) { + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2371,11 +2370,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current - //in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2415,9 +2414,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - //check if need to interpolate after ts range - while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); + // check if need to interpolate after ts range + // except for fill(next) + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From d44bce1c6f64936106d2550f16550088e6b1008c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 6 Aug 2022 17:20:15 +0800 Subject: [PATCH 27/51] fix: update vnode's commitTerm --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 650c98eb88..460d0b6fbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -223,6 +223,8 @@ int vnodeCommit(SVnode *pVnode) { vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + pVnode->state.commitTerm = pVnode->state.applyTerm; + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; From c9b2374a0ecd1a1f7d516767f7471d3eaf459b0f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 17:55:23 +0800 Subject: [PATCH 28/51] fix: fix show database result error --- source/common/src/systable.c | 16 ++++++++++++---- source/dnode/mnode/impl/src/mndDb.c | 8 ++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 675d3bbfdd..a79082ab23 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -354,11 +354,19 @@ static const SSysTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}}; void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { - *pInfosTableMeta = infosMeta; - *size = tListLen(infosMeta); + if (pInfosTableMeta) { + *pInfosTableMeta = infosMeta; + } + if (size) { + *size = tListLen(infosMeta); + } } void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size) { - *pPerfsTableMeta = perfsMeta; - *size = tListLen(perfsMeta); + if (pPerfsTableMeta) { + *pPerfsTableMeta = perfsMeta; + } + if (size) { + *size = tListLen(perfsMeta); + } } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0120e1146d..66f81a3dba 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1691,13 +1691,17 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1); + size_t numOfTables = 0; + getInfosDbMeta(NULL, &numOfTables); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1); + numOfTables = 0; + getPerfDbMeta(NULL, &numOfTables); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; pShow->sysDbRsp = true; From 0753087e88fedb9de6776073c99916c9154a021c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 17:59:30 +0800 Subject: [PATCH 29/51] start timer for particular msg --- source/libs/transport/src/transCli.c | 112 +++++++++++++-------------- 1 file changed, 53 insertions(+), 59 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 6458fc99b7..b8346370e9 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -66,12 +66,13 @@ typedef struct SCliThrd { int64_t pid; // pid uv_loop_t* loop; SAsyncPool* asyncPool; - uv_idle_t* idle; uv_prepare_t* prepare; - uv_timer_t timer; void* pool; // conn pool + SArray* timerList; + // msg queue + queue msg; TdThreadMutex msgMtx; SDelayQueue* delayQueue; @@ -333,9 +334,14 @@ void cliHandleResp(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - if (uv_is_active((uv_handle_t*)conn->timer)) { - tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); - uv_timer_stop(conn->timer); + if (conn->timer) { + if (uv_is_active((uv_handle_t*)conn->timer)) { + tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); + uv_timer_stop(conn->timer); + } + conn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer = NULL; } STransMsgHead* pHead = NULL; @@ -468,7 +474,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { - tTrace("%s conn except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + tTrace("%s conn %p except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); cliHandleExceptImpl(conn, -1); } @@ -632,11 +638,6 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { conn->connReq.data = conn; - // set read timeout - conn->timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); - uv_timer_init(pThrd->loop, conn->timer); - conn->timer->data = conn; - transReqQueueInit(&conn->wreqQueue); transQueueInit(&conn->cliMsgs, NULL); @@ -653,13 +654,24 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { + SCliThrd* pThrd = conn->hostThrd; tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); transRemoveExHandle(transGetRefMgt(), conn->refId); - conn->refId = -1; + if (conn->task != NULL) { + transDQCancel(pThrd->timeoutQueue, conn->task); + conn->task = NULL; + } + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; + conn->timer = NULL; + } + if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { uv_read_stop(conn->stream); @@ -671,17 +683,15 @@ static void cliDestroy(uv_handle_t* handle) { if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) { return; } - SCliConn* conn = handle->data; + SCliThrd* pThrd = conn->hostThrd; if (conn->timer != NULL) { uv_timer_stop(conn->timer); - taosMemoryFree(conn->timer); + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; conn->timer = NULL; } - if (conn->task != NULL) { - transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); - conn->task = NULL; - } + transRemoveExHandle(transGetRefMgt(), conn->refId); taosMemoryFree(conn->ip); conn->stream->data = NULL; @@ -772,6 +782,15 @@ void cliSend(SCliConn* pConn) { } if (pTransInst->startTimer != NULL && pTransInst->startTimer(0, pMsg->msgType)) { + uv_timer_t* timer = taosArrayPop(pThrd->timerList); + if (timer == NULL) { + tDebug("no avaiable timer, create"); + timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, timer); + } + timer->data = pConn; + pConn->timer = timer; + tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); } @@ -792,8 +811,8 @@ void cliConnCb(uv_connect_t* req, int status) { } // int addrlen = sizeof(pConn->addr); struct sockaddr peername, sockname; - int addrlen = sizeof(peername); + int addrlen = sizeof(peername); uv_tcp_getpeername((uv_tcp_t*)pConn->stream, &peername, &addrlen); transGetSockDebugInfo(&peername, pConn->dst); @@ -817,7 +836,6 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); - uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -890,8 +908,8 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { - STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; + STransConnCtx* pCtx = pMsg->ctx; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); if (!EPSET_IS_VALID(&pCtx->epSet)) { @@ -977,36 +995,6 @@ static void cliAsyncCb(uv_async_t* handle) { } if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } -static void cliIdleCb(uv_idle_t* handle) { - SCliThrd* thrd = handle->data; - tTrace("do idle work"); - - SAsyncPool* pool = thrd->asyncPool; - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - - SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - if (pMsg == NULL) { - continue; - } - (*cliAsyncHandle[pMsg->type])(pMsg, thrd); - count++; - } - } - tTrace("prepare work end"); - if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); -} static void cliPrepareCb(uv_prepare_t* handle) { SCliThrd* thrd = handle->data; tTrace("prepare work start"); @@ -1096,19 +1084,20 @@ static SCliThrd* createThrdObj() { uv_loop_init(pThrd->loop); pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 5, pThrd, cliAsyncCb); - uv_timer_init(pThrd->loop, &pThrd->timer); - pThrd->timer.data = pThrd; - - // pThrd->idle = taosMemoryCalloc(1, sizeof(uv_idle_t)); - // uv_idle_init(pThrd->loop, pThrd->idle); - // pThrd->idle->data = pThrd; - // uv_idle_start(pThrd->idle, cliIdleCb); pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); uv_prepare_init(pThrd->loop, pThrd->prepare); pThrd->prepare->data = pThrd; uv_prepare_start(pThrd->prepare, cliPrepareCb); + int32_t timerSize = 512; + pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); + for (int i = 0; i < timerSize; i++) { + uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, timer); + taosArrayPush(pThrd->timerList, &timer); + } + pThrd->pool = createConnPool(4); transDQCreate(pThrd->loop, &pThrd->delayQueue); @@ -1131,7 +1120,12 @@ static void destroyThrdObj(SCliThrd* pThrd) { transDQDestroy(pThrd->delayQueue, destroyCmsg); transDQDestroy(pThrd->timeoutQueue, NULL); - taosMemoryFree(pThrd->idle); + for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { + uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + taosMemoryFree(timer); + } + taosArrayDestroy(pThrd->timerList); + taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); From fd654b126a16d32ae9d78d4078cb4a03cf885e9a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 18:00:26 +0800 Subject: [PATCH 30/51] fix: fix taosd memory leak issue --- include/libs/executor/executor.h | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/executorimpl.c | 4 +++- source/libs/qworker/inc/qwMsg.h | 2 +- source/libs/qworker/src/qworker.c | 5 ++++- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index ab33af6acf..7bc2ede4fc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -103,7 +103,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo * @return */ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model); + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model); /** * diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b62ff2bef1..631d88c684 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -987,7 +987,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - const char* sql, EOPTR_EXEC_MODEL model); + char* sql, EOPTR_EXEC_MODEL model); int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f080549578..897ddb59ec 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -341,7 +341,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table } int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { assert(pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index eaffbc2579..3a61661c5a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4492,7 +4492,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT } int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - const char* sql, EOPTR_EXEC_MODEL model) { + char* sql, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -4503,6 +4503,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; + sql = NULL; (*pTaskInfo)->pSubplan = pPlan; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); @@ -4515,6 +4516,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead return code; _complete: + taosMemoryFree(sql); doDestroyTask(*pTaskInfo); terrno = code; return code; diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 3ee870ef96..3ff5b5950f 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -25,7 +25,7 @@ extern "C" { int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF); int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql); +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char* sql); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 36d85f1f12..04fca6f0d9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -508,7 +508,7 @@ _return: } -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char* sql) { int32_t code = 0; bool queryRsped = false; SSubplan *plan = NULL; @@ -536,6 +536,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { } code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, sql, OPTR_EXEC_MODEL_BATCH); + sql = NULL; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -561,6 +562,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { _return: + taosMemoryFree(sql); + input.code = code; input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); From 0beed37cfd3c613f336e9e472011b26184d13216 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 6 Aug 2022 18:06:02 +0800 Subject: [PATCH 31/51] fix: insert tsma data --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 5 ++++- source/dnode/vnode/src/meta/metaTable.c | 2 ++ source/dnode/vnode/src/sma/smaTimeRange.c | 10 +++++++++- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f836cd76ac..f6d8ea51c4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -89,7 +89,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; -int32_t tsQuerySmaOptimize = 0; +int32_t tsQuerySmaOptimize = 1; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 6411d06081..006d9e749c 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -489,7 +489,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); ASSERT(smaObj.uid != 0); char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; - snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name); + snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb",pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN); smaObj.stbUid = pStb->uid; @@ -603,6 +603,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + mDebug("mndSma: create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", + pCreate->name, smaObj.uid, smaObj.stbUid, smaObj.dstTbUid, smaObj.dstTbName, smaObj.dstVgId); + code = 0; _OVER: diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index fdf2da5558..7427f79509 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -381,6 +381,8 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); return -1; + } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + terrno = TSDB_CODE_SUCCESS; } metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 6d71f3a250..7f69acc889 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -116,8 +116,10 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { } // create stable to save tsma result in dstVgId + SName stbFullName = {0}; + tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateStbReq pReq = {0}; - pReq.name = pCfg->dstTbName; + pReq.name = (char*)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; @@ -125,6 +127,12 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { return -1; } + + smaDebug("vgId:%d, success to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 + " dstTb:%s dstVg:%d", + SMA_VID(pSma), pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, pCfg->dstVgId); + } else { + ASSERT(0); } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fbbe5bc695..15cf183b2a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -351,7 +351,6 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // TODO: remove the function void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } @@ -852,6 +851,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { submitBlkRsp.code = terrno; + pRsp->code = terrno; tDecoderClear(&decoder); taosArrayDestroy(createTbReq.ctb.tagName); goto _exit; From 3a3c6fac6bee1e454deecdbc8b94aa2f4ea1336a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 6 Aug 2022 18:08:46 +0800 Subject: [PATCH 32/51] start timer for particular msg --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b8346370e9..dc7ffa9b13 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -782,7 +782,7 @@ void cliSend(SCliConn* pConn) { } if (pTransInst->startTimer != NULL && pTransInst->startTimer(0, pMsg->msgType)) { - uv_timer_t* timer = taosArrayPop(pThrd->timerList); + uv_timer_t* timer = taosArrayGetSize(pThrd->timerList) > 0 ? *(uv_timer_t**)taosArrayPop(pThrd->timerList) : NULL; if (timer == NULL) { tDebug("no avaiable timer, create"); timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); From bb7f89dc24a9641501629b4a419fdce64411a23d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 6 Aug 2022 18:50:58 +0800 Subject: [PATCH 33/51] fix: tsma query data --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f6d8ea51c4..f836cd76ac 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -89,7 +89,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; -int32_t tsQuerySmaOptimize = 1; +int32_t tsQuerySmaOptimize = 0; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, From 428a75171c447d6939effb1224f9b4e30c82a49d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 18:53:36 +0800 Subject: [PATCH 34/51] fix: remove failed case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f44e81433d..56ff125fda 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -216,7 +216,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_query # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +#python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 From d556d9290edefb39d43300b0b854fe49e16cee91 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 6 Aug 2022 19:19:26 +0800 Subject: [PATCH 35/51] feat: update taostools 3c7dafe for3.0 (#15816) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools 8e3b3ee * fix: remove submodules * feat: update taos-tools c529299 * feat: update taos-tools 9dc2fec for 3.0 * fix: optim upx * feat: update taos-tools f4e456a for 3.0 * feat: update taos-tools 2a2def1 for 3.0 * feat: update taos-tools c9cc20f for 3.0 * feat: update taostoosl 8a5e336 for 3.0 * feat: update taostools 3c7dafe for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 620faa636b..9ee0ea526c 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 8a5e336 + GIT_TAG 3c7dafe SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From a16a6b73c8ff73976daa63a667a339184113e4d1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 6 Aug 2022 20:11:12 +0800 Subject: [PATCH 36/51] fix: force at least one row for agg/window group --- source/libs/executor/src/executorimpl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7c19ad024e..b99efce106 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1448,6 +1448,10 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ pRow->numOfRows = pResInfo->numOfRes; } } + // TODO: if all expr skips all blocks, e.g. all null inputs for max function, output one row in final result. + if (pRow->numOfRows == 0) { + pRow->numOfRows = 1; + } } // todo extract method with copytoSSDataBlock From 9e32cd4fca4435f38a7795ba2c9710cc4e6ba7d8 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 6 Aug 2022 21:30:38 +0800 Subject: [PATCH 37/51] test: modify testcases of muti-mnodes --- tests/pytest/util/sql.py | 48 +++++++-------------- tests/script/tsim/sync/3Replica1VgElect.sim | 4 +- tests/script/tsim/sync/3Replica5VgElect.sim | 22 ++-------- tests/system-test/2-query/tsbsQuery.py | 40 +++++++++-------- 4 files changed, 44 insertions(+), 70 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 4c50af6bac..753c41e094 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -84,10 +84,10 @@ class TDSql: self.queryResult = None tdLog.info("sql:%s, expect error occured" % (sql)) - def query(self, sql, row_tag=None,queyTimes=10): + def query(self, sql, row_tag=None,queryTimes=10): self.sql = sql i=1 - while i <= queyTimes: + while i <= queryTimes: try: self.cursor.execute(sql) self.queryResult = self.cursor.fetchall() @@ -97,26 +97,15 @@ class TDSql: return self.queryResult return self.queryRows except Exception as e: - i+=1 tdLog.notice("Try to query again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 time.sleep(1) pass - else: - try: - tdLog.notice("Try the last query ") - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - if row_tag: - return self.queryResult - return self.queryRows - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - traceback.print_exc() - raise Exception(repr(e)) def is_err_sql(self, sql): @@ -305,28 +294,23 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queyTimes=10): + def execute(self, sql,queryTimes=10): self.sql = sql i=1 - while i <= queyTimes: + while i <= queryTimes: try: self.affectedRows = self.cursor.execute(sql) return self.affectedRows except Exception as e: - i+=1 tdLog.notice("Try to execute sql again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 time.sleep(1) pass - else: - try: - tdLog.notice("Try the last execute sql ") - self.affectedRows = self.cursor.execute(sql) - return self.affectedRows - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index 4e127654ee..7cd291e56f 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -48,7 +48,7 @@ endi $replica = 3 $vgroups = 1 -print ============= create database +print ============= create database db sql create database db replica $replica vgroups $vgroups $loop_cnt = 0 @@ -135,7 +135,7 @@ endw $totalTblNum = $tbNum * 2 sleep 1000 sql show tables -print ====> expect $totalTblNum and infinsert $rows in fact +print ====> expect $totalTblNum and insert $rows in fact if $rows != $totalTblNum then return -1 endi diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index c4ab9bd4bc..a9858acbfb 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -202,7 +202,7 @@ else endi vg_ready: -print ====> create stable/child table +print ====> create stable stb /child table ctb and general table ntb sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) sql show stables @@ -225,7 +225,7 @@ endw $totalTblNum = $tbNum * 2 sleep 1000 sql show tables -print ====> expect $totalTblNum and infinsert $rows in fact +print ====> expect $totalTblNum and insert $rows in fact if $rows != $totalTblNum then return -1 endi @@ -237,6 +237,7 @@ sql show vgroups $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId +print ====> switch_leader_to_offine_loop switch_leader_to_offine_loop: print $dnodeId @@ -271,22 +272,7 @@ system sh/exec.sh -n $dnodeId -s start $switch_loop_cnt = $switch_loop_cnt + 1 print $switch_loop_cnt -if $switch_loop_cnt == 1 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 2 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 3 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 4 then +if $switch_loop_cnt <= 4 then sql show vgroups $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 75b33f1a5e..617f7e7464 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -4,6 +4,8 @@ import sys import datetime import inspect import random +from util.dnodes import TDDnode +from util.dnodes import tdDnodes from util.log import * from util.sql import * @@ -30,10 +32,10 @@ class TDTestCase: for i in range(ctbNum): tagValue = 'beijing' if (i % 10 == 0): - sql += " %s%d using %s (name,fleet,driver,device_version) tags('truck_%d', 'South%d','Trish%d','v2.%d')"%(ctbPrefix,i,stbName,i,i,i,i) + sql += " %s%d using %s (name,fleet,driver,device_version,load_capacity,fuel_capacity,nominal_fuel_consumption) tags('truck_%d', 'South%d','Trish%d','v2.%d', 1500+%d*20, 150+%d*2, 5+%d)"%(ctbPrefix,i,stbName,i,i,i,i,(1500+i*20),(150+i*2),(5+i)) else: model = 'H-%d'%i - sql += " %s%d using %s tags('truck_%d', 'South%d','Trish%d','%s','v2.%d')"%(ctbPrefix,i,stbName,i,i,i,model,i) + sql += " %s%d using %s tags('truck_%d', 'South%d','Trish%d','%s','v2.%d', %d, %d,%d)"%(ctbPrefix,i,stbName,i,i,i,model,i,(1500+i*20),(150+i*2),(5+i)) if (i > 0) and (i%1000 == 0): tsql.execute(sql) sql = pre_create @@ -55,10 +57,10 @@ class TDTestCase: sql += " %s%d values "%(ctbPrefix,i) for j in range(rowsPerTbl): if(ctbPrefix=="rct"): - sql += f"({startTs+j*60000}, {80+j}, {90+j}, {85+j}, {30+j*10}, {1.2*j}, {221+j*2}, {20+j*0.2}, {1500+j*20}, {150+j*2},{5+j}) " + sql += f"({startTs+j*60000}, {80+j}, {90+j}, {85+j}, {30+j*10}, {1.2*j}, {221+j*2}, {20+j*0.2}) " elif ( ctbPrefix=="dct"): status= random.randint(0,1) - sql += f"( {startTs+j*60000}, {1+j*0.1},{1400+j*15}, {status},{1500+j*20}, {150+j*2},{5+j} ) " + sql += f"( {startTs+j*60000}, {1+j*0.1},{1400+j*15}, {status} ) " # tdLog.debug("1insert sql:%s"%sql) if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): # tdLog.debug("2insert sql:%s"%sql) @@ -79,22 +81,22 @@ class TDTestCase: stabname2="diagnostics" ctbnamePre1="rct" ctbnamePre2="dct" - ctbNums=40 + ctbNums=50 self.ctbNums=ctbNums - rowNUms=200 + rowNUms=5000 ts=1451606400000 tdSql.execute(f"create database {dbname};") tdSql.execute(f"use {dbname} ") tdSql.execute(f''' - create table {stabname1} (ts timestamp,latitude double,longitude double,elevation double,velocity double,heading double,grade double,fuel_consumption double,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)); + create table {stabname1} (ts timestamp,latitude double,longitude double,elevation double,velocity double,heading double,grade double,fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30),load_capacity double,fuel_capacity double,nominal_fuel_consumption double); ''') tdSql.execute(f''' - create table {stabname2} (ts timestamp,fuel_state double,current_load double,status bigint,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)) ; + create table {stabname2} (ts timestamp,fuel_state double,current_load double,status bigint) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30),load_capacity double,fuel_capacity double,nominal_fuel_consumption double) ; ''') self.create_ctable(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums) self.create_ctable(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums) - self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=1000) - self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=1000) + self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=10000) + self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=10000) # for i in range(ctbNum): # if i %10 == 0 : # # tdLog.debug(f"create table rct{i} using readings (name,fleet,driver,model,device_version) tags ('truck_{i}','South{i}','Trish{i}', NULL,'v2.3')") @@ -131,7 +133,7 @@ class TDTestCase: # tdLog.info("avg value check pass , it work as expected ,sql is \"%s\" "%check_query ) - def tsbsIotQuery(self): + def tsbsIotQuery(self,insertinto=True): tdSql.execute("use db_tsbs") @@ -143,10 +145,11 @@ class TDTestCase: # test insert into - tdSql.execute("create table testsnode (ts timestamp, c1 float,c2 binary(30),c3 binary(30),c4 binary(30)) ;") - tdSql.query("insert into testsnode SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") - - tdSql.query("insert into testsnode(ts,c1,c2,c3,c4) SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") + if insertinto == True : + tdSql.execute("create table testsnode (ts timestamp, c1 float,c2 binary(30),c3 binary(30),c4 binary(30)) ;") + tdSql.query("insert into testsnode SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") + + tdSql.query("insert into testsnode(ts,c1,c2,c3,c4) SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") # test paitition interval fill @@ -182,8 +185,7 @@ class TDTestCase: # # 6. avg-daily-driving-session # #taosc core dumped - # tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") - # tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query(" SELECT _wstart as ts,name,floor(avg(velocity)/5) AS mv FROM readings WHERE name is not null AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0);") # tdSql.query("select name,diff(mv) AS difka FROM (SELECT ts,name,mv FROM (SELECT _wstart as ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0))) group BY name ;") # tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") @@ -234,7 +236,9 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() self.tsbsIotQuery() - + tdDnodes.stop(1) + tdDnodes.start(1) + self.tsbsIotQuery(False) def stop(self): tdSql.close() From 81d3a2c3856ee566a1a7ed6a53ecdfe53ca08b67 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 6 Aug 2022 21:39:53 +0800 Subject: [PATCH 38/51] fix: last/first must return not null values --- include/libs/function/functionMgt.h | 1 + source/libs/executor/src/executorimpl.c | 13 +++++++++---- source/libs/function/src/functionMgt.c | 12 ++++++++++++ tests/script/tsim/parser/function.sim | 6 +++--- tests/system-test/2-query/avg.py | 2 +- .../2-query/distribute_agg_apercentile.py | 2 +- tests/system-test/2-query/distribute_agg_max.py | 2 +- tests/system-test/2-query/distribute_agg_min.py | 2 +- tests/system-test/2-query/distribute_agg_spread.py | 2 +- tests/system-test/2-query/sample.py | 4 ++-- 10 files changed, 32 insertions(+), 14 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f6f2ceb957..e2846c5974 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -202,6 +202,7 @@ bool fmIsForbidStreamFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); +bool fmIsReturnNotNullFunc(int32_t funcId); bool fmIsSelectValueFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b99efce106..09094b1b14 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1437,7 +1437,8 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u pAggInfo->groupId = groupId; } -static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_t* rowCellOffset) { +static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowCellOffset) { + bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset); if (!isRowEntryInitialized(pResInfo)) { @@ -1447,9 +1448,13 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } + + if (fmIsReturnNotNullFunc(pCtx[j].functionId)) { + returnNotNull = true; + } } // TODO: if all expr skips all blocks, e.g. all null inputs for max function, output one row in final result. - if (pRow->numOfRows == 0) { + if (pRow->numOfRows == 0 && !returnNotNull) { pRow->numOfRows = 1; } } @@ -1462,7 +1467,7 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowCellOffset); if (pRow->numOfRows == 0) { releaseBufPage(pBuf, page); return 0; @@ -1518,7 +1523,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowCellOffset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; releaseBufPage(pBuf, page); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 339ff3808b..88efa7c2ff 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -221,6 +221,18 @@ bool fmIsLastRowFunc(int32_t funcId) { return FUNCTION_TYPE_LAST_ROW == funcMgtBuiltins[funcId].type; } +bool fmIsReturnNotNullFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_LAST == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_LAST_PARTIAL == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_LAST_MERGE == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST_PARTIAL == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST_MERGE == funcMgtBuiltins[funcId].type; +} + bool fmIsSelectValueFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 110901a6e1..9423f53e3a 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -327,8 +327,8 @@ endi print =================>td-2610 sql select stddev(k) from tm2 where ts='2020-12-29 18:46:19.109' -if $rows != 0 then - print expect 0, actual:$rows +if $rows != 1 then + print expect 1, actual:$rows return -1 endi sql select twa(k) from tm2 where ts='2020-12-29 18:46:19.109' @@ -406,7 +406,7 @@ if $data00 != 1.378704626 then endi sql select stddev(c) from m1 -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index ea7c3329ea..2afcc29ac8 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -295,7 +295,7 @@ class TDTestCase: tdSql.checkData(0, 0, 4.500000000) tdSql.query(f" select avg(c1) from {dbname}.stb1 where c1 is null ") - tdSql.checkRows(0) + tdSql.checkRows(1) def avg_func_filter(self, dbname="db"): diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index ad754ad805..34164acfa4 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -86,7 +86,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2.800000000) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index a7b31a2084..f67088d9e6 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -168,7 +168,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select max(c1) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select max(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,10) diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index cc50092451..29e7c5f519 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -167,7 +167,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select min(c1) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select min(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2) diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index 842a74628d..135a61d194 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -195,7 +195,7 @@ class TDTestCase: def distribute_agg_query(self): # basic filter tdSql.query("select spread(c1) from stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query("select spread(c1) from stb1 where t1=1") tdSql.checkData(0,0,8.000000000) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index b6bdd8926e..f09265c300 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -743,7 +743,7 @@ class TDTestCase: # filter data tdSql.query(" select sample(c1, 20 ) from t1 where c1 is null ") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(" select sample(c1, 20 ) from t1 where c1 =6 ") tdSql.checkRows(1) @@ -891,4 +891,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From e2aa735572119690c26c53d92f72b204d8641750 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Sat, 6 Aug 2022 21:56:47 +0800 Subject: [PATCH 39/51] test(go): fix golang example (#15812) --- docs/examples/go/connect/afconn/main.go | 3 +- docs/examples/go/connect/cgoexample/main.go | 3 +- docs/examples/go/connect/restexample/main.go | 3 +- docs/examples/go/connect/wrapper/main.go | 17 ---------- docs/examples/go/insert/json/main.go | 6 ++-- docs/examples/go/insert/line/main.go | 3 +- docs/examples/go/insert/sql/main.go | 22 ++++++------- docs/examples/go/insert/stmt/main.go | 2 +- docs/examples/go/insert/telnet/main.go | 6 ++-- docs/examples/go/query/sync/main.go | 12 +++---- .../zh/07-develop/03-insert-data/_go_stmt.mdx | 2 +- docs/zh/14-reference/11-docker/index.md | 2 +- tests/docs-examples-test/go.sh | 31 +++++++++++++++++++ 13 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 docs/examples/go/connect/wrapper/main.go create mode 100644 tests/docs-examples-test/go.sh diff --git a/docs/examples/go/connect/afconn/main.go b/docs/examples/go/connect/afconn/main.go index 3c16212a45..bb2574a01b 100644 --- a/docs/examples/go/connect/afconn/main.go +++ b/docs/examples/go/connect/afconn/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -10,7 +11,7 @@ func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) defer conn.Close() if err != nil { - fmt.Println("failed to connect, err:", err) + log.Fatalln("failed to connect, err:", err) } else { fmt.Println("connected") } diff --git a/docs/examples/go/connect/cgoexample/main.go b/docs/examples/go/connect/cgoexample/main.go index 57b8d5d91f..881cf15ee3 100644 --- a/docs/examples/go/connect/cgoexample/main.go +++ b/docs/examples/go/connect/cgoexample/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosSql" ) @@ -11,7 +12,7 @@ func main() { var taosDSN = "root:taosdata@tcp(localhost:6030)/" taos, err := sql.Open("taosSql", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) + log.Fatalln("failed to connect TDengine, err:", err) return } fmt.Println("Connected") diff --git a/docs/examples/go/connect/restexample/main.go b/docs/examples/go/connect/restexample/main.go index f10151f04f..67a129bf9c 100644 --- a/docs/examples/go/connect/restexample/main.go +++ b/docs/examples/go/connect/restexample/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosRestful" ) @@ -11,7 +12,7 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) + log.Fatalln("failed to connect TDengine, err:", err) return } fmt.Println("Connected") diff --git a/docs/examples/go/connect/wrapper/main.go b/docs/examples/go/connect/wrapper/main.go deleted file mode 100644 index a459539db3..0000000000 --- a/docs/examples/go/connect/wrapper/main.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/taosdata/driver-go/v3/wrapper" -) - -func main() { - conn, err := wrapper.TaosConnect("localhost", "root", "taosdata", "", 6030) - defer wrapper.TaosClose(conn) - if err != nil { - fmt.Println("fail to connect, err:", err) - } else { - fmt.Println("connected") - } -} diff --git a/docs/examples/go/insert/json/main.go b/docs/examples/go/insert/json/main.go index 805c123342..e470cc6bd4 100644 --- a/docs/examples/go/insert/json/main.go +++ b/docs/examples/go/insert/json/main.go @@ -1,7 +1,7 @@ package main import ( - "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -20,7 +20,7 @@ func prepareDatabase(conn *af.Connector) { func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) if err != nil { - fmt.Println("fail to connect, err:", err) + log.Fatalln("fail to connect, err:", err) } defer conn.Close() prepareDatabase(conn) @@ -32,6 +32,6 @@ func main() { err = conn.OpenTSDBInsertJsonPayload(payload) if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/insert/line/main.go b/docs/examples/go/insert/line/main.go index 1a09edc40c..a270ab2633 100644 --- a/docs/examples/go/insert/line/main.go +++ b/docs/examples/go/insert/line/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -33,6 +34,6 @@ func main() { err = conn.InfluxDBInsertLines(lines, "ms") if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/insert/sql/main.go b/docs/examples/go/insert/sql/main.go index a493ef5fb0..57f881314c 100644 --- a/docs/examples/go/insert/sql/main.go +++ b/docs/examples/go/insert/sql/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosRestful" ) @@ -10,28 +11,26 @@ import ( func createStable(taos *sql.DB) { _, err := taos.Exec("CREATE DATABASE power") if err != nil { - fmt.Println("failed to create database, err:", err) + log.Fatalln("failed to create database, err:", err) } _, err = taos.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") if err != nil { - fmt.Println("failed to create stable, err:", err) + log.Fatalln("failed to create stable, err:", err) } } func insertData(taos *sql.DB) { - sql := `INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)` + sql := `INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)` result, err := taos.Exec(sql) if err != nil { - fmt.Println("failed to insert, err:", err) - return + log.Fatalln("failed to insert, err:", err) } rowsAffected, err := result.RowsAffected() if err != nil { - fmt.Println("failed to get affected rows, err:", err) - return + log.Fatalln("failed to get affected rows, err:", err) } fmt.Println("RowsAffected", rowsAffected) } @@ -40,8 +39,7 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + log.Fatalln("failed to connect TDengine, err:", err) } defer taos.Close() createStable(taos) diff --git a/docs/examples/go/insert/stmt/main.go b/docs/examples/go/insert/stmt/main.go index d23a6e345c..bcbb8e632b 100644 --- a/docs/examples/go/insert/stmt/main.go +++ b/docs/examples/go/insert/stmt/main.go @@ -5,8 +5,8 @@ import ( "time" "github.com/taosdata/driver-go/v3/af" - "github.com/taosdata/driver-go/v3/af/param" "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" ) func checkErr(err error, prompt string) { diff --git a/docs/examples/go/insert/telnet/main.go b/docs/examples/go/insert/telnet/main.go index 4cf4375db5..63e9a56dbf 100644 --- a/docs/examples/go/insert/telnet/main.go +++ b/docs/examples/go/insert/telnet/main.go @@ -1,7 +1,7 @@ package main import ( - "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -20,7 +20,7 @@ func prepareDatabase(conn *af.Connector) { func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) if err != nil { - fmt.Println("fail to connect, err:", err) + log.Fatalln("fail to connect, err:", err) } defer conn.Close() prepareDatabase(conn) @@ -37,6 +37,6 @@ func main() { err = conn.OpenTSDBInsertTelnetLines(lines) if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/query/sync/main.go b/docs/examples/go/query/sync/main.go index add422e385..e37164f47f 100644 --- a/docs/examples/go/query/sync/main.go +++ b/docs/examples/go/query/sync/main.go @@ -2,7 +2,7 @@ package main import ( "database/sql" - "fmt" + "log" "time" _ "github.com/taosdata/driver-go/v3/taosRestful" @@ -12,14 +12,12 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/power" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + log.Fatalln("failed to connect TDengine, err:", err) } defer taos.Close() rows, err := taos.Query("SELECT ts, current FROM meters LIMIT 2") if err != nil { - fmt.Println("failed to select from table, err:", err) - return + log.Fatalln("failed to select from table, err:", err) } defer rows.Close() @@ -30,9 +28,9 @@ func main() { } err := rows.Scan(&r.ts, &r.current) if err != nil { - fmt.Println("scan error:\n", err) + log.Fatalln("scan error:\n", err) return } - fmt.Println(r.ts, r.current) + log.Fatalln(r.ts, r.current) } } diff --git a/docs/zh/07-develop/03-insert-data/_go_stmt.mdx b/docs/zh/07-develop/03-insert-data/_go_stmt.mdx index d65a96549f..db4e227c43 100644 --- a/docs/zh/07-develop/03-insert-data/_go_stmt.mdx +++ b/docs/zh/07-develop/03-insert-data/_go_stmt.mdx @@ -3,6 +3,6 @@ ``` :::tip -driver-go 的模块 `github.com/taosdata/driver-go/v2/wrapper` 是 C 接口的底层封装。使用这个模块也可以实现参数绑定写入。 +driver-go 的模块 `github.com/taosdata/driver-go/v3/wrapper` 是 C 接口的底层封装。使用这个模块也可以实现参数绑定写入。 ::: diff --git a/docs/zh/14-reference/11-docker/index.md b/docs/zh/14-reference/11-docker/index.md index d0b7f14204..e40f0dbd35 100644 --- a/docs/zh/14-reference/11-docker/index.md +++ b/docs/zh/14-reference/11-docker/index.md @@ -147,7 +147,7 @@ import ( "fmt" "time" - _ "github.com/taosdata/driver-go/v2/taosSql" + _ "github.com/taosdata/driver-go/v3/taosSql" ) type config struct { diff --git a/tests/docs-examples-test/go.sh b/tests/docs-examples-test/go.sh new file mode 100644 index 0000000000..185661e8a7 --- /dev/null +++ b/tests/docs-examples-test/go.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +taosd >>/dev/null 2>&1 & +taosadapter >>/dev/null 2>&1 & + +cd ../../docs/examples/go + +go mod tidy + +go run ./connect/afconn/main.go +go run ./connect/cgoexample/main.go +go run ./connect/restexample/main.go + +taos -s "drop database if exists test" +go run ./insert/json/main.go +taos -s "drop database if exists test" +go run ./insert/line/main.go +taos -s "drop database if exists power" +go run ./insert/sql/main.go +taos -s "drop database if exists power" +go run ./insert/stmt/main.go +taos -s "drop database if exists test" +go run ./insert/telnet/main.go + +go run ./query/sync/main.go + +taos -s "drop topic if exists example_tmq_topic" +taos -s "drop database if exists example_tmq" +go run ./sub/main.go From be8c3628c5bc126d2c82dec67f453f5329138915 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 7 Aug 2022 06:08:12 +0800 Subject: [PATCH 40/51] fix: trigger ci test --- source/libs/executor/src/executorimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5666dded83..cffd38c98a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1453,7 +1453,8 @@ static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t nu returnNotNull = true; } } - // TODO: if all expr skips all blocks, e.g. all null inputs for max function, output one row in final result. + // if all expr skips all blocks, e.g. all null inputs for max function, output one row in final result. + // except for first/last, which require not null output, output no rows if (pRow->numOfRows == 0 && !returnNotNull) { pRow->numOfRows = 1; } From 25ae819dbaac4fe616a3345c10db932bb6446f6f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 7 Aug 2022 06:12:33 +0800 Subject: [PATCH 41/51] fix: add tsim/query/udf.sim to ci test --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 356eaed57e..0a859b2045 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -179,6 +179,7 @@ ./test.sh -f tsim/query/scalarFunction.sim ./test.sh -f tsim/query/scalarNull.sim ./test.sh -f tsim/query/session.sim +./test.sh -f tsim/query/udf.sim # ---- qnode ./test.sh -f tsim/qnode/basic1.sim From 03aae25b632558eeae4dbd93a8f3ca572296b924 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 7 Aug 2022 06:39:26 +0800 Subject: [PATCH 42/51] fix: function rename and trigger ci test --- include/libs/function/functionMgt.h | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/function/src/functionMgt.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e2846c5974..479432eebe 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -202,7 +202,7 @@ bool fmIsForbidStreamFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); -bool fmIsReturnNotNullFunc(int32_t funcId); +bool fmIsNotNullOutputFunc(int32_t funcId); bool fmIsSelectValueFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cffd38c98a..7e77799702 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1449,7 +1449,7 @@ static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t nu pRow->numOfRows = pResInfo->numOfRes; } - if (fmIsReturnNotNullFunc(pCtx[j].functionId)) { + if (fmIsNotNullOutputFunc(pCtx[j].functionId)) { returnNotNull = true; } } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 88efa7c2ff..0235f22f55 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -221,7 +221,7 @@ bool fmIsLastRowFunc(int32_t funcId) { return FUNCTION_TYPE_LAST_ROW == funcMgtBuiltins[funcId].type; } -bool fmIsReturnNotNullFunc(int32_t funcId) { +bool fmIsNotNullOutputFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; } From d1cc52f4d8c96445056645292ac4f081393d9379 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 7 Aug 2022 10:53:18 +0800 Subject: [PATCH 43/51] doc(tmq) --- docs/zh/07-develop/07-tmq.md | 37 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 358c824ffa..a9a658b321 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -1,14 +1,18 @@ --- sidebar_label: 消息队列 -description: "数据订阅与推送服务。连续写入到 TDengine 中的时序数据能够被自动推送到订阅客户端。" +description: "数据订阅与推送服务。写入到 TDengine 中的时序数据能够被自动推送到订阅客户端。" title: 消息队列 --- -基于数据天然的时间序列特性，TDengine 的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致，均可视为系统中插入一条带时间戳的新记录。同时，TDengine 在内部严格按照数据时间序列单调递增的方式保存数据。本质上来说，TDengine 中每一张表均可视为一个标准的消息队列。 -TDengine 内嵌支持消息订阅与推送服务（下文都简称TMQ）。使用系统提供的 API，用户可使用普通查询语句订阅数据库中的一张或多张表，或整个库。客户端启动订阅后，定时或按需轮询服务器是否有新的记录到达，有新的记录到达就会将结果反馈到客户。 +为了实时获取写入TDengine的数据，或者以事件到达顺序处理数据，TDengine提供了类似消息队列产品的数据订阅、消费接口，它支持以消费者组的方式，使多个消费者分布式、多线程地同时订阅一个topic，共享消费进度。并且提供了消息的ACK机制，在宕机、重启等复杂环境下确保at least once消费。 + +使用TDengine一体化的数据订阅功能，除了以上标准的消息队列特性，还能在订阅同时通过标签、表名、列、表达式等多种方法过滤所需数据，并且支持对数据进行函数变换、预处理（包括标量udf计算）。 + +为了实现上述功能，TDengine提供了多种灵活的WAL文件切换与保留机制：可以按照时间或文件大小来保留WAL文件（详见create database语句）。在消费时，TDengine从WAL中获取数据，并经过过滤、变换等操作，将数据推送给消费者。 + + -TMQ提供了提交机制来保证消息队列的可靠性和正确性。在调用方法上，支持自动提交和手动提交。 TMQ 的 API 中，与订阅相关的主要数据结构和API如下： @@ -82,18 +86,11 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; 2、超级表订阅 语法：CREATE TOPIC topic_name AS STABLE stbName -- 订阅某超级表的全部数据，schema变更不受限，schema变更后写入的数据将以最新schema返回 -- 在tmq的返回消息中schema是块级别的，每块的schema可能不一样 -- 列变更后写入的数据若未落盘，将以写入时的schema返回 -- 列变更后写入的数据若已落盘，将以落盘时的schema返回 - -3、db订阅 -语法：CREATE TOPIC topic_name AS DATABASE db_name - -- 订阅某一db的全部数据，schema变更不受限 -- 在tmq的返回消息中schema是块级别的，每块的schema可能不一样 -- 列变更后写入的数据若未落盘，将以写入时的schema返回 -- 列变更后写入的数据若已落盘，将以落盘时的schema返回 +与select * from stbName订阅的区别是： +- 不会限制用户的schema变更 +- 返回的是非结构化的数据：返回数据的schema会随之超级表的schema变化而变化 +- 用户对于要处理的每一个数据块都可能有不同的schema，因此，必须重新获取schema +- 返回数据不带有tag 三、创建consumer @@ -121,7 +118,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; tmq_conf_set(conf, "group.id", "cgrpName"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "experimental.snapshot.enable", "true"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -151,9 +148,9 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; /* 循环poll消息 */ int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t consumeDelay = 5000; + int32_t timeOut = 5000; while (running) { - TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeOut); if (tmqmsg) { msgCnt++; totalRows += msg_process(tmqmsg); @@ -190,7 +187,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; int32_t* length = taos_fetch_lengths(msg); int32_t precision = taos_result_precision(msg); const char* tbName = tmq_get_table_name(msg); - rows++; + rows++; taos_print_row(buf, row, fields, numOfFields); printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); } From 662f194cd96841de479a5f6764051d989c205356 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 7 Aug 2022 13:20:45 +0800 Subject: [PATCH 44/51] Update 07-tmq.md --- docs/zh/07-develop/07-tmq.md | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index a9a658b321..c39249e6d2 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -1,18 +1,20 @@ --- -sidebar_label: 消息队列 +sidebar_label: 数据订阅 description: "数据订阅与推送服务。写入到 TDengine 中的时序数据能够被自动推送到订阅客户端。" -title: 消息队列 +title: 数据订阅 --- +为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine提供了类似消息队列产品的数据订阅、消费接口。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。 -为了实时获取写入TDengine的数据，或者以事件到达顺序处理数据，TDengine提供了类似消息队列产品的数据订阅、消费接口，它支持以消费者组的方式，使多个消费者分布式、多线程地同时订阅一个topic，共享消费进度。并且提供了消息的ACK机制，在宕机、重启等复杂环境下确保at least once消费。 +与 kafka 一样，你需要定义 topic, 但 TDengine 的 topic 可以是一张超级表，或一张子表。不仅如此，你可以通过标签、表名、列、表达式等多种方法过滤所需数据，并且支持对数据进行函数变换、预处理（包括标量udf计算）。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤交给 TDengine，而不是应用完成，有效的减少传输的数据量。 -使用TDengine一体化的数据订阅功能，除了以上标准的消息队列特性，还能在订阅同时通过标签、表名、列、表达式等多种方法过滤所需数据，并且支持对数据进行函数变换、预处理（包括标量udf计算）。 - -为了实现上述功能，TDengine提供了多种灵活的WAL文件切换与保留机制：可以按照时间或文件大小来保留WAL文件（详见create database语句）。在消费时，TDengine从WAL中获取数据，并经过过滤、变换等操作，将数据推送给消费者。 +消费者订阅 topic 后，可以实时获得最新的数据。多个消费者可以组成一个消费者组 (consumer group), 一个消费者组里的多个消费者共享消费进度，便于多线程分布式的消费数据，提高数据通吐率。但不同消费者组即使消费同一个topic, 并不共享消费进度。一个消费者组可以订阅多个 topic。如果订阅的是超级表，数据可能会分布在多个不同的vnode上，也就是多个shard上，这样一个消费组里有多个消费者可以提高消费效率。TDengine 的消息队列提供了消息的ACK机制，在宕机、重启等复杂环境下确保at least once消费。 +为了实现上述功能，TDengine 采用了灵活的 WAL (Write-Ahead-Log) 文件切换与保留机制：可以按照时间或文件大小来保留WAL文件（详见create database语句）。在消费时，TDengine 从 WAL 中获取数据，并经过过滤、变换等操作，将数据推送给消费者。 +本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。 +## 主要数据结构和API TMQ 的 API 中，与订阅相关的主要数据结构和API如下： @@ -51,7 +53,9 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm 这些 API 的文档请见 [C/C++ Connector](/reference/connector/cpp)，下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码可以在 [tmq.c](https://github.com/taosdata/TDengine/blob/3.0/examples/c/tmq.c) 看到。 -一、首先完成建库、建一张超级表和多张子表，并每个子表插入若干条数据记录： +## 写入数据 + +首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如： ```sql drop database if exists tmqdb; @@ -67,14 +71,15 @@ insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22'); insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33'); ``` -二、创建topic： +## 创建topic： ```sql create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; ``` -注：TMQ支持多种订阅类型： -1、列订阅 +TMQ支持多种订阅类型： + +### 列订阅 语法：CREATE TOPIC topic_name as subquery 通过select语句订阅（包括select *，或select ts, c1等指定列描述订阅，可以带条件过滤、标量函数计算，但不支持聚合函数、不支持时间窗口聚合） @@ -83,7 +88,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; - 被订阅或用于计算的column和tag不可被删除、修改 - 若发生schema变更，新增的column不出现在结果中 -2、超级表订阅 +### 超级表订阅 语法：CREATE TOPIC topic_name AS STABLE stbName 与select * from stbName订阅的区别是： @@ -92,7 +97,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; - 用户对于要处理的每一个数据块都可能有不同的schema，因此，必须重新获取schema - 返回数据不带有tag -三、创建consumer +## 创建 consumer 目前支持的config： @@ -128,7 +133,9 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; return tmq; ``` -四、创建订阅主题列表 +## 创建 topic 列表 + +单个consumer支持同时订阅多个topic。 ```sql tmq_list_t* topicList = tmq_list_new(); @@ -136,9 +143,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; return topicList; ``` -单个consumer支持同时订阅多个topic。 - -五、启动订阅并开始消费 +## 启动订阅并开始消费 ```sql /* 启动订阅 */ @@ -196,7 +201,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; } ``` -五、结束消费 +## 结束消费 ```sql /* 取消订阅 */ @@ -206,7 +211,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; tmq_consumer_close(tmq); ``` -六、删除topic +## 删除topic 如果不再需要，可以删除创建topic，但注意：只有没有被订阅的topic才能别删除。 @@ -215,7 +220,7 @@ create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; drop topic topicName; ``` -七、状态查看 +## 状态查看 1、topics：查询已经创建的topic From ef849670aba9c0b5e23f7a0dc7b1583cb46f3a9b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 7 Aug 2022 13:36:06 +0800 Subject: [PATCH 45/51] Update 07-tmq.md --- docs/zh/07-develop/07-tmq.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index c39249e6d2..7faccdcec1 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -97,9 +97,9 @@ TMQ支持多种订阅类型： - 用户对于要处理的每一个数据块都可能有不同的schema，因此，必须重新获取schema - 返回数据不带有tag -## 创建 consumer +## 创建 consumer 以及consumer group -目前支持的config： +对于consumer, 目前支持的config包括： | 参数名称 | 参数值 | 备注 | | ---------------------------- | ------------------------------ | ------------------------------------------------------ | @@ -133,6 +133,9 @@ TMQ支持多种订阅类型： return tmq; ``` +上述配置中包括consumer group ID，如果多个 consumer 指定的 consumer group ID一样，则自动形成一个consumer group，共享消费进度。 + + ## 创建 topic 列表 单个consumer支持同时订阅多个topic。 From bf9b2ef8f0143f9707eff3851715ed21c185e4bb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 7 Aug 2022 14:59:17 +0800 Subject: [PATCH 46/51] enh: rsma fetch trigger --- include/common/tmsg.h | 25 +++++ source/dnode/vnode/src/sma/smaCommit.c | 69 ++++++++------ source/dnode/vnode/src/sma/smaEnv.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 122 ++++++++++++++++++++++--- 4 files changed, 178 insertions(+), 40 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3c3071c8df..54acc62216 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2657,6 +2657,31 @@ typedef struct { SEpSet epSet; } SVgEpSet; +typedef struct { + int64_t refId; + uint64_t handle; +} SRSmaFetchMsg; + +static FORCE_INLINE int32_t tEncodeSRSmaFetchMsg(SEncoder* pCoder, const SRSmaFetchMsg* pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->refId) < 0) return -1; + if (tEncodeU64(pCoder, pReq->handle) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static FORCE_INLINE int32_t tDecodeSRSmaFetchMsg(SDecoder* pCoder, SRSmaFetchMsg* pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->refId) < 0) return -1; + if (tDecodeU64(pCoder, &pReq->handle) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 3c88b6f5cb..8074c76806 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -308,12 +308,12 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); // step 1: set rsma stat @@ -337,18 +337,26 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } // step 3: swap rsmaInfoHash and iRsmaInfoHash - ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + ASSERT(RSMA_INFO_HASH(pRSmaStat)); + ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); RSMA_INFO_HASH(pRSmaStat) = taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + // step 4: others pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; @@ -383,44 +391,51 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); // step 1: merge rsmaInfoHash and iRsmaInfoHash - taosWLockLatch(SMA_ENV_LOCK(pSmaEnv)); + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { - // TODO: optimization - just switch the hash pointer if rsmaInfoHash is empty - } - - void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); - while (pIter) { - tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - - if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { - taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), - *pSuid); - } else { - // free the resources - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - tdFreeRSmaInfo(pSma, pRSmaInfo, false); - smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), - *pSuid); + // just switch the hash pointer if rsmaInfoHash is empty + if (taosHashGetSize(RSMA_IMU_INFO_HASH(pRSmaStat)) > 0) { + SHashObj *infoHash = RSMA_INFO_HASH(pRSmaStat); + RSMA_INFO_HASH(pRSmaStat) = RSMA_IMU_INFO_HASH(pRSmaStat); + RSMA_IMU_INFO_HASH(pRSmaStat) = infoHash; } + } else { + void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); + while (pIter) { + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { + taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } else { + // free the resources + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + tdFreeRSmaInfo(pSma, pRSmaInfo, false); + smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + } } taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; - taosWUnLockLatch(SMA_ENV_LOCK(pSmaEnv)); + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); // step 2: cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4b831225bc..5345d68c3b 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -17,7 +17,7 @@ typedef struct SSmaStat SSmaStat; -#define SMA_MGMT_REF_NUM 10240 +#define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 1c6d65c4d8..dadf92b322 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -38,6 +38,7 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType); static void tdRSmaFetchTrigger(void *param, void *tmrId); +static void tdRSmaFetchTrigger2(void *param, void *tmrId); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -567,6 +568,30 @@ static void tdDestroySDataBlockArray(SArray *pArray) { taosArrayDestroy(pArray); } +/** + * @brief retention of rsma1/rsma2 + * + * @param pSma + * @param now + * @return int32_t + */ +int32_t smaDoRetention(SSma *pSma, int64_t now) { + int32_t code = TSDB_CODE_SUCCESS; + if (VND_IS_RSMA(pSma->pVnode)) { + return code; + } + + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pSma->pRSmaTsdb[i]) { + code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + if (code) goto _end; + } + } + +_end: + return code; +} + static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType) { SArray *pResult = NULL; @@ -697,7 +722,7 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return pRSmaInfo; } - if (RSMA_COMMIT_STAT(pStat) == 0) { + if (RSMA_COMMIT_STAT(pStat) == 0) { // return NULL if not in committing stat return NULL; } @@ -705,23 +730,29 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SRSmaInfo *pCowRSmaInfo = NULL; // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) { // 2-phase lock + if (!(pCowRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)))) { // 2-phase lock void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (iRSmaInfo) { SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; if (pIRSmaInfo) { if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); return NULL; } smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid); if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); return NULL; } } } + } else { + pCowRSmaInfo = *(SRSmaInfo **)pCowRSmaInfo; + ASSERT(!pCowRSmaInfo); } // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -1365,19 +1396,86 @@ _end: tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); } -int32_t smaDoRetention(SSma *pSma, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; - if (VND_IS_RSMA(pSma->pVnode)) { - return code; +/** + * @brief trigger to get rsma result + * + * @param param + * @param tmrId + */ +static void tdRSmaFetchTrigger2(void *param, void *tmrId) { + SRSmaInfoItem *pItem = param; + SSma *pSma = NULL; + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + + if (!pStat) { + smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pItem->refId); + return; } - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); - if (code) goto _end; + pSma = pStat->pSma; + + // if rsma trigger stat in paused, cancelled or finished, not start fetch task + int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); + switch (rsmaTriggerStat) { + case TASK_TRIGGER_STAT_PAUSED: + case TASK_TRIGGER_STAT_CANCELLED: { + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 + " refId:%d", + SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, + &pItem->tmrId); + } + return; } + default: + break; + } + + SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + goto _end; + } + + int8_t fetchTriggerStat = + atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + switch (fetchTriggerStat) { + case TASK_TRIGGER_STAT_ACTIVE: { + smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), + pItem->level, pRSmaInfo->suid); + + // sync procedure => async process + tdRefRSmaInfo(pSma, pRSmaInfo); + + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + qSetMultiStreamInput(pItem->taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); + tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); + tdCleanupStreamInputDataBlock(pItem->taskInfo); + + tdUnRefRSmaInfo(pSma, pRSmaInfo); + // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId); + } break; + case TASK_TRIGGER_STAT_PAUSED: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + } break; + case TASK_TRIGGER_STAT_INACTIVE: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + } break; + case TASK_TRIGGER_STAT_INIT: { + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma), + pItem->level, pRSmaInfo->suid); + } break; + default: { + smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + } break; } _end: - return code; -} \ No newline at end of file + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); +} From 1c79b81a767a23f907244018d3471f1bb5a2ef8c Mon Sep 17 00:00:00 2001 From: tomchon Date: Sun, 7 Aug 2022 15:29:17 +0800 Subject: [PATCH 47/51] test: commet vnode testcases of replica 3 --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1f6e8ce1f5..0428014291 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -216,7 +216,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_query # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 From 963bc989e303abe58bf67d5fc31fbf614c3d4c04 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 7 Aug 2022 17:45:07 +0800 Subject: [PATCH 48/51] enh: rsma code optimization --- include/common/tmsg.h | 9 +- include/util/tlockfree.h | 6 +- source/dnode/vnode/src/inc/sma.h | 17 ++- source/dnode/vnode/src/sma/smaRollup.c | 146 ++++++++++++++----------- source/dnode/vnode/src/sma/smaUtil.c | 40 +++---- 5 files changed, 117 insertions(+), 101 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 54acc62216..716f51933e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2659,14 +2659,16 @@ typedef struct { typedef struct { int64_t refId; - uint64_t handle; + int64_t suid; + int8_t level; } SRSmaFetchMsg; static FORCE_INLINE int32_t tEncodeSRSmaFetchMsg(SEncoder* pCoder, const SRSmaFetchMsg* pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI64(pCoder, pReq->refId) < 0) return -1; - if (tEncodeU64(pCoder, pReq->handle) < 0) return -1; + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeI8(pCoder, pReq->level) < 0) return -1; tEndEncode(pCoder); return 0; @@ -2676,7 +2678,8 @@ static FORCE_INLINE int32_t tDecodeSRSmaFetchMsg(SDecoder* pCoder, SRSmaFetchMsg if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pReq->refId) < 0) return -1; - if (tDecodeU64(pCoder, &pReq->handle) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->level) < 0) return -1; tEndDecode(pCoder); return 0; diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 8db6be8860..82dd5a0e8b 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -25,9 +25,9 @@ extern "C" { // reference counting typedef void (*_ref_fn_t)(const void *pObj); -#define T_REF_DECLARE() \ - struct { \ - int32_t val; \ +#define T_REF_DECLARE() \ + struct { \ + volatile int32_t val; \ } _ref; #define T_REF_REGISTER_FUNC(s, e) \ diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1aee08027c..51711e6e97 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -115,8 +115,6 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - void *taskInfo; // qTaskInfo_t - int64_t refId; tmr_h tmrId; int32_t maxDelay; int8_t level; @@ -126,13 +124,20 @@ struct SRSmaInfoItem { struct SRSmaInfo { STSchema *pTSchema; int64_t suid; + int64_t refId; // refId of SRSmaStat int8_t delFlag; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t + void *iTaskInfo[TSDB_RETENTION_L2]; // immutable }; -#define RSMA_INFO_HEAD_LEN 24 -#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) -#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) + +#define RSMA_INFO_HEAD_LEN 32 +#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) +#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) +#define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) +#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) +#define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -223,7 +228,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc); +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc); void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index dadf92b322..ba14b657bf 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -32,13 +32,11 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *ui static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, - STSchema *pTSchema, tb_uid_t suid, int8_t level); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, int8_t level); static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); -static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static void tdRSmaFetchTrigger2(void *param, void *tmrId); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -116,17 +114,26 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; - if (pItem->taskInfo) { - if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, - pItem->tmrId, i + 1); - taosTmrStopA(&pItem->tmrId); - } - tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); + + if (isDeepFree && pItem->tmrId) { + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, + pItem->tmrId, i + 1); + taosTmrStopA(&pItem->tmrId); + } + + if (isDeepFree && pInfo->taskInfo[i]) { + tdFreeQTaskInfo(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); } + + if (pInfo->iTaskInfo[i]) { + tdFreeQTaskInfo(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); + } else { + smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", + SMA_VID(pSma), pInfo->suid, i + 1); + } } if (isDeepFree) { taosMemoryFreeClear(pInfo->pTSchema); @@ -156,6 +163,11 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_FAILED; } + if (!taosArrayGetSize(tbUids)) { + smaDebug("vgId:%d, no need to update tbUidList for suid:%" PRIi64 " since Empty tbUids", SMA_VID(pSma), *suid); + return TSDB_CODE_SUCCESS; + } + pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid); if (!pRSmaInfo) { @@ -164,23 +176,16 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_FAILED; } - if (pRSmaInfo->items[0].taskInfo) { - if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - } - - if (pRSmaInfo->items[1].taskInfo) { - if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pRSmaInfo->taskInfo[i]) { + if ((qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, + terrstr()); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); + } } } @@ -268,13 +273,12 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat .initTqReader = 1, }; - SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - pItem->refId = RSMA_REF_ID(pStat); - pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); - if (!pItem->taskInfo) { + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; return TSDB_CODE_FAILED; } + SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = @@ -343,6 +347,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; + pRSmaInfo->refId = RSMA_REF_ID(pStat); T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { @@ -592,7 +597,7 @@ _end: return code; } -static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType) { SArray *pResult = NULL; SSma *pSma = pStat->pSma; @@ -601,7 +606,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSDataBlock *output = NULL; uint64_t ts; - int32_t code = qExecTask(pItem->taskInfo, &output, &ts); + int32_t code = qExecTask(taskInfo, &output, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); @@ -662,29 +667,32 @@ _err: return TSDB_CODE_FAILED; } -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, - STSchema *pTSchema, tb_uid_t suid, int8_t level) { - if (!pItem || !pItem->taskInfo) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, + int8_t level) { + int32_t idx = level - 1; + if (!pInfo || !RSMA_INFO_QTASK(pInfo, idx)) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; } - if (!pTSchema) { + if (!pInfo->pTSchema) { smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, - pItem->taskInfo, suid); + RSMA_INFO_QTASK(pInfo, idx), suid); - if (qSetMultiStreamInput(pItem->taskInfo, pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT + if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT); + tdRSmaFetchAndSubmitResult(RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid, pStat, + STREAM_INPUT__DATA_SUBMIT); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); if (smaMgmt.tmrHandle) { @@ -735,7 +743,7 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { if (iRSmaInfo) { SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; if (pIRSmaInfo) { - if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + if (tdCloneRSmaInfo(pSma, &pCowRSmaInfo, pIRSmaInfo) < 0) { // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); @@ -780,6 +788,9 @@ void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { return; } + // unlock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { @@ -788,6 +799,8 @@ void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid); } + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); } static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { @@ -797,7 +810,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - if (!pRSmaInfo->items[0].taskInfo) { + if (!RSMA_INFO_QTASK(pRSmaInfo,0)) { smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } @@ -805,8 +818,8 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb if (inputType == STREAM_INPUT__DATA_SUBMIT) { tdRefRSmaInfo(pSma, pRSmaInfo); - tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1); - tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L1); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L2); tdUnRefRSmaInfo(pSma, pRSmaInfo); } @@ -1028,9 +1041,9 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * } if (pItem->type == TSDB_RETENTION_L1) { - qTaskInfo = pRSmaInfo->items[0].taskInfo; + qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 0); } else if (pItem->type == TSDB_RETENTION_L2) { - qTaskInfo = pRSmaInfo->items[1].taskInfo; + qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 1); } else { ASSERT(0); } @@ -1227,7 +1240,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; + qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); if (!taskInfo) { smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; @@ -1312,6 +1325,7 @@ _err: return TSDB_CODE_FAILED; } +#if 0 /** * @brief trigger to get rsma result * @@ -1395,6 +1409,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { _end: tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); } +#endif /** * @brief trigger to get rsma result @@ -1402,14 +1417,20 @@ _end: * @param param * @param tmrId */ -static void tdRSmaFetchTrigger2(void *param, void *tmrId) { +static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; SSma *pSma = NULL; - SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + return; + } + + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); if (!pStat) { smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, - pItem->refId); + pRSmaInfo->refId); return; } @@ -1420,10 +1441,10 @@ static void tdRSmaFetchTrigger2(void *param, void *tmrId) { switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: { - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", - SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); + SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaInfo->refId); if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); @@ -1434,11 +1455,6 @@ static void tdRSmaFetchTrigger2(void *param, void *tmrId) { break; } - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - goto _end; - } - int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (fetchTriggerStat) { @@ -1450,9 +1466,11 @@ static void tdRSmaFetchTrigger2(void *param, void *tmrId) { tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetMultiStreamInput(pItem->taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); - tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); - tdCleanupStreamInputDataBlock(pItem->taskInfo); + qTaskInfo_t taskInfo = pRSmaInfo->taskInfo[pItem->level - 1]; + qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); + tdRSmaFetchAndSubmitResult(taskInfo, pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, + STREAM_INPUT__DATA_BLOCK); + tdCleanupStreamInputDataBlock(taskInfo); tdUnRefRSmaInfo(pSma, pRSmaInfo); // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); @@ -1477,5 +1495,5 @@ static void tdRSmaFetchTrigger2(void *param, void *tmrId) { } _end: - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 9dce166afa..93d102dc5b 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -349,29 +349,20 @@ _err: /** * @brief pTSchema is shared - * - * @param pSma - * @param pDest - * @param pSrc - * @return int32_t + * + * @param pSma + * @param pDest + * @param pSrc + * @return int32_t */ -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc) { SVnode *pVnode = pSma->pVnode; SRSmaParam *param = NULL; if (!pSrc) { + *pDest = NULL; return TSDB_CODE_SUCCESS; } - if (!pDest) { - pDest = taosMemoryCalloc(1, sizeof(SRSmaInfo)); - if (!pDest) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - - memcpy(pDest, pSrc, sizeof(SRSmaInfo)); - SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid); @@ -384,21 +375,20 @@ int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { ASSERT(mr.me.uid == pSrc->suid); if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; - for (int i = 0; i < TSDB_RETENTION_L2; ++i) { - SRSmaInfoItem *pItem = &pSrc->items[i]; - if (pItem->taskInfo) { - tdCloneQTaskInfo(pSma, pDest->items[i].taskInfo, pItem->taskInfo, param, pSrc->suid, i); - } + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i); } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); } metaReaderClear(&mr); + + *pDest = pSrc; // pointer copy + return TSDB_CODE_SUCCESS; _err: + *pDest = NULL; metaReaderClear(&mr); - tdFreeRSmaInfo(pSma, pDest, false); + // tdFreeRSmaInfo(pSma, pDest, false); return TSDB_CODE_FAILED; -} - -// ... \ No newline at end of file +} \ No newline at end of file From c2c7ccf776118d55255d3463a8ccb5016179d770 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 7 Aug 2022 20:06:31 +0800 Subject: [PATCH 49/51] enh: rsma info lifecyle mgmt --- source/dnode/vnode/src/inc/sma.h | 9 +- source/dnode/vnode/src/sma/smaCommit.c | 54 +++++-- source/dnode/vnode/src/sma/smaEnv.c | 5 +- source/dnode/vnode/src/sma/smaRollup.c | 207 +++++++------------------ source/dnode/vnode/src/sma/smaUtil.c | 30 ++-- 5 files changed, 117 insertions(+), 188 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 51711e6e97..944d7759b2 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -115,10 +115,10 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - tmr_h tmrId; - int32_t maxDelay; int8_t level; int8_t triggerStat; + int32_t maxDelay; + tmr_h tmrId; }; struct SRSmaInfo { @@ -173,8 +173,8 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); -int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId); +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId); int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); @@ -233,7 +233,6 @@ void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 8074c76806..373cfdfb47 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -402,7 +402,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { // step 1: merge rsmaInfoHash and iRsmaInfoHash // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - +#if 0 if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { // just switch the hash pointer if rsmaInfoHash is empty if (taosHashGetSize(RSMA_IMU_INFO_HASH(pRSmaStat)) > 0) { @@ -411,25 +411,47 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { RSMA_IMU_INFO_HASH(pRSmaStat) = infoHash; } } else { - void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); - while (pIter) { - tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); +#endif +#if 1 + void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); + while (pIter) { + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { - taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), - *pSuid); - } else { - // free the resources - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - tdFreeRSmaInfo(pSma, pRSmaInfo, false); - smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), - *pSuid); + if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + int32_t refVal = T_REF_VAL_GET(pRSmaInfo); + if (refVal == 0) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + smaDebug( + "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " + "table:%" PRIi64, + SMA_VID(pSma), *pSuid); + } else { + smaDebug( + "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " + "table:%" PRIi64, + SMA_VID(pSma), refVal, *pSuid); + } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + continue; } - - pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } else { + // free the resources + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + tdFreeRSmaInfo(pSma, pRSmaInfo, false); + smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), + *pSuid); } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); } +#endif + // } taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 5345d68c3b..ccb6ad3a72 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -171,7 +171,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { if (!pRSmaInfo) return 0; - + int ref = T_REF_INC(pRSmaInfo); smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); return 0; @@ -183,9 +183,6 @@ int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int ref = T_REF_DEC(pRSmaInfo); smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); - if (ref == 0) { - tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid); - } return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ba14b657bf..fd2222c5e4 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -32,11 +32,14 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *ui static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, int8_t level); -static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); -static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, - int8_t blkType); -static void tdRSmaFetchTrigger(void *param, void *tmrId); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, + int8_t level); +static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); +static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); + +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, + SRSmaStat *pStat, int8_t blkType); +static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -114,7 +117,7 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; - + if (isDeepFree && pItem->tmrId) { smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, i + 1); @@ -168,7 +171,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_SUCCESS; } - pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid); + pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, *suid); if (!pRSmaInfo) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); @@ -179,6 +182,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { if ((qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true) < 0)) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); return TSDB_CODE_FAILED; @@ -189,6 +193,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) } } + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -417,7 +422,7 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid); + SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pReq->suid); if (!pRSmaInfo) { smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name, @@ -429,8 +434,10 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { RSMA_INFO_SET_DEL(pRSmaInfo); tdUnRefRSmaInfo(pSma, pRSmaInfo); - // save to file + tdReleaseRSmaInfo(pSma, pRSmaInfo); + // save to file + // TODO smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -575,10 +582,10 @@ static void tdDestroySDataBlockArray(SArray *pArray) { /** * @brief retention of rsma1/rsma2 - * - * @param pSma - * @param now - * @return int32_t + * + * @param pSma + * @param now + * @return int32_t */ int32_t smaDoRetention(SSma *pSma, int64_t now) { int32_t code = TSDB_CODE_SUCCESS; @@ -597,8 +604,8 @@ _end: return code; } -static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, - int8_t blkType) { +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, + SRSmaStat *pStat, int8_t blkType) { SArray *pResult = NULL; SSma *pSma = pStat->pSma; @@ -711,7 +718,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType * @param suid * @return SRSmaInfo* */ -static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { +static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; @@ -725,14 +732,23 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } + taosRLockLatch(SMA_ENV_LOCK(pEnv)); pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } + tdRefRSmaInfo(pSma, pRSmaInfo); + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return pRSmaInfo; } if (RSMA_COMMIT_STAT(pStat) == 0) { // return NULL if not in committing stat + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat SRSmaInfo *pCowRSmaInfo = NULL; @@ -742,7 +758,7 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (iRSmaInfo) { SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; - if (pIRSmaInfo) { + if (pIRSmaInfo && !RSMA_INFO_IS_DEL(pIRSmaInfo)) { if (tdCloneRSmaInfo(pSma, &pCowRSmaInfo, pIRSmaInfo) < 0) { // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -762,68 +778,40 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { pCowRSmaInfo = *(SRSmaInfo **)pCowRSmaInfo; ASSERT(!pCowRSmaInfo); } + + if(pCowRSmaInfo) { + tdRefRSmaInfo(pSma, pCowRSmaInfo); + } // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); return pCowRSmaInfo; } -/** - * @brief During the drop procedure, only need to delete the object in rsmaInfoHash. - * - * @param pSma - * @param suid - * @return SRSmaInfo* - */ -void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SRSmaInfo *pRSmaInfo = NULL; - - if (!pEnv) { - return; +static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { + if (pInfo) { + tdUnRefRSmaInfo(pSma, pInfo); } - - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - if (!pStat || !RSMA_INFO_HASH(pStat)) { - return; - } - - // unlock - taosWLockLatch(SMA_ENV_LOCK(pEnv)); - - pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (pRSmaInfo) { - if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tdFreeRSmaInfo(pSma, pRSmaInfo, true); - } - taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid); - } - // unlock - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); } static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid); + SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } - if (!RSMA_INFO_QTASK(pRSmaInfo,0)) { + if (!RSMA_INFO_QTASK(pRSmaInfo, 0)) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - tdRefRSmaInfo(pSma, pRSmaInfo); - tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L2); - - tdUnRefRSmaInfo(pSma, pRSmaInfo); } + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -1034,7 +1022,7 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * SRSmaInfo *pRSmaInfo = NULL; void *qTaskInfo = NULL; - pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid); + pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pItem->suid); if (!pRSmaInfo) { smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid); return TSDB_CODE_SUCCESS; @@ -1049,11 +1037,13 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * } if (!qTaskInfo) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid); return TSDB_CODE_SUCCESS; } if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, pItem->type, terrstr()); return TSDB_CODE_FAILED; @@ -1061,6 +1051,7 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type); + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -1239,6 +1230,12 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + infoHash = taosHashIterate(pInfoHash, infoHash); + continue; + } + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); if (!taskInfo) { @@ -1325,92 +1322,6 @@ _err: return TSDB_CODE_FAILED; } -#if 0 -/** - * @brief trigger to get rsma result - * - * @param param - * @param tmrId - */ -static void tdRSmaFetchTrigger(void *param, void *tmrId) { - SRSmaInfoItem *pItem = param; - SSma *pSma = NULL; - SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); - - if (!pStat) { - smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, - pItem->refId); - return; - } - - pSma = pStat->pSma; - - // if rsma trigger stat in paused, cancelled or finished, not start fetch task - int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); - switch (rsmaTriggerStat) { - case TASK_TRIGGER_STAT_PAUSED: - case TASK_TRIGGER_STAT_CANCELLED: { - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 - " refId:%d", - SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); - if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, - &pItem->tmrId); - } - return; - } - default: - break; - } - - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - goto _end; - } - - int8_t fetchTriggerStat = - atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); - switch (fetchTriggerStat) { - case TASK_TRIGGER_STAT_ACTIVE: { - smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), - pItem->level, pRSmaInfo->suid); - - // sync procedure => async process - tdRefRSmaInfo(pSma, pRSmaInfo); - - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetMultiStreamInput(pItem->taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); - tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); - tdCleanupStreamInputDataBlock(pItem->taskInfo); - - tdUnRefRSmaInfo(pSma, pRSmaInfo); - // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - } break; - case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - } break; - case TASK_TRIGGER_STAT_INACTIVE: { - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - } break; - case TASK_TRIGGER_STAT_INIT: { - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma), - pItem->level, pRSmaInfo->suid); - } break; - default: { - smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - } break; - } - -_end: - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); -} -#endif - /** * @brief trigger to get rsma result * @@ -1426,7 +1337,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { return; } - SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); if (!pStat) { smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, @@ -1441,7 +1352,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: { - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaInfo->refId); @@ -1463,7 +1374,6 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { pItem->level, pRSmaInfo->suid); // sync procedure => async process - tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; qTaskInfo_t taskInfo = pRSmaInfo->taskInfo[pItem->level - 1]; @@ -1472,9 +1382,6 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { STREAM_INPUT__DATA_BLOCK); tdCleanupStreamInputDataBlock(taskInfo); - tdUnRefRSmaInfo(pSma, pRSmaInfo); - // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", @@ -1495,5 +1402,5 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } _end: - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 93d102dc5b..d9f38ffd09 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -287,22 +287,22 @@ int32_t tdRemoveTFile(STFile *pTFile) { } // smaXXXUtil ================ -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); if (!pResult) { - smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + smaWarn("rsma acquire ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); } else { - smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + smaDebug("rsma acquire ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); } return pResult; } -int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { - smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + smaWarn("rsma release ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + smaDebug("rsma release ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); return TSDB_CODE_SUCCESS; } @@ -313,7 +313,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t char *pOutput = NULL; int32_t len = 0; - if (qSerializeTaskStatus(srcTaskInfo, &pOutput, &len) < 0) { + if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, terrstr()); goto _err; @@ -337,13 +337,15 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t goto _err; } - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); + smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); taosMemoryFreeClear(pOutput); return TSDB_CODE_SUCCESS; _err: taosMemoryFreeClear(pOutput); tdFreeQTaskInfo(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); return TSDB_CODE_FAILED; } @@ -376,19 +378,21 @@ int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc) { if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i); + if (tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i) < 0) { + goto _err; + } } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); } metaReaderClear(&mr); - - *pDest = pSrc; // pointer copy - + + *pDest = pSrc; // pointer copy + return TSDB_CODE_SUCCESS; _err: *pDest = NULL; metaReaderClear(&mr); - // tdFreeRSmaInfo(pSma, pDest, false); + smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, terrstr()); return TSDB_CODE_FAILED; } \ No newline at end of file From d95b465d1e71f3ef17c67b7433141e815437b52f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 7 Aug 2022 20:06:43 +0800 Subject: [PATCH 50/51] fix: add make install to jenkinsfile so that udf can pass CI --- tests/parallel_test/container_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index deaea4fa40..a4538c90e4 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -52,7 +52,7 @@ fi docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT;make install" ret=$? exit $ret From 7ae3333c3018fe802221ab4b27b49a7305db33c4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 8 Aug 2022 08:37:29 +0800 Subject: [PATCH 51/51] fix: compile_udf.sh outside of tdengine --- tests/parallel_test/container_build.sh | 2 +- tests/script/sh/compile_udf.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index a4538c90e4..deaea4fa40 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -52,7 +52,7 @@ fi docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT;make install" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT" ret=$? exit $ret diff --git a/tests/script/sh/compile_udf.sh b/tests/script/sh/compile_udf.sh index 12e922b2df..5ff3f2bc8a 100755 --- a/tests/script/sh/compile_udf.sh +++ b/tests/script/sh/compile_udf.sh @@ -3,8 +3,8 @@ set +e rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so mkdir -p /tmp/udf echo "compile udf bit_and and sqr_sum" -gcc -fPIC -shared sh/bit_and.c -o /tmp/udf/libbitand.so -gcc -fPIC -shared sh/sqr_sum.c -o /tmp/udf/libsqrsum.so +gcc -fPIC -shared sh/bit_and.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libbitand.so +gcc -fPIC -shared sh/sqr_sum.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libsqrsum.so echo "debug show /tmp/udf/*.so" ls /tmp/udf/*.so